visualtest/src/testharness.c

/* See COPYING.txt for the full license governing this code. */
/**
 *  \file testharness.c 
 *
 *  Source file for the test harness.
 */

#include <stdlib.h>
#include <SDL_test.h>
#include <SDL.h>
#include <SDL_assert.h>
#include "SDL_visualtest_harness_argparser.h"
#include "SDL_visualtest_process.h"
#include "SDL_visualtest_variators.h"
#include "SDL_visualtest_screenshot.h"
#include "SDL_visualtest_mischelper.h"

#if defined(__WIN32__) && !defined(__CYGWIN__)
#include <direct.h>
#elif defined(__WIN32__) && defined(__CYGWIN__)
#include <signal.h>
#elif defined(__LINUX__)
#include <sys/stat.h>
#include <sys/types.h>
#include <signal.h>
#else
#error "Unsupported platform"
#endif

/** Code for the user event triggered when a new action is to be executed */
#define ACTION_TIMER_EVENT 0
/** Code for the user event triggered when the maximum timeout is reached */
#define KILL_TIMER_EVENT 1
/** FPS value used for delays in the action loop */
#define ACTION_LOOP_FPS 10

/** Value returned by RunSUTAndTest() when the test has passed */
#define TEST_PASSED 1
/** Value returned by RunSUTAndTest() when the test has failed */
#define TEST_FAILED 0
/** Value returned by RunSUTAndTest() on a fatal error */
#define TEST_ERROR -1

static SDL_ProcessInfo pinfo;
static SDL_ProcessExitStatus sut_exitstatus;
static SDLVisualTest_HarnessState state;
static SDLVisualTest_Variator variator;
static SDLVisualTest_ActionNode* current; /* the current action being performed */
static SDL_TimerID action_timer, kill_timer;

/* returns a char* to be passed as the format argument of a printf-style function. */
static char*
usage()
{
    return "Usage: \n%s --sutapp xyz"
           " [--sutargs abc | --parameter-config xyz.parameters"
           " [--variator exhaustive|random]" 
           " [--num-variations N] [--no-launch]] [--timeout hh:mm:ss]"
           " [--action-config xyz.actions]"
           " [--output-dir /path/to/output]"
           " [--verify-dir /path/to/verify]"
           " or --config app.config";
}

/* register Ctrl+C handlers */
#if defined(__LINUX__) || defined(__CYGWIN__)
static void
CtrlCHandlerCallback(int signum)
{
    SDL_Event event;
    SDLTest_Log("Ctrl+C received");
    event.type = SDL_QUIT;
    SDL_PushEvent(&event);
}
#endif

static Uint32
ActionTimerCallback(Uint32 interval, void* param)
{
    SDL_Event event;
    SDL_UserEvent userevent;
    Uint32 next_action_time;

    /* push an event to handle the action */
    userevent.type = SDL_USEREVENT;
    userevent.code = ACTION_TIMER_EVENT;
    userevent.data1 = &current->action;
    userevent.data2 = NULL;

    event.type = SDL_USEREVENT;
    event.user = userevent;
    SDL_PushEvent(&event);

    /* calculate the new interval and return it */
    if(current->next)
        next_action_time = current->next->action.time - current->action.time;
    else
    {
        next_action_time = 0;
        action_timer = 0;
    }

    current = current->next;
    return next_action_time;
}

static Uint32
KillTimerCallback(Uint32 interval, void* param)
{
    SDL_Event event;
    SDL_UserEvent userevent;

    userevent.type = SDL_USEREVENT;
    userevent.code = KILL_TIMER_EVENT;
    userevent.data1 = NULL;
    userevent.data2 = NULL;

    event.type = SDL_USEREVENT;
    event.user = userevent;
    SDL_PushEvent(&event);

    kill_timer = 0;
    return 0;
}

static int
ProcessAction(SDLVisualTest_Action* action, int* sut_running, char* args)
{
    if(!action || !sut_running)
        return TEST_ERROR;

    switch(action->type)
    {
        case SDL_ACTION_KILL:
            SDLTest_Log("Action: Kill SUT");
            if(SDL_IsProcessRunning(&pinfo) == 1 &&
               !SDL_KillProcess(&pinfo, &sut_exitstatus))
            {
                SDLTest_LogError("SDL_KillProcess() failed");
                return TEST_ERROR;
            }
            *sut_running = 0;
        break;

        case SDL_ACTION_QUIT:
            SDLTest_Log("Action: Quit SUT");
            if(SDL_IsProcessRunning(&pinfo) == 1 &&
               !SDL_QuitProcess(&pinfo, &sut_exitstatus))
            {
                SDLTest_LogError("SDL_QuitProcess() failed");
                return TEST_FAILED;
            }
            *sut_running = 0;
        break;

        case SDL_ACTION_LAUNCH:
        {
            char* path;
            char* args;
            SDL_ProcessInfo action_process;
            SDL_ProcessExitStatus ps;

            path = action->extra.process.path;
            args = action->extra.process.args;
            if(args)
            {
                SDLTest_Log("Action: Launch process: %s with arguments: %s",
                            path, args);
            }
            else
                SDLTest_Log("Action: Launch process: %s", path);
            if(!SDL_LaunchProcess(path, args, &action_process))
            {
                SDLTest_LogError("SDL_LaunchProcess() failed");
                return TEST_ERROR;
            }

            /* small delay so that the process can do its job */
            SDL_Delay(1000);

            if(SDL_IsProcessRunning(&action_process) > 0)
            {
                SDLTest_LogError("Process %s took too long too complete."
                                    " Force killing...", action->extra);
                if(!SDL_KillProcess(&action_process, &ps))
                {
                    SDLTest_LogError("SDL_KillProcess() failed");
                    return TEST_ERROR;
                }
            }
        }
        break;

        case SDL_ACTION_SCREENSHOT:
        {
            char path[MAX_PATH_LEN], hash[33];

            SDLTest_Log("Action: Take screenshot");
            /* can't take a screenshot if the SUT isn't running */
            if(SDL_IsProcessRunning(&pinfo) != 1)
            {
                SDLTest_LogError("SUT has quit.");
                *sut_running = 0;
                return TEST_FAILED;
            }

            /* file name for the screenshot image */
            SDLVisualTest_HashString(args, hash);
            SDL_snprintf(path, MAX_PATH_LEN, "%s/%s", state.output_dir, hash);
            if(!SDLVisualTest_ScreenshotProcess(&pinfo, path))
            {
                SDLTest_LogError("SDLVisualTest_ScreenshotProcess() failed");
                return TEST_ERROR;
            }
        }
        break;

        case SDL_ACTION_VERIFY:
        {
            int ret;

            SDLTest_Log("Action: Verify screenshot");
            ret = SDLVisualTest_VerifyScreenshots(args, state.output_dir,
                                                  state.verify_dir);

            if(ret == -1)
            {
                SDLTest_LogError("SDLVisualTest_VerifyScreenshots() failed");
                return TEST_ERROR;
            }
            else if(ret == 0)
            {
                SDLTest_Log("Verification failed: Images were not equal.");
                return TEST_FAILED;
            }
            else if(ret == 1)
                SDLTest_Log("Verification successful.");
            else
            {
                SDLTest_Log("Verfication skipped.");
                return TEST_FAILED;
            }
        }
        break;

        default:
            SDLTest_LogError("Invalid action type");
            return TEST_ERROR;
        break;
    }

    return TEST_PASSED;
}

static int
RunSUTAndTest(char* sutargs, int variation_num)
{
    int success, sut_running, return_code;
    char hash[33];
    SDL_Event event;

    return_code = TEST_PASSED;

    if(!sutargs)
    {
        SDLTest_LogError("sutargs argument cannot be NULL");
        return_code = TEST_ERROR;
        goto runsutandtest_cleanup_generic;
    }

    SDLVisualTest_HashString(sutargs, hash);
    SDLTest_Log("Hash: %s", hash);

    success = SDL_LaunchProcess(state.sutapp, sutargs, &pinfo);
    if(!success)
    {
        SDLTest_Log("Could not launch SUT.");
        return_code = TEST_ERROR;
        goto runsutandtest_cleanup_generic;
    }
    SDLTest_Log("SUT launch successful.");
    SDLTest_Log("Process will be killed in %d milliseconds", state.timeout);
    sut_running = 1;

    /* launch the timers */
    SDLTest_Log("Performing actions..");
    current = state.action_queue.front;
    action_timer = 0;
    kill_timer = 0;
    if(current)
    {
        action_timer = SDL_AddTimer(current->action.time, ActionTimerCallback, NULL);
        if(!action_timer)
        {
            SDLTest_LogError("SDL_AddTimer() failed");
            return_code = TEST_ERROR;
            goto runsutandtest_cleanup_timer;
        }
    }
    kill_timer = SDL_AddTimer(state.timeout, KillTimerCallback, NULL);
    if(!kill_timer)
    {
        SDLTest_LogError("SDL_AddTimer() failed");
        return_code = TEST_ERROR;
        goto runsutandtest_cleanup_timer;
    }

    /* the timer stops running if the actions queue is empty, and the
       SUT stops running if it crashes or if we encounter a KILL/QUIT action */
    while(sut_running)
    {
        /* process the actions by using an event queue */
        while(SDL_PollEvent(&event))
        {
            if(event.type == SDL_USEREVENT)
            {
                if(event.user.code == ACTION_TIMER_EVENT)
                {
                    SDLVisualTest_Action* action;

                    action = (SDLVisualTest_Action*)event.user.data1;

                    switch(ProcessAction(action, &sut_running, sutargs))
                    {
                        case TEST_PASSED:
                        break;

                        case TEST_FAILED:
                            return_code = TEST_FAILED;
                            goto runsutandtest_cleanup_timer;
                        break;

                        default:
                            SDLTest_LogError("ProcessAction() failed");
                            return_code = TEST_ERROR;
                            goto runsutandtest_cleanup_timer;
                    }
                }
                else if(event.user.code == KILL_TIMER_EVENT)
                {
                    SDLTest_LogError("Maximum timeout reached. Force killing..");
                    return_code = TEST_FAILED;
                    goto runsutandtest_cleanup_timer;
                }
            }
            else if(event.type == SDL_QUIT)
            {
                SDLTest_LogError("Received QUIT event. Testharness is quitting..");
                return_code = TEST_ERROR;
                goto runsutandtest_cleanup_timer;
            }
        }
        SDL_Delay(1000/ACTION_LOOP_FPS);
    }

    SDLTest_Log("SUT exit code was: %d", sut_exitstatus.exit_status);
    if(sut_exitstatus.exit_status == 0)
    {
        return_code = TEST_PASSED;
        goto runsutandtest_cleanup_timer;
    }
    else
    {
        return_code = TEST_FAILED;
        goto runsutandtest_cleanup_timer;
    }

    return_code = TEST_ERROR;
    goto runsutandtest_cleanup_generic;

runsutandtest_cleanup_timer:
    if(action_timer && !SDL_RemoveTimer(action_timer))
    {
        SDLTest_Log("SDL_RemoveTimer() failed");
        return_code = TEST_ERROR;
    }

    if(kill_timer && !SDL_RemoveTimer(kill_timer))
    {
        SDLTest_Log("SDL_RemoveTimer() failed");
        return_code = TEST_ERROR;
    }
/* runsutandtest_cleanup_process: */
    if(SDL_IsProcessRunning(&pinfo) && !SDL_KillProcess(&pinfo, &sut_exitstatus))
    {
        SDLTest_Log("SDL_KillProcess() failed");
        return_code = TEST_ERROR;
    }
runsutandtest_cleanup_generic:
    return return_code;
}

/** Entry point for testharness */
int
main(int argc, char* argv[])
{
    int i, passed, return_code, failed;

    /* freeing resources, linux style! */
    return_code = 0;

    if(argc < 2)
    {
        SDLTest_Log(usage(), argv[0]);
        goto cleanup_generic;
    }

#if defined(__LINUX__) || defined(__CYGWIN__)
    signal(SIGINT, CtrlCHandlerCallback);
#endif

    /* parse arguments */
    if(!SDLVisualTest_ParseHarnessArgs(argv + 1, &state))
    {
        SDLTest_Log(usage(), argv[0]);
        return_code = 1;
        goto cleanup_generic;
    }
    SDLTest_Log("Parsed harness arguments successfully.");

    /* initialize SDL */
    if(SDL_Init(SDL_INIT_TIMER) == -1)
    {
        SDLTest_LogError("SDL_Init() failed.");
        SDLVisualTest_FreeHarnessState(&state);
        return_code = 1;
        goto cleanup_harness_state;
    }

    /* create an output directory if none exists */
#if defined(__LINUX__) || defined(__CYGWIN__)
    mkdir(state.output_dir, 0777);
#elif defined(__WIN32__)
    _mkdir(state.output_dir);
#else
#error "Unsupported platform"
#endif

    /* test with sutargs */
    if(SDL_strlen(state.sutargs))
    {
        SDLTest_Log("Running: %s %s", state.sutapp, state.sutargs);
        if(!state.no_launch)
        {
            switch(RunSUTAndTest(state.sutargs, 0))
            {
                case TEST_PASSED:
                    SDLTest_Log("Status: PASSED");
                break;

                case TEST_FAILED:
                    SDLTest_Log("Status: FAILED");
                break;

                case TEST_ERROR:
                    SDLTest_LogError("Some error occurred while testing.");
                    return_code = 1;
                    goto cleanup_sdl;
                break;
            }
        }
    }

    if(state.sut_config.num_options > 0)
    {
        char* variator_name = state.variator_type == SDL_VARIATOR_RANDOM ?
                              "RANDOM" : "EXHAUSTIVE";
        if(state.num_variations > 0)
            SDLTest_Log("Testing SUT with variator: %s for %d variations",
                        variator_name, state.num_variations);
        else
            SDLTest_Log("Testing SUT with variator: %s and ALL variations",
                        variator_name);
        /* initialize the variator */
        if(!SDLVisualTest_InitVariator(&variator, &state.sut_config,
                                       state.variator_type, 0))
        {
            SDLTest_LogError("Could not initialize variator");
            return_code = 1;
            goto cleanup_sdl;
        }

        /* iterate through all the variations */
        passed = 0;
        failed = 0;
        for(i = 0; state.num_variations > 0 ? (i < state.num_variations) : 1; i++)
        {
            char* args = SDLVisualTest_GetNextVariation(&variator);
            if(!args)
                break;
            SDLTest_Log("\nVariation number: %d\nArguments: %s", i + 1, args);

            if(!state.no_launch)
            {
                switch(RunSUTAndTest(args, i + 1))
                {
                    case TEST_PASSED:
                        SDLTest_Log("Status: PASSED");
                        passed++;
                    break;

                    case TEST_FAILED:
                        SDLTest_Log("Status: FAILED");
                        failed++;
                    break;

                    case TEST_ERROR:
                        SDLTest_LogError("Some error occurred while testing.");
                        goto cleanup_variator;
                    break;
                }
            }
        }
        if(!state.no_launch)
        {
            /* report stats */
            SDLTest_Log("Testing complete.");
            SDLTest_Log("%d/%d tests passed.", passed, passed + failed);
        }
        goto cleanup_variator;
    }
 
    goto cleanup_sdl;

cleanup_variator:
    SDLVisualTest_FreeVariator(&variator);
cleanup_sdl:
    SDL_Quit();
cleanup_harness_state:
    SDLVisualTest_FreeHarnessState(&state);
cleanup_generic:
    return return_code;
}