| /* See COPYING.txt for the full license governing this code. */ |
| /** |
| * \file testharness.c |
| * |
| * Source file for the test harness. |
| */ |
| |
| #include <stdlib.h> |
| #include <SDL_test.h> |
| #include <SDL.h> |
| #include <SDL_assert.h> |
| #include "SDL_visualtest_harness_argparser.h" |
| #include "SDL_visualtest_process.h" |
| #include "SDL_visualtest_variators.h" |
| #include "SDL_visualtest_screenshot.h" |
| #include "SDL_visualtest_mischelper.h" |
| |
| #if defined(__WIN32__) && !defined(__CYGWIN__) |
| #include <direct.h> |
| #elif defined(__WIN32__) && defined(__CYGWIN__) |
| #include <signal.h> |
| #elif defined(__LINUX__) |
| #include <sys/stat.h> |
| #include <sys/types.h> |
| #include <signal.h> |
| #else |
| #error "Unsupported platform" |
| #endif |
| |
| /** Code for the user event triggered when a new action is to be executed */ |
| #define ACTION_TIMER_EVENT 0 |
| /** Code for the user event triggered when the maximum timeout is reached */ |
| #define KILL_TIMER_EVENT 1 |
| /** FPS value used for delays in the action loop */ |
| #define ACTION_LOOP_FPS 10 |
| |
| /** Value returned by RunSUTAndTest() when the test has passed */ |
| #define TEST_PASSED 1 |
| /** Value returned by RunSUTAndTest() when the test has failed */ |
| #define TEST_FAILED 0 |
| /** Value returned by RunSUTAndTest() on a fatal error */ |
| #define TEST_ERROR -1 |
| |
| static SDL_ProcessInfo pinfo; |
| static SDL_ProcessExitStatus sut_exitstatus; |
| static SDLVisualTest_HarnessState state; |
| static SDLVisualTest_Variator variator; |
| static SDLVisualTest_ActionNode* current; /* the current action being performed */ |
| static SDL_TimerID action_timer, kill_timer; |
| |
| /* returns a char* to be passed as the format argument of a printf-style function. */ |
| static char* |
| usage() |
| { |
| return "Usage: \n%s --sutapp xyz" |
| " [--sutargs abc | --parameter-config xyz.parameters" |
| " [--variator exhaustive|random]" |
| " [--num-variations N] [--no-launch]] [--timeout hh:mm:ss]" |
| " [--action-config xyz.actions]" |
| " [--output-dir /path/to/output]" |
| " [--verify-dir /path/to/verify]" |
| " or --config app.config"; |
| } |
| |
| /* register Ctrl+C handlers */ |
| #if defined(__LINUX__) || defined(__CYGWIN__) |
| static void |
| CtrlCHandlerCallback(int signum) |
| { |
| SDL_Event event; |
| SDLTest_Log("Ctrl+C received"); |
| event.type = SDL_QUIT; |
| SDL_PushEvent(&event); |
| } |
| #endif |
| |
| static Uint32 |
| ActionTimerCallback(Uint32 interval, void* param) |
| { |
| SDL_Event event; |
| SDL_UserEvent userevent; |
| Uint32 next_action_time; |
| |
| /* push an event to handle the action */ |
| userevent.type = SDL_USEREVENT; |
| userevent.code = ACTION_TIMER_EVENT; |
| userevent.data1 = ¤t->action; |
| userevent.data2 = NULL; |
| |
| event.type = SDL_USEREVENT; |
| event.user = userevent; |
| SDL_PushEvent(&event); |
| |
| /* calculate the new interval and return it */ |
| if(current->next) |
| next_action_time = current->next->action.time - current->action.time; |
| else |
| { |
| next_action_time = 0; |
| action_timer = 0; |
| } |
| |
| current = current->next; |
| return next_action_time; |
| } |
| |
| static Uint32 |
| KillTimerCallback(Uint32 interval, void* param) |
| { |
| SDL_Event event; |
| SDL_UserEvent userevent; |
| |
| userevent.type = SDL_USEREVENT; |
| userevent.code = KILL_TIMER_EVENT; |
| userevent.data1 = NULL; |
| userevent.data2 = NULL; |
| |
| event.type = SDL_USEREVENT; |
| event.user = userevent; |
| SDL_PushEvent(&event); |
| |
| kill_timer = 0; |
| return 0; |
| } |
| |
| static int |
| ProcessAction(SDLVisualTest_Action* action, int* sut_running, char* args) |
| { |
| if(!action || !sut_running) |
| return TEST_ERROR; |
| |
| switch(action->type) |
| { |
| case SDL_ACTION_KILL: |
| SDLTest_Log("Action: Kill SUT"); |
| if(SDL_IsProcessRunning(&pinfo) == 1 && |
| !SDL_KillProcess(&pinfo, &sut_exitstatus)) |
| { |
| SDLTest_LogError("SDL_KillProcess() failed"); |
| return TEST_ERROR; |
| } |
| *sut_running = 0; |
| break; |
| |
| case SDL_ACTION_QUIT: |
| SDLTest_Log("Action: Quit SUT"); |
| if(SDL_IsProcessRunning(&pinfo) == 1 && |
| !SDL_QuitProcess(&pinfo, &sut_exitstatus)) |
| { |
| SDLTest_LogError("SDL_QuitProcess() failed"); |
| return TEST_FAILED; |
| } |
| *sut_running = 0; |
| break; |
| |
| case SDL_ACTION_LAUNCH: |
| { |
| char* path; |
| char* args; |
| SDL_ProcessInfo action_process; |
| SDL_ProcessExitStatus ps; |
| |
| path = action->extra.process.path; |
| args = action->extra.process.args; |
| if(args) |
| { |
| SDLTest_Log("Action: Launch process: %s with arguments: %s", |
| path, args); |
| } |
| else |
| SDLTest_Log("Action: Launch process: %s", path); |
| if(!SDL_LaunchProcess(path, args, &action_process)) |
| { |
| SDLTest_LogError("SDL_LaunchProcess() failed"); |
| return TEST_ERROR; |
| } |
| |
| /* small delay so that the process can do its job */ |
| SDL_Delay(1000); |
| |
| if(SDL_IsProcessRunning(&action_process) > 0) |
| { |
| SDLTest_LogError("Process %s took too long too complete." |
| " Force killing...", action->extra); |
| if(!SDL_KillProcess(&action_process, &ps)) |
| { |
| SDLTest_LogError("SDL_KillProcess() failed"); |
| return TEST_ERROR; |
| } |
| } |
| } |
| break; |
| |
| case SDL_ACTION_SCREENSHOT: |
| { |
| char path[MAX_PATH_LEN], hash[33]; |
| |
| SDLTest_Log("Action: Take screenshot"); |
| /* can't take a screenshot if the SUT isn't running */ |
| if(SDL_IsProcessRunning(&pinfo) != 1) |
| { |
| SDLTest_LogError("SUT has quit."); |
| *sut_running = 0; |
| return TEST_FAILED; |
| } |
| |
| /* file name for the screenshot image */ |
| SDLVisualTest_HashString(args, hash); |
| SDL_snprintf(path, MAX_PATH_LEN, "%s/%s", state.output_dir, hash); |
| if(!SDLVisualTest_ScreenshotProcess(&pinfo, path)) |
| { |
| SDLTest_LogError("SDLVisualTest_ScreenshotProcess() failed"); |
| return TEST_ERROR; |
| } |
| } |
| break; |
| |
| case SDL_ACTION_VERIFY: |
| { |
| int ret; |
| |
| SDLTest_Log("Action: Verify screenshot"); |
| ret = SDLVisualTest_VerifyScreenshots(args, state.output_dir, |
| state.verify_dir); |
| |
| if(ret == -1) |
| { |
| SDLTest_LogError("SDLVisualTest_VerifyScreenshots() failed"); |
| return TEST_ERROR; |
| } |
| else if(ret == 0) |
| { |
| SDLTest_Log("Verification failed: Images were not equal."); |
| return TEST_FAILED; |
| } |
| else if(ret == 1) |
| SDLTest_Log("Verification successful."); |
| else |
| { |
| SDLTest_Log("Verfication skipped."); |
| return TEST_FAILED; |
| } |
| } |
| break; |
| |
| default: |
| SDLTest_LogError("Invalid action type"); |
| return TEST_ERROR; |
| break; |
| } |
| |
| return TEST_PASSED; |
| } |
| |
| static int |
| RunSUTAndTest(char* sutargs, int variation_num) |
| { |
| int success, sut_running, return_code; |
| char hash[33]; |
| SDL_Event event; |
| |
| return_code = TEST_PASSED; |
| |
| if(!sutargs) |
| { |
| SDLTest_LogError("sutargs argument cannot be NULL"); |
| return_code = TEST_ERROR; |
| goto runsutandtest_cleanup_generic; |
| } |
| |
| SDLVisualTest_HashString(sutargs, hash); |
| SDLTest_Log("Hash: %s", hash); |
| |
| success = SDL_LaunchProcess(state.sutapp, sutargs, &pinfo); |
| if(!success) |
| { |
| SDLTest_Log("Could not launch SUT."); |
| return_code = TEST_ERROR; |
| goto runsutandtest_cleanup_generic; |
| } |
| SDLTest_Log("SUT launch successful."); |
| SDLTest_Log("Process will be killed in %d milliseconds", state.timeout); |
| sut_running = 1; |
| |
| /* launch the timers */ |
| SDLTest_Log("Performing actions.."); |
| current = state.action_queue.front; |
| action_timer = 0; |
| kill_timer = 0; |
| if(current) |
| { |
| action_timer = SDL_AddTimer(current->action.time, ActionTimerCallback, NULL); |
| if(!action_timer) |
| { |
| SDLTest_LogError("SDL_AddTimer() failed"); |
| return_code = TEST_ERROR; |
| goto runsutandtest_cleanup_timer; |
| } |
| } |
| kill_timer = SDL_AddTimer(state.timeout, KillTimerCallback, NULL); |
| if(!kill_timer) |
| { |
| SDLTest_LogError("SDL_AddTimer() failed"); |
| return_code = TEST_ERROR; |
| goto runsutandtest_cleanup_timer; |
| } |
| |
| /* the timer stops running if the actions queue is empty, and the |
| SUT stops running if it crashes or if we encounter a KILL/QUIT action */ |
| while(sut_running) |
| { |
| /* process the actions by using an event queue */ |
| while(SDL_PollEvent(&event)) |
| { |
| if(event.type == SDL_USEREVENT) |
| { |
| if(event.user.code == ACTION_TIMER_EVENT) |
| { |
| SDLVisualTest_Action* action; |
| |
| action = (SDLVisualTest_Action*)event.user.data1; |
| |
| switch(ProcessAction(action, &sut_running, sutargs)) |
| { |
| case TEST_PASSED: |
| break; |
| |
| case TEST_FAILED: |
| return_code = TEST_FAILED; |
| goto runsutandtest_cleanup_timer; |
| break; |
| |
| default: |
| SDLTest_LogError("ProcessAction() failed"); |
| return_code = TEST_ERROR; |
| goto runsutandtest_cleanup_timer; |
| } |
| } |
| else if(event.user.code == KILL_TIMER_EVENT) |
| { |
| SDLTest_LogError("Maximum timeout reached. Force killing.."); |
| return_code = TEST_FAILED; |
| goto runsutandtest_cleanup_timer; |
| } |
| } |
| else if(event.type == SDL_QUIT) |
| { |
| SDLTest_LogError("Received QUIT event. Testharness is quitting.."); |
| return_code = TEST_ERROR; |
| goto runsutandtest_cleanup_timer; |
| } |
| } |
| SDL_Delay(1000/ACTION_LOOP_FPS); |
| } |
| |
| SDLTest_Log("SUT exit code was: %d", sut_exitstatus.exit_status); |
| if(sut_exitstatus.exit_status == 0) |
| { |
| return_code = TEST_PASSED; |
| goto runsutandtest_cleanup_timer; |
| } |
| else |
| { |
| return_code = TEST_FAILED; |
| goto runsutandtest_cleanup_timer; |
| } |
| |
| return_code = TEST_ERROR; |
| goto runsutandtest_cleanup_generic; |
| |
| runsutandtest_cleanup_timer: |
| if(action_timer && !SDL_RemoveTimer(action_timer)) |
| { |
| SDLTest_Log("SDL_RemoveTimer() failed"); |
| return_code = TEST_ERROR; |
| } |
| |
| if(kill_timer && !SDL_RemoveTimer(kill_timer)) |
| { |
| SDLTest_Log("SDL_RemoveTimer() failed"); |
| return_code = TEST_ERROR; |
| } |
| /* runsutandtest_cleanup_process: */ |
| if(SDL_IsProcessRunning(&pinfo) && !SDL_KillProcess(&pinfo, &sut_exitstatus)) |
| { |
| SDLTest_Log("SDL_KillProcess() failed"); |
| return_code = TEST_ERROR; |
| } |
| runsutandtest_cleanup_generic: |
| return return_code; |
| } |
| |
| /** Entry point for testharness */ |
| int |
| main(int argc, char* argv[]) |
| { |
| int i, passed, return_code, failed; |
| |
| /* freeing resources, linux style! */ |
| return_code = 0; |
| |
| if(argc < 2) |
| { |
| SDLTest_Log(usage(), argv[0]); |
| goto cleanup_generic; |
| } |
| |
| #if defined(__LINUX__) || defined(__CYGWIN__) |
| signal(SIGINT, CtrlCHandlerCallback); |
| #endif |
| |
| /* parse arguments */ |
| if(!SDLVisualTest_ParseHarnessArgs(argv + 1, &state)) |
| { |
| SDLTest_Log(usage(), argv[0]); |
| return_code = 1; |
| goto cleanup_generic; |
| } |
| SDLTest_Log("Parsed harness arguments successfully."); |
| |
| /* initialize SDL */ |
| if(SDL_Init(SDL_INIT_TIMER) == -1) |
| { |
| SDLTest_LogError("SDL_Init() failed."); |
| SDLVisualTest_FreeHarnessState(&state); |
| return_code = 1; |
| goto cleanup_harness_state; |
| } |
| |
| /* create an output directory if none exists */ |
| #if defined(__LINUX__) || defined(__CYGWIN__) |
| mkdir(state.output_dir, 0777); |
| #elif defined(__WIN32__) |
| _mkdir(state.output_dir); |
| #else |
| #error "Unsupported platform" |
| #endif |
| |
| /* test with sutargs */ |
| if(SDL_strlen(state.sutargs)) |
| { |
| SDLTest_Log("Running: %s %s", state.sutapp, state.sutargs); |
| if(!state.no_launch) |
| { |
| switch(RunSUTAndTest(state.sutargs, 0)) |
| { |
| case TEST_PASSED: |
| SDLTest_Log("Status: PASSED"); |
| break; |
| |
| case TEST_FAILED: |
| SDLTest_Log("Status: FAILED"); |
| break; |
| |
| case TEST_ERROR: |
| SDLTest_LogError("Some error occurred while testing."); |
| return_code = 1; |
| goto cleanup_sdl; |
| break; |
| } |
| } |
| } |
| |
| if(state.sut_config.num_options > 0) |
| { |
| char* variator_name = state.variator_type == SDL_VARIATOR_RANDOM ? |
| "RANDOM" : "EXHAUSTIVE"; |
| if(state.num_variations > 0) |
| SDLTest_Log("Testing SUT with variator: %s for %d variations", |
| variator_name, state.num_variations); |
| else |
| SDLTest_Log("Testing SUT with variator: %s and ALL variations", |
| variator_name); |
| /* initialize the variator */ |
| if(!SDLVisualTest_InitVariator(&variator, &state.sut_config, |
| state.variator_type, 0)) |
| { |
| SDLTest_LogError("Could not initialize variator"); |
| return_code = 1; |
| goto cleanup_sdl; |
| } |
| |
| /* iterate through all the variations */ |
| passed = 0; |
| failed = 0; |
| for(i = 0; state.num_variations > 0 ? (i < state.num_variations) : 1; i++) |
| { |
| char* args = SDLVisualTest_GetNextVariation(&variator); |
| if(!args) |
| break; |
| SDLTest_Log("\nVariation number: %d\nArguments: %s", i + 1, args); |
| |
| if(!state.no_launch) |
| { |
| switch(RunSUTAndTest(args, i + 1)) |
| { |
| case TEST_PASSED: |
| SDLTest_Log("Status: PASSED"); |
| passed++; |
| break; |
| |
| case TEST_FAILED: |
| SDLTest_Log("Status: FAILED"); |
| failed++; |
| break; |
| |
| case TEST_ERROR: |
| SDLTest_LogError("Some error occurred while testing."); |
| goto cleanup_variator; |
| break; |
| } |
| } |
| } |
| if(!state.no_launch) |
| { |
| /* report stats */ |
| SDLTest_Log("Testing complete."); |
| SDLTest_Log("%d/%d tests passed.", passed, passed + failed); |
| } |
| goto cleanup_variator; |
| } |
| |
| goto cleanup_sdl; |
| |
| cleanup_variator: |
| SDLVisualTest_FreeVariator(&variator); |
| cleanup_sdl: |
| SDL_Quit(); |
| cleanup_harness_state: |
| SDLVisualTest_FreeHarnessState(&state); |
| cleanup_generic: |
| return return_code; |
| } |