From 7a36070a95010fff87d572e9d700cff87d74706a Mon Sep 17 00:00:00 2001 From: Andreas Schiffler Date: Tue, 11 Mar 2014 07:17:56 -0700 Subject: [PATCH] Fix bug/add test coverage for SDLTest_GenerateRunSeed helper; improve test harness adding output of repro steps for failures; improve negative test for SDL_GetError/SDL_SetError --- src/test/SDL_test_harness.c | 48 +++++++++++++++++++++++++++++++--- test/testautomation_platform.c | 44 ++++++++++++++++++++++++++----- test/testautomation_sdltest.c | 38 ++++++++++++++++++++++++++- 3 files changed, 119 insertions(+), 11 deletions(-) diff --git a/src/test/SDL_test_harness.c b/src/test/SDL_test_harness.c index 6c36f9d80..93f8dd703 100644 --- a/src/test/SDL_test_harness.c +++ b/src/test/SDL_test_harness.c @@ -67,12 +67,13 @@ SDLTest_GenerateRunSeed(const int length) seed = (char *)SDL_malloc((length + 1) * sizeof(char)); if (seed == NULL) { SDLTest_LogError("SDL_malloc for run seed output buffer failed."); + SDL_Error(SDL_ENOMEM); return NULL; } /* Generate a random string of alphanumeric characters */ SDLTest_RandomInitTime(&randomContext); - for (counter = 0; counter < length - 1; ++counter) { + for (counter = 0; counter < length; counter++) { unsigned int number = SDLTest_Random(&randomContext); char ch = (char) (number % (91 - 48)) + 48; if (ch >= 58 && ch <= 64) { @@ -80,7 +81,7 @@ SDLTest_GenerateRunSeed(const int length) } seed[counter] = ch; } - seed[counter] = '\0'; + seed[length] = '\0'; return seed; } @@ -141,7 +142,8 @@ SDLTest_GenerateExecKey(char *runSeed, char *suiteName, char *testName, int iter entireStringLength = runSeedLength + suiteNameLength + testNameLength + iterationStringLength + 1; buffer = (char *)SDL_malloc(entireStringLength); if (buffer == NULL) { - SDLTest_LogError("SDL_malloc failed to allocate buffer for execKey generation."); + SDLTest_LogError("Failed to allocate buffer for execKey generation."); + SDL_Error(SDL_ENOMEM); return 0; } SDL_snprintf(buffer, entireStringLength, "%s%s%s%d", runSeed, suiteName, testName, iteration); @@ -347,7 +349,7 @@ float GetClock() } /** -* \brief Execute a test suite using the given run seend and execution key. +* \brief Execute a test suite using the given run seed and execution key. * * The filter string is matched to the suite name (full comparison) to select a single suite, * or if no suite matches, it is matched to the test names (full comparison) to select a single test. @@ -362,6 +364,8 @@ float GetClock() */ int SDLTest_RunSuites(SDLTest_TestSuiteReference *testSuites[], const char *userRunSeed, Uint64 userExecKey, const char *filter, int testIterations) { + int totalNumberOfTests = 0; + int failedNumberOfTests = 0; int suiteCounter; int testCounter; int iterationCounter; @@ -392,6 +396,7 @@ int SDLTest_RunSuites(SDLTest_TestSuiteReference *testSuites[], const char *user Uint32 testSkippedCount = 0; Uint32 countSum = 0; char *logFormat = (char *)SDLTest_LogSummaryFormat; + SDLTest_TestCaseReference **failedTests; /* Sanitize test iterations */ if (testIterations < 1) { @@ -421,6 +426,27 @@ int SDLTest_RunSuites(SDLTest_TestSuiteReference *testSuites[], const char *user /* Log run with fuzzer parameters */ SDLTest_Log("::::: Test Run /w seed '%s' started\n", runSeed); + /* Count the total number of tests */ + suiteCounter = 0; + while (testSuites[suiteCounter]) { + testSuite=(SDLTest_TestSuiteReference *)testSuites[suiteCounter]; + suiteCounter++; + testCounter = 0; + while (testSuite->testCases[testCounter]) + { + testCounter++; + totalNumberOfTests++; + } + } + + /* Pre-allocate an array for tracking failed tests (potentially all test cases) */ + failedTests = (SDLTest_TestCaseReference **)SDL_malloc(totalNumberOfTests * sizeof(SDLTest_TestCaseReference *)); + if (failedTests == NULL) { + SDLTest_LogError("Unable to allocate cache for failed tests"); + SDL_Error(SDL_ENOMEM); + return -1; + } + /* Initialize filtering */ if (filter != NULL && filter[0] != '\0') { /* Loop over all suites to check if we have a filter match */ @@ -580,6 +606,11 @@ int SDLTest_RunSuites(SDLTest_TestSuiteReference *testSuites[], const char *user break; } + /* Collect failed test case references for repro-step display */ + if (testResult == TEST_RESULT_FAILED) { + failedTests[failedNumberOfTests] = testCase; + failedNumberOfTests++; + } } } @@ -630,6 +661,15 @@ int SDLTest_RunSuites(SDLTest_TestSuiteReference *testSuites[], const char *user SDLTest_LogError((char *)SDLTest_FinalResultFormat, "Run /w seed", runSeed, "Failed"); } + /* Print repro steps for failed tests */ + if (failedNumberOfTests > 0) { + SDLTest_Log("Harness input to repro failures:"); + for (testCounter = 0; testCounter < failedNumberOfTests; testCounter++) { + SDLTest_Log(" --seed %s --filter %s", runSeed, failedTests[testCounter]->name); + } + } + SDL_free(failedTests); + SDLTest_Log("Exit code: %d", runResult); return runResult; } diff --git a/test/testautomation_platform.c b/test/testautomation_platform.c index ba7d30369..849d4e387 100644 --- a/test/testautomation_platform.c +++ b/test/testautomation_platform.c @@ -283,6 +283,7 @@ int platform_testDefaultInit(void *arg) */ int platform_testGetSetClearError(void *arg) { + int result; const char *testError = "Testing"; char *lastError; int len; @@ -301,8 +302,9 @@ int platform_testGetSetClearError(void *arg) "SDL_GetError(): no message expected, len: %i", len); } - SDL_SetError("%s", testError); + result = SDL_SetError("%s", testError); SDLTest_AssertPass("SDL_SetError()"); + SDLTest_AssertCheck(result == -1, "SDL_SetError: expected -1, got: %i", result); lastError = (char *)SDL_GetError(); SDLTest_AssertCheck(lastError != NULL, "SDL_GetError() != NULL"); @@ -333,12 +335,14 @@ int platform_testGetSetClearError(void *arg) */ int platform_testSetErrorEmptyInput(void *arg) { + int result; const char *testError = ""; char *lastError; int len; - SDL_SetError("%s", testError); + result = SDL_SetError("%s", testError); SDLTest_AssertPass("SDL_SetError()"); + SDLTest_AssertCheck(result == -1, "SDL_SetError: expected -1, got: %i", result); lastError = (char *)SDL_GetError(); SDLTest_AssertCheck(lastError != NULL, "SDL_GetError() != NULL"); @@ -369,7 +373,8 @@ int platform_testSetErrorEmptyInput(void *arg) */ int platform_testSetErrorInvalidInput(void *arg) { - const char *testError = NULL; + int result; + const char *invalidError = NULL; const char *probeError = "Testing"; char *lastError; int len; @@ -379,8 +384,9 @@ int platform_testSetErrorInvalidInput(void *arg) SDLTest_AssertPass("SDL_ClearError()"); /* Check for no-op */ - SDL_SetError(testError); + result = SDL_SetError(invalidError); SDLTest_AssertPass("SDL_SetError()"); + SDLTest_AssertCheck(result == -1, "SDL_SetError: expected -1, got: %i", result); lastError = (char *)SDL_GetError(); SDLTest_AssertCheck(lastError != NULL, "SDL_GetError() != NULL"); @@ -397,12 +403,14 @@ int platform_testSetErrorInvalidInput(void *arg) } /* Set */ - SDL_SetError(probeError); + result = SDL_SetError(probeError); SDLTest_AssertPass("SDL_SetError()"); + SDLTest_AssertCheck(result == -1, "SDL_SetError: expected -1, got: %i", result); /* Check for no-op */ - SDL_SetError(testError); + result = SDL_SetError(invalidError); SDLTest_AssertPass("SDL_SetError()"); + SDLTest_AssertCheck(result == -1, "SDL_SetError: expected -1, got: %i", result); lastError = (char *)SDL_GetError(); SDLTest_AssertCheck(lastError != NULL, "SDL_GetError() != NULL"); @@ -419,6 +427,30 @@ int platform_testSetErrorInvalidInput(void *arg) lastError); } + /* Reset */ + SDL_ClearError(); + SDLTest_AssertPass("SDL_ClearError()"); + + /* Set and check */ + result = SDL_SetError(probeError); + SDLTest_AssertPass("SDL_SetError()"); + SDLTest_AssertCheck(result == -1, "SDL_SetError: expected -1, got: %i", result); + lastError = (char *)SDL_GetError(); + SDLTest_AssertCheck(lastError != NULL, + "SDL_GetError() != NULL"); + if (lastError != NULL) + { + len = SDL_strlen(lastError); + SDLTest_AssertCheck(len == SDL_strlen(probeError), + "SDL_GetError(): expected message len %i, was len: %i", + SDL_strlen(probeError), + len); + SDLTest_AssertCheck(SDL_strcmp(lastError, probeError) == 0, + "SDL_GetError(): expected message '%s', was message: '%s'", + probeError, + lastError); + } + /* Clean up */ SDL_ClearError(); SDLTest_AssertPass("SDL_ClearError()"); diff --git a/test/testautomation_sdltest.c b/test/testautomation_sdltest.c index d9ce7d281..646027a6c 100644 --- a/test/testautomation_sdltest.c +++ b/test/testautomation_sdltest.c @@ -21,6 +21,39 @@ /* Test case functions */ +/* Forward declarations for internal harness functions */ +extern char *SDLTest_GenerateRunSeed(const int length); + +/** + * @brief Calls to SDLTest_GenerateRunSeed() + */ +int +sdltest_generateRunSeed(void *arg) +{ + char* result; + int i, l; + + for (i = 1; i <= 10; i += 3) { + result = SDLTest_GenerateRunSeed((const int)i); + SDLTest_AssertPass("Call to SDLTest_GenerateRunSeed()"); + SDLTest_AssertCheck(result != NULL, "Verify returned value is not NULL"); + if (result != NULL) { + l = SDL_strlen(result); + SDLTest_AssertCheck(l == i, "Verify length of returned value is %d, got: %d", i, l); + SDL_free(result); + } + } + + /* Negative cases */ + for (i = -2; i <= 0; i++) { + result = SDLTest_GenerateRunSeed((const int)i); + SDLTest_AssertPass("Call to SDLTest_GenerateRunSeed()"); + SDLTest_AssertCheck(result == NULL, "Verify returned value is not NULL"); + } + + return TEST_COMPLETED; +} + /** * @brief Calls to SDLTest_GetFuzzerInvocationCount() */ @@ -1263,11 +1296,14 @@ static const SDLTest_TestCaseReference sdltestTest13 = static const SDLTest_TestCaseReference sdltestTest14 = { (SDLTest_TestCaseFp)sdltest_randomAsciiStringOfSize, "sdltest_randomAsciiStringOfSize", "Calls to fixed size ASCII string generator", TEST_ENABLED }; +static const SDLTest_TestCaseReference sdltestTest15 = + { (SDLTest_TestCaseFp)sdltest_generateRunSeed, "sdltest_generateRunSeed", "Checks internal harness function SDLTest_GenerateRunSeed", TEST_ENABLED }; + /* Sequence of SDL_test test cases */ static const SDLTest_TestCaseReference *sdltestTests[] = { &sdltestTest1, &sdltestTest2, &sdltestTest3, &sdltestTest4, &sdltestTest5, &sdltestTest6, &sdltestTest7, &sdltestTest8, &sdltestTest9, &sdltestTest10, &sdltestTest11, &sdltestTest12, - &sdltestTest13, &sdltestTest14, NULL + &sdltestTest13, &sdltestTest14, &sdltestTest15, NULL }; /* SDL_test test suite (global) */