Fix bug/add test coverage for SDLTest_GenerateRunSeed helper; improve test harness adding output of repro steps for failures; improve negative test for SDL_GetError/SDL_SetError

src/test/SDL_test_harness.c

@@ -67,12 +67,13 @@ SDLTest_GenerateRunSeed(const int length)
     seed = (char *)SDL_malloc((length + 1) * sizeof(char));
     if (seed == NULL) {
         SDLTest_LogError("SDL_malloc for run seed output buffer failed.");
+        SDL_Error(SDL_ENOMEM);
         return NULL;
     }
 
     /* Generate a random string of alphanumeric characters */
     SDLTest_RandomInitTime(&randomContext);
-    for (counter = 0; counter < length - 1; ++counter) {
+    for (counter = 0; counter < length; counter++) {
         unsigned int number = SDLTest_Random(&randomContext);
         char ch = (char) (number % (91 - 48)) + 48;
         if (ch >= 58 && ch <= 64) {
@@ -80,7 +81,7 @@ SDLTest_GenerateRunSeed(const int length)
         }
         seed[counter] = ch;
     }
-    seed[counter] = '\0';
+    seed[length] = '\0';
 
     return seed;
 }
@@ -141,7 +142,8 @@ SDLTest_GenerateExecKey(char *runSeed, char *suiteName, char *testName, int iter
     entireStringLength  = runSeedLength + suiteNameLength + testNameLength + iterationStringLength + 1;
     buffer = (char *)SDL_malloc(entireStringLength);
     if (buffer == NULL) {
-        SDLTest_LogError("SDL_malloc failed to allocate buffer for execKey generation.");
+        SDLTest_LogError("Failed to allocate buffer for execKey generation.");
+        SDL_Error(SDL_ENOMEM);
         return 0;
     }
     SDL_snprintf(buffer, entireStringLength, "%s%s%s%d", runSeed, suiteName, testName, iteration);
@@ -347,7 +349,7 @@ float GetClock()
 }
 
 /**
-* \brief Execute a test suite using the given run seend and execution key.
+* \brief Execute a test suite using the given run seed and execution key.
 *
 * The filter string is matched to the suite name (full comparison) to select a single suite,
 * or if no suite matches, it is matched to the test names (full comparison) to select a single test.
@@ -362,6 +364,8 @@ float GetClock()
 */
 int SDLTest_RunSuites(SDLTest_TestSuiteReference *testSuites[], const char *userRunSeed, Uint64 userExecKey, const char *filter, int testIterations)
 {
+    int totalNumberOfTests = 0;
+    int failedNumberOfTests = 0;
     int suiteCounter;
     int testCounter;
     int iterationCounter;
@@ -392,6 +396,7 @@ int SDLTest_RunSuites(SDLTest_TestSuiteReference *testSuites[], const char *user
     Uint32 testSkippedCount = 0;
     Uint32 countSum = 0;
     char *logFormat = (char *)SDLTest_LogSummaryFormat;
+    SDLTest_TestCaseReference **failedTests;
 
     /* Sanitize test iterations */
     if (testIterations < 1) {
@@ -421,6 +426,27 @@ int SDLTest_RunSuites(SDLTest_TestSuiteReference *testSuites[], const char *user
     /* Log run with fuzzer parameters */
     SDLTest_Log("::::: Test Run /w seed '%s' started\n", runSeed);
 
+	/* Count the total number of tests */
+    suiteCounter = 0;
+    while (testSuites[suiteCounter]) {
+        testSuite=(SDLTest_TestSuiteReference *)testSuites[suiteCounter];
+        suiteCounter++;
+        testCounter = 0;
+        while (testSuite->testCases[testCounter])
+        {
+            testCounter++;
+			totalNumberOfTests++;
+		}
+	}
+
+	/* Pre-allocate an array for tracking failed tests (potentially all test cases) */
+	failedTests = (SDLTest_TestCaseReference **)SDL_malloc(totalNumberOfTests * sizeof(SDLTest_TestCaseReference *));
+	if (failedTests == NULL) {	
+	   SDLTest_LogError("Unable to allocate cache for failed tests");
+           SDL_Error(SDL_ENOMEM);	   
+           return -1;
+	}
+
     /* Initialize filtering */
     if (filter != NULL && filter[0] != '\0') {
         /* Loop over all suites to check if we have a filter match */
@@ -580,6 +606,11 @@ int SDLTest_RunSuites(SDLTest_TestSuiteReference *testSuites[], const char *user
                         break;
                     }
 
+                    /* Collect failed test case references for repro-step display */
+                    if (testResult == TEST_RESULT_FAILED) {
+                        failedTests[failedNumberOfTests] = testCase;
+                        failedNumberOfTests++;
+                    }
                 }
             }
 
@@ -630,6 +661,15 @@ int SDLTest_RunSuites(SDLTest_TestSuiteReference *testSuites[], const char *user
         SDLTest_LogError((char *)SDLTest_FinalResultFormat, "Run /w seed", runSeed, "Failed");
     }
 
+    /* Print repro steps for failed tests */
+    if (failedNumberOfTests > 0) {
+        SDLTest_Log("Harness input to repro failures:");
+        for (testCounter = 0; testCounter < failedNumberOfTests; testCounter++) {
+          SDLTest_Log(" --seed %s --filter %s", runSeed, failedTests[testCounter]->name);
+        }
+    }
+    SDL_free(failedTests);
+
     SDLTest_Log("Exit code: %d", runResult);
     return runResult;
 }

test/testautomation_platform.c

@@ -283,6 +283,7 @@ int platform_testDefaultInit(void *arg)
  */
 int platform_testGetSetClearError(void *arg)
 {
+   int result;
    const char *testError = "Testing";
    char *lastError;
    int len;
@@ -301,8 +302,9 @@ int platform_testGetSetClearError(void *arg)
              "SDL_GetError(): no message expected, len: %i", len);
    }
 
-   SDL_SetError("%s", testError);
+   result = SDL_SetError("%s", testError);
    SDLTest_AssertPass("SDL_SetError()");
+   SDLTest_AssertCheck(result == -1, "SDL_SetError: expected -1, got: %i", result);
    lastError = (char *)SDL_GetError();
    SDLTest_AssertCheck(lastError != NULL,
              "SDL_GetError() != NULL");
@@ -333,12 +335,14 @@ int platform_testGetSetClearError(void *arg)
  */
 int platform_testSetErrorEmptyInput(void *arg)
 {
+   int result;
    const char *testError = "";
    char *lastError;
    int len;
 
-   SDL_SetError("%s", testError);
+   result = SDL_SetError("%s", testError);
    SDLTest_AssertPass("SDL_SetError()");
+   SDLTest_AssertCheck(result == -1, "SDL_SetError: expected -1, got: %i", result);
    lastError = (char *)SDL_GetError();
    SDLTest_AssertCheck(lastError != NULL,
              "SDL_GetError() != NULL");
@@ -369,7 +373,8 @@ int platform_testSetErrorEmptyInput(void *arg)
  */
 int platform_testSetErrorInvalidInput(void *arg)
 {
-   const char *testError = NULL;
+   int result;
+   const char *invalidError = NULL;
    const char *probeError = "Testing";
    char *lastError;
    int len;
@@ -379,8 +384,9 @@ int platform_testSetErrorInvalidInput(void *arg)
    SDLTest_AssertPass("SDL_ClearError()");
 
    /* Check for no-op */
-   SDL_SetError(testError);
+   result = SDL_SetError(invalidError);
    SDLTest_AssertPass("SDL_SetError()");
+   SDLTest_AssertCheck(result == -1, "SDL_SetError: expected -1, got: %i", result);
    lastError = (char *)SDL_GetError();
    SDLTest_AssertCheck(lastError != NULL,
              "SDL_GetError() != NULL");
@@ -397,12 +403,38 @@ int platform_testSetErrorInvalidInput(void *arg)
    }
 
    /* Set */
-   SDL_SetError(probeError);
+   result = SDL_SetError(probeError);
    SDLTest_AssertPass("SDL_SetError()");
+   SDLTest_AssertCheck(result == -1, "SDL_SetError: expected -1, got: %i", result);
 
    /* Check for no-op */
-   SDL_SetError(testError);
+   result = SDL_SetError(invalidError);
    SDLTest_AssertPass("SDL_SetError()");
+   SDLTest_AssertCheck(result == -1, "SDL_SetError: expected -1, got: %i", result);
+   lastError = (char *)SDL_GetError();
+   SDLTest_AssertCheck(lastError != NULL,
+             "SDL_GetError() != NULL");
+   if (lastError != NULL)
+   {
+     len = SDL_strlen(lastError);
+     SDLTest_AssertCheck(len == SDL_strlen(probeError),
+             "SDL_GetError(): expected message len %i, was len: %i",
+             SDL_strlen(probeError),
+             len);
+     SDLTest_AssertCheck(SDL_strcmp(lastError, probeError) == 0,
+             "SDL_GetError(): expected message '%s', was message: '%s'",
+             probeError,
+             lastError);
+   }
+
+   /* Reset */
+   SDL_ClearError();
+   SDLTest_AssertPass("SDL_ClearError()");
+
+   /* Set and check */
+   result = SDL_SetError(probeError);
+   SDLTest_AssertPass("SDL_SetError()");
+   SDLTest_AssertCheck(result == -1, "SDL_SetError: expected -1, got: %i", result);
    lastError = (char *)SDL_GetError();
    SDLTest_AssertCheck(lastError != NULL,
              "SDL_GetError() != NULL");

test/testautomation_sdltest.c

@@ -21,6 +21,39 @@
 
 /* Test case functions */
 
+/* Forward declarations for internal harness functions */
+extern char *SDLTest_GenerateRunSeed(const int length);
+
+/**
+ * @brief Calls to SDLTest_GenerateRunSeed()
+ */
+int
+sdltest_generateRunSeed(void *arg)
+{
+  char* result;
+  int i, l;
+  
+  for (i = 1; i <= 10; i += 3) {   
+     result = SDLTest_GenerateRunSeed((const int)i);
+     SDLTest_AssertPass("Call to SDLTest_GenerateRunSeed()");
+     SDLTest_AssertCheck(result != NULL, "Verify returned value is not NULL");
+     if (result != NULL) {
+       l = SDL_strlen(result);
+       SDLTest_AssertCheck(l == i, "Verify length of returned value is %d, got: %d", i, l);
+       SDL_free(result);
+     }
+  }
+
+  /* Negative cases */
+  for (i = -2; i <= 0; i++) {   
+     result = SDLTest_GenerateRunSeed((const int)i);
+     SDLTest_AssertPass("Call to SDLTest_GenerateRunSeed()");
+     SDLTest_AssertCheck(result == NULL, "Verify returned value is not NULL");
+  }
+  
+  return TEST_COMPLETED;
+}
+
 /**
  * @brief Calls to SDLTest_GetFuzzerInvocationCount()
  */
@@ -1263,11 +1296,14 @@ static const SDLTest_TestCaseReference sdltestTest13 =
 static const SDLTest_TestCaseReference sdltestTest14 =
         { (SDLTest_TestCaseFp)sdltest_randomAsciiStringOfSize, "sdltest_randomAsciiStringOfSize", "Calls to fixed size ASCII string generator", TEST_ENABLED };
 
+static const SDLTest_TestCaseReference sdltestTest15 =
+        { (SDLTest_TestCaseFp)sdltest_generateRunSeed, "sdltest_generateRunSeed", "Checks internal harness function SDLTest_GenerateRunSeed", TEST_ENABLED };
+
 /* Sequence of SDL_test test cases */
 static const SDLTest_TestCaseReference *sdltestTests[] =  {
     &sdltestTest1, &sdltestTest2, &sdltestTest3, &sdltestTest4, &sdltestTest5, &sdltestTest6,
     &sdltestTest7, &sdltestTest8, &sdltestTest9, &sdltestTest10, &sdltestTest11, &sdltestTest12,
-    &sdltestTest13, &sdltestTest14, NULL
+    &sdltestTest13, &sdltestTest14, &sdltestTest15, NULL
 };
 
 /* SDL_test test suite (global) */