SDL  2.0
SDL_test_harness.c
Go to the documentation of this file.
1 /*
2  Simple DirectMedia Layer
3  Copyright (C) 1997-2016 Sam Lantinga <slouken@libsdl.org>
4 
5  This software is provided 'as-is', without any express or implied
6  warranty. In no event will the authors be held liable for any damages
7  arising from the use of this software.
8 
9  Permission is granted to anyone to use this software for any purpose,
10  including commercial applications, and to alter it and redistribute it
11  freely, subject to the following restrictions:
12 
13  1. The origin of this software must not be misrepresented; you must not
14  claim that you wrote the original software. If you use this software
15  in a product, an acknowledgment in the product documentation would be
16  appreciated but is not required.
17  2. Altered source versions must be plainly marked as such, and must not be
18  misrepresented as being the original software.
19  3. This notice may not be removed or altered from any source distribution.
20 */
21 
22 #include "SDL_config.h"
23 
24 #include "SDL_test.h"
25 
26 #include <stdio.h>
27 #include <stdlib.h>
28 #include <string.h>
29 #include <time.h>
30 
31 /* Invalid test name/description message format */
32 #define SDLTEST_INVALID_NAME_FORMAT "(Invalid)"
33 
34 /* Log summary message format */
35 #define SDLTEST_LOG_SUMMARY_FORMAT "%s Summary: Total=%d Passed=%d Failed=%d Skipped=%d"
36 
37 /* Final result message format */
38 #define SDLTEST_FINAL_RESULT_FORMAT ">>> %s '%s': %s\n"
39 
40 /* ! \brief Timeout for single test case execution */
42 
43 /**
44 * Generates a random run seed string for the harness. The generated seed
45 * will contain alphanumeric characters (0-9A-Z).
46 *
47 * Note: The returned string needs to be deallocated by the caller.
48 *
49 * \param length The length of the seed string to generate
50 *
51 * \returns The generated seed string
52 */
53 char *
55 {
56  char *seed = NULL;
57  SDLTest_RandomContext randomContext;
58  int counter;
59 
60  /* Sanity check input */
61  if (length <= 0) {
62  SDLTest_LogError("The length of the harness seed must be >0.");
63  return NULL;
64  }
65 
66  /* Allocate output buffer */
67  seed = (char *)SDL_malloc((length + 1) * sizeof(char));
68  if (seed == NULL) {
69  SDLTest_LogError("SDL_malloc for run seed output buffer failed.");
71  return NULL;
72  }
73 
74  /* Generate a random string of alphanumeric characters */
75  SDLTest_RandomInitTime(&randomContext);
76  for (counter = 0; counter < length; counter++) {
77  unsigned int number = SDLTest_Random(&randomContext);
78  char ch = (char) (number % (91 - 48)) + 48;
79  if (ch >= 58 && ch <= 64) {
80  ch = 65;
81  }
82  seed[counter] = ch;
83  }
84  seed[length] = '\0';
85 
86  return seed;
87 }
88 
89 /**
90 * Generates an execution key for the fuzzer.
91 *
92 * \param runSeed The run seed to use
93 * \param suiteName The name of the test suite
94 * \param testName The name of the test
95 * \param iteration The iteration count
96 *
97 * \returns The generated execution key to initialize the fuzzer with.
98 *
99 */
100 Uint64
101 SDLTest_GenerateExecKey(char *runSeed, char *suiteName, char *testName, int iteration)
102 {
103  SDLTest_Md5Context md5Context;
104  Uint64 *keys;
105  char iterationString[16];
106  Uint32 runSeedLength;
107  Uint32 suiteNameLength;
108  Uint32 testNameLength;
109  Uint32 iterationStringLength;
110  Uint32 entireStringLength;
111  char *buffer;
112 
113  if (runSeed == NULL || runSeed[0] == '\0') {
114  SDLTest_LogError("Invalid runSeed string.");
115  return -1;
116  }
117 
118  if (suiteName == NULL || suiteName[0] == '\0') {
119  SDLTest_LogError("Invalid suiteName string.");
120  return -1;
121  }
122 
123  if (testName == NULL || testName[0] == '\0') {
124  SDLTest_LogError("Invalid testName string.");
125  return -1;
126  }
127 
128  if (iteration <= 0) {
129  SDLTest_LogError("Invalid iteration count.");
130  return -1;
131  }
132 
133  /* Convert iteration number into a string */
134  SDL_memset(iterationString, 0, sizeof(iterationString));
135  SDL_snprintf(iterationString, sizeof(iterationString) - 1, "%d", iteration);
136 
137  /* Combine the parameters into single string */
138  runSeedLength = SDL_strlen(runSeed);
139  suiteNameLength = SDL_strlen(suiteName);
140  testNameLength = SDL_strlen(testName);
141  iterationStringLength = SDL_strlen(iterationString);
142  entireStringLength = runSeedLength + suiteNameLength + testNameLength + iterationStringLength + 1;
143  buffer = (char *)SDL_malloc(entireStringLength);
144  if (buffer == NULL) {
145  SDLTest_LogError("Failed to allocate buffer for execKey generation.");
147  return 0;
148  }
149  SDL_snprintf(buffer, entireStringLength, "%s%s%s%d", runSeed, suiteName, testName, iteration);
150 
151  /* Hash string and use half of the digest as 64bit exec key */
152  SDLTest_Md5Init(&md5Context);
153  SDLTest_Md5Update(&md5Context, (unsigned char *)buffer, entireStringLength);
154  SDLTest_Md5Final(&md5Context);
155  SDL_free(buffer);
156  keys = (Uint64 *)md5Context.digest;
157 
158  return keys[0];
159 }
160 
161 /**
162 * \brief Set timeout handler for test.
163 *
164 * Note: SDL_Init(SDL_INIT_TIMER) will be called if it wasn't done so before.
165 *
166 * \param timeout Timeout interval in seconds.
167 * \param callback Function that will be called after timeout has elapsed.
168 *
169 * \return Timer id or -1 on failure.
170 */
173 {
174  Uint32 timeoutInMilliseconds;
175  SDL_TimerID timerID;
176 
177  if (callback == NULL) {
178  SDLTest_LogError("Timeout callback can't be NULL");
179  return -1;
180  }
181 
182  if (timeout < 0) {
183  SDLTest_LogError("Timeout value must be bigger than zero.");
184  return -1;
185  }
186 
187  /* Init SDL timer if not initialized before */
188  if (SDL_WasInit(SDL_INIT_TIMER) == 0) {
190  SDLTest_LogError("Failed to init timer subsystem: %s", SDL_GetError());
191  return -1;
192  }
193  }
194 
195  /* Set timer */
196  timeoutInMilliseconds = timeout * 1000;
197  timerID = SDL_AddTimer(timeoutInMilliseconds, (SDL_TimerCallback)callback, 0x0);
198  if (timerID == 0) {
199  SDLTest_LogError("Creation of SDL timer failed: %s", SDL_GetError());
200  return -1;
201  }
202 
203  return timerID;
204 }
205 
206 /**
207 * \brief Timeout handler. Aborts test run and exits harness process.
208 */
209 void
211 {
212  SDLTest_LogError("TestCaseTimeout timer expired. Aborting test run.");
213  exit(TEST_ABORTED); /* bail out from the test */
214 }
215 
216 /**
217 * \brief Execute a test using the given execution key.
218 *
219 * \param testSuite Suite containing the test case.
220 * \param testCase Case to execute.
221 * \param execKey Execution key for the fuzzer.
222 * \param forceTestRun Force test to run even if test was disabled in suite.
223 *
224 * \returns Test case result.
225 */
226 int
228 {
229  SDL_TimerID timer = 0;
230  int testCaseResult = 0;
231  int testResult = 0;
232  int fuzzerCount;
233 
234  if (testSuite==NULL || testCase==NULL || testSuite->name==NULL || testCase->name==NULL)
235  {
236  SDLTest_LogError("Setup failure: testSuite or testCase references NULL");
238  }
239 
240  if (!testCase->enabled && forceTestRun == SDL_FALSE)
241  {
242  SDLTest_Log(SDLTEST_FINAL_RESULT_FORMAT, "Test", testCase->name, "Skipped (Disabled)");
243  return TEST_RESULT_SKIPPED;
244  }
245 
246  /* Initialize fuzzer */
247  SDLTest_FuzzerInit(execKey);
248 
249  /* Reset assert tracker */
251 
252  /* Set timeout timer */
254 
255  /* Maybe run suite initalizer function */
256  if (testSuite->testSetUp) {
257  testSuite->testSetUp(0x0);
259  SDLTest_LogError(SDLTEST_FINAL_RESULT_FORMAT, "Suite Setup", testSuite->name, "Failed");
261  }
262  }
263 
264  /* Run test case function */
265  testCaseResult = testCase->testCase(0x0);
266 
267  /* Convert test execution result into harness result */
268  if (testCaseResult == TEST_SKIPPED) {
269  /* Test was programatically skipped */
270  testResult = TEST_RESULT_SKIPPED;
271  } else if (testCaseResult == TEST_STARTED) {
272  /* Test did not return a TEST_COMPLETED value; assume it failed */
273  testResult = TEST_RESULT_FAILED;
274  } else if (testCaseResult == TEST_ABORTED) {
275  /* Test was aborted early; assume it failed */
276  testResult = TEST_RESULT_FAILED;
277  } else {
278  /* Perform failure analysis based on asserts */
279  testResult = SDLTest_AssertSummaryToTestResult();
280  }
281 
282  /* Maybe run suite cleanup function (ignore failed asserts) */
283  if (testSuite->testTearDown) {
284  testSuite->testTearDown(0x0);
285  }
286 
287  /* Cancel timeout timer */
288  if (timer) {
289  SDL_RemoveTimer(timer);
290  }
291 
292  /* Report on asserts and fuzzer usage */
293  fuzzerCount = SDLTest_GetFuzzerInvocationCount();
294  if (fuzzerCount > 0) {
295  SDLTest_Log("Fuzzer invocations: %d", fuzzerCount);
296  }
297 
298  /* Final log based on test execution result */
299  if (testCaseResult == TEST_SKIPPED) {
300  /* Test was programatically skipped */
301  SDLTest_Log(SDLTEST_FINAL_RESULT_FORMAT, "Test", testCase->name, "Skipped (Programmatically)");
302  } else if (testCaseResult == TEST_STARTED) {
303  /* Test did not return a TEST_COMPLETED value; assume it failed */
304  SDLTest_LogError(SDLTEST_FINAL_RESULT_FORMAT, "Test", testCase->name, "Failed (test started, but did not return TEST_COMPLETED)");
305  } else if (testCaseResult == TEST_ABORTED) {
306  /* Test was aborted early; assume it failed */
307  SDLTest_LogError(SDLTEST_FINAL_RESULT_FORMAT, "Test", testCase->name, "Failed (Aborted)");
308  } else {
310  }
311 
312  return testResult;
313 }
314 
315 /* Prints summary of all suites/tests contained in the given reference */
317 {
318  int suiteCounter;
319  int testCounter;
320  SDLTest_TestSuiteReference *testSuite;
321  SDLTest_TestCaseReference *testCase;
322 
323  /* Loop over all suites */
324  suiteCounter = 0;
325  while(&testSuites[suiteCounter]) {
326  testSuite=&testSuites[suiteCounter];
327  suiteCounter++;
328  SDLTest_Log("Test Suite %i - %s\n", suiteCounter,
329  (testSuite->name) ? testSuite->name : SDLTEST_INVALID_NAME_FORMAT);
330 
331  /* Loop over all test cases */
332  testCounter = 0;
333  while(testSuite->testCases[testCounter])
334  {
335  testCase=(SDLTest_TestCaseReference *)testSuite->testCases[testCounter];
336  testCounter++;
337  SDLTest_Log(" Test Case %i - %s: %s", testCounter,
338  (testCase->name) ? testCase->name : SDLTEST_INVALID_NAME_FORMAT,
339  (testCase->description) ? testCase->description : SDLTEST_INVALID_NAME_FORMAT);
340  }
341  }
342 }
343 
344 /* Gets a timer value in seconds */
345 float GetClock()
346 {
347  float currentClock = (float)clock();
348  return currentClock / (float)CLOCKS_PER_SEC;
349 }
350 
351 /**
352 * \brief Execute a test suite using the given run seed and execution key.
353 *
354 * The filter string is matched to the suite name (full comparison) to select a single suite,
355 * or if no suite matches, it is matched to the test names (full comparison) to select a single test.
356 *
357 * \param testSuites Suites containing the test case.
358 * \param userRunSeed Custom run seed provided by user, or NULL to autogenerate one.
359 * \param userExecKey Custom execution key provided by user, or 0 to autogenerate one.
360 * \param filter Filter specification. NULL disables. Case sensitive.
361 * \param testIterations Number of iterations to run each test case.
362 *
363 * \returns Test run result; 0 when all tests passed, 1 if any tests failed.
364 */
365 int SDLTest_RunSuites(SDLTest_TestSuiteReference *testSuites[], const char *userRunSeed, Uint64 userExecKey, const char *filter, int testIterations)
366 {
367  int totalNumberOfTests = 0;
368  int failedNumberOfTests = 0;
369  int suiteCounter;
370  int testCounter;
371  int iterationCounter;
372  SDLTest_TestSuiteReference *testSuite;
373  SDLTest_TestCaseReference *testCase;
374  const char *runSeed = NULL;
375  char *currentSuiteName;
376  char *currentTestName;
377  Uint64 execKey;
378  float runStartSeconds;
379  float suiteStartSeconds;
380  float testStartSeconds;
381  float runEndSeconds;
382  float suiteEndSeconds;
383  float testEndSeconds;
384  float runtime;
385  int suiteFilter = 0;
386  char *suiteFilterName = NULL;
387  int testFilter = 0;
388  char *testFilterName = NULL;
389  SDL_bool forceTestRun = SDL_FALSE;
390  int testResult = 0;
391  int runResult = 0;
392  Uint32 totalTestFailedCount = 0;
393  Uint32 totalTestPassedCount = 0;
394  Uint32 totalTestSkippedCount = 0;
395  Uint32 testFailedCount = 0;
396  Uint32 testPassedCount = 0;
397  Uint32 testSkippedCount = 0;
398  Uint32 countSum = 0;
399  SDLTest_TestCaseReference **failedTests;
400 
401  /* Sanitize test iterations */
402  if (testIterations < 1) {
403  testIterations = 1;
404  }
405 
406  /* Generate run see if we don't have one already */
407  if (userRunSeed == NULL || userRunSeed[0] == '\0') {
408  runSeed = SDLTest_GenerateRunSeed(16);
409  if (runSeed == NULL) {
410  SDLTest_LogError("Generating a random seed failed");
411  return 2;
412  }
413  } else {
414  runSeed = userRunSeed;
415  }
416 
417 
418  /* Reset per-run counters */
419  totalTestFailedCount = 0;
420  totalTestPassedCount = 0;
421  totalTestSkippedCount = 0;
422 
423  /* Take time - run start */
424  runStartSeconds = GetClock();
425 
426  /* Log run with fuzzer parameters */
427  SDLTest_Log("::::: Test Run /w seed '%s' started\n", runSeed);
428 
429  /* Count the total number of tests */
430  suiteCounter = 0;
431  while (testSuites[suiteCounter]) {
432  testSuite=(SDLTest_TestSuiteReference *)testSuites[suiteCounter];
433  suiteCounter++;
434  testCounter = 0;
435  while (testSuite->testCases[testCounter])
436  {
437  testCounter++;
438  totalNumberOfTests++;
439  }
440  }
441 
442  /* Pre-allocate an array for tracking failed tests (potentially all test cases) */
443  failedTests = (SDLTest_TestCaseReference **)SDL_malloc(totalNumberOfTests * sizeof(SDLTest_TestCaseReference *));
444  if (failedTests == NULL) {
445  SDLTest_LogError("Unable to allocate cache for failed tests");
447  return -1;
448  }
449 
450  /* Initialize filtering */
451  if (filter != NULL && filter[0] != '\0') {
452  /* Loop over all suites to check if we have a filter match */
453  suiteCounter = 0;
454  while (testSuites[suiteCounter] && suiteFilter == 0) {
455  testSuite=(SDLTest_TestSuiteReference *)testSuites[suiteCounter];
456  suiteCounter++;
457  if (testSuite->name != NULL && SDL_strcmp(filter, testSuite->name) == 0) {
458  /* Matched a suite name */
459  suiteFilter = 1;
460  suiteFilterName = testSuite->name;
461  SDLTest_Log("Filtering: running only suite '%s'", suiteFilterName);
462  break;
463  }
464 
465  /* Within each suite, loop over all test cases to check if we have a filter match */
466  testCounter = 0;
467  while (testSuite->testCases[testCounter] && testFilter == 0)
468  {
469  testCase=(SDLTest_TestCaseReference *)testSuite->testCases[testCounter];
470  testCounter++;
471  if (testCase->name != NULL && SDL_strcmp(filter, testCase->name) == 0) {
472  /* Matched a test name */
473  suiteFilter = 1;
474  suiteFilterName = testSuite->name;
475  testFilter = 1;
476  testFilterName = testCase->name;
477  SDLTest_Log("Filtering: running only test '%s' in suite '%s'", testFilterName, suiteFilterName);
478  break;
479  }
480  }
481  }
482 
483  if (suiteFilter == 0 && testFilter == 0) {
484  SDLTest_LogError("Filter '%s' did not match any test suite/case.", filter);
485  SDLTest_Log("Exit code: 2");
486  SDL_free(failedTests);
487  return 2;
488  }
489  }
490 
491  /* Loop over all suites */
492  suiteCounter = 0;
493  while(testSuites[suiteCounter]) {
494  testSuite=(SDLTest_TestSuiteReference *)testSuites[suiteCounter];
495  currentSuiteName = (char *)((testSuite->name) ? testSuite->name : SDLTEST_INVALID_NAME_FORMAT);
496  suiteCounter++;
497 
498  /* Filter suite if flag set and we have a name */
499  if (suiteFilter == 1 && suiteFilterName != NULL && testSuite->name != NULL &&
500  SDL_strcmp(suiteFilterName, testSuite->name) != 0) {
501  /* Skip suite */
502  SDLTest_Log("===== Test Suite %i: '%s' skipped\n",
503  suiteCounter,
504  currentSuiteName);
505  } else {
506 
507  /* Reset per-suite counters */
508  testFailedCount = 0;
509  testPassedCount = 0;
510  testSkippedCount = 0;
511 
512  /* Take time - suite start */
513  suiteStartSeconds = GetClock();
514 
515  /* Log suite started */
516  SDLTest_Log("===== Test Suite %i: '%s' started\n",
517  suiteCounter,
518  currentSuiteName);
519 
520  /* Loop over all test cases */
521  testCounter = 0;
522  while(testSuite->testCases[testCounter])
523  {
524  testCase=(SDLTest_TestCaseReference *)testSuite->testCases[testCounter];
525  currentTestName = (char *)((testCase->name) ? testCase->name : SDLTEST_INVALID_NAME_FORMAT);
526  testCounter++;
527 
528  /* Filter tests if flag set and we have a name */
529  if (testFilter == 1 && testFilterName != NULL && testCase->name != NULL &&
530  SDL_strcmp(testFilterName, testCase->name) != 0) {
531  /* Skip test */
532  SDLTest_Log("===== Test Case %i.%i: '%s' skipped\n",
533  suiteCounter,
534  testCounter,
535  currentTestName);
536  } else {
537  /* Override 'disabled' flag if we specified a test filter (i.e. force run for debugging) */
538  if (testFilter == 1 && !testCase->enabled) {
539  SDLTest_Log("Force run of disabled test since test filter was set");
540  forceTestRun = SDL_TRUE;
541  }
542 
543  /* Take time - test start */
544  testStartSeconds = GetClock();
545 
546  /* Log test started */
547  SDLTest_Log("----- Test Case %i.%i: '%s' started",
548  suiteCounter,
549  testCounter,
550  currentTestName);
551  if (testCase->description != NULL && testCase->description[0] != '\0') {
552  SDLTest_Log("Test Description: '%s'",
553  (testCase->description) ? testCase->description : SDLTEST_INVALID_NAME_FORMAT);
554  }
555 
556  /* Loop over all iterations */
557  iterationCounter = 0;
558  while(iterationCounter < testIterations)
559  {
560  iterationCounter++;
561 
562  if (userExecKey != 0) {
563  execKey = userExecKey;
564  } else {
565  execKey = SDLTest_GenerateExecKey((char *)runSeed, testSuite->name, testCase->name, iterationCounter);
566  }
567 
568  SDLTest_Log("Test Iteration %i: execKey %" SDL_PRIu64, iterationCounter, execKey);
569  testResult = SDLTest_RunTest(testSuite, testCase, execKey, forceTestRun);
570 
571  if (testResult == TEST_RESULT_PASSED) {
572  testPassedCount++;
573  totalTestPassedCount++;
574  } else if (testResult == TEST_RESULT_SKIPPED) {
575  testSkippedCount++;
576  totalTestSkippedCount++;
577  } else {
578  testFailedCount++;
579  totalTestFailedCount++;
580  }
581  }
582 
583  /* Take time - test end */
584  testEndSeconds = GetClock();
585  runtime = testEndSeconds - testStartSeconds;
586  if (runtime < 0.0f) runtime = 0.0f;
587 
588  if (testIterations > 1) {
589  /* Log test runtime */
590  SDLTest_Log("Runtime of %i iterations: %.1f sec", testIterations, runtime);
591  SDLTest_Log("Average Test runtime: %.5f sec", runtime / (float)testIterations);
592  } else {
593  /* Log test runtime */
594  SDLTest_Log("Total Test runtime: %.1f sec", runtime);
595  }
596 
597  /* Log final test result */
598  switch (testResult) {
599  case TEST_RESULT_PASSED:
600  SDLTest_Log(SDLTEST_FINAL_RESULT_FORMAT, "Test", currentTestName, "Passed");
601  break;
602  case TEST_RESULT_FAILED:
603  SDLTest_LogError(SDLTEST_FINAL_RESULT_FORMAT, "Test", currentTestName, "Failed");
604  break;
606  SDLTest_LogError(SDLTEST_FINAL_RESULT_FORMAT,"Test", currentTestName, "No Asserts");
607  break;
608  }
609 
610  /* Collect failed test case references for repro-step display */
611  if (testResult == TEST_RESULT_FAILED) {
612  failedTests[failedNumberOfTests] = testCase;
613  failedNumberOfTests++;
614  }
615  }
616  }
617 
618  /* Take time - suite end */
619  suiteEndSeconds = GetClock();
620  runtime = suiteEndSeconds - suiteStartSeconds;
621  if (runtime < 0.0f) runtime = 0.0f;
622 
623  /* Log suite runtime */
624  SDLTest_Log("Total Suite runtime: %.1f sec", runtime);
625 
626  /* Log summary and final Suite result */
627  countSum = testPassedCount + testFailedCount + testSkippedCount;
628  if (testFailedCount == 0)
629  {
630  SDLTest_Log(SDLTEST_LOG_SUMMARY_FORMAT, "Suite", countSum, testPassedCount, testFailedCount, testSkippedCount);
631  SDLTest_Log(SDLTEST_FINAL_RESULT_FORMAT, "Suite", currentSuiteName, "Passed");
632  }
633  else
634  {
635  SDLTest_LogError(SDLTEST_LOG_SUMMARY_FORMAT, "Suite", countSum, testPassedCount, testFailedCount, testSkippedCount);
636  SDLTest_LogError(SDLTEST_FINAL_RESULT_FORMAT, "Suite", currentSuiteName, "Failed");
637  }
638 
639  }
640  }
641 
642  /* Take time - run end */
643  runEndSeconds = GetClock();
644  runtime = runEndSeconds - runStartSeconds;
645  if (runtime < 0.0f) runtime = 0.0f;
646 
647  /* Log total runtime */
648  SDLTest_Log("Total Run runtime: %.1f sec", runtime);
649 
650  /* Log summary and final run result */
651  countSum = totalTestPassedCount + totalTestFailedCount + totalTestSkippedCount;
652  if (totalTestFailedCount == 0)
653  {
654  runResult = 0;
655  SDLTest_Log(SDLTEST_LOG_SUMMARY_FORMAT, "Run", countSum, totalTestPassedCount, totalTestFailedCount, totalTestSkippedCount);
656  SDLTest_Log(SDLTEST_FINAL_RESULT_FORMAT, "Run /w seed", runSeed, "Passed");
657  }
658  else
659  {
660  runResult = 1;
661  SDLTest_LogError(SDLTEST_LOG_SUMMARY_FORMAT, "Run", countSum, totalTestPassedCount, totalTestFailedCount, totalTestSkippedCount);
662  SDLTest_LogError(SDLTEST_FINAL_RESULT_FORMAT, "Run /w seed", runSeed, "Failed");
663  }
664 
665  /* Print repro steps for failed tests */
666  if (failedNumberOfTests > 0) {
667  SDLTest_Log("Harness input to repro failures:");
668  for (testCounter = 0; testCounter < failedNumberOfTests; testCounter++) {
669  SDLTest_Log(" --seed %s --filter %s", runSeed, failedTests[testCounter]->name);
670  }
671  }
672  SDL_free(failedTests);
673 
674  SDLTest_Log("Exit code: %d", runResult);
675  return runResult;
676 }
#define SDL_GetError
#define SDLTEST_INVALID_NAME_FORMAT
void SDLTest_LogTestSuiteSummary(SDLTest_TestSuiteReference *testSuites)
#define TEST_ABORTED
SDLTest_TestCaseTearDownFp testTearDown
int SDLTest_RunSuites(SDLTest_TestSuiteReference *testSuites[], const char *userRunSeed, Uint64 userExecKey, const char *filter, int testIterations)
Execute a test suite using the given run seed and execution key.
char * SDLTest_GenerateRunSeed(const int length)
Uint32(* SDL_TimerCallback)(Uint32 interval, void *param)
Definition: SDL_timer.h:81
#define SDLTEST_FINAL_RESULT_FORMAT
GLfloat f
#define SDL_InitSubSystem
static Uint32 SDLTest_TestCaseTimeout
void SDLTest_BailOut()
Timeout handler. Aborts test run and exits harness process.
float GetClock()
GLuint const GLchar * name
uint32_t Uint32
An unsigned 32-bit integer type.
Definition: SDL_stdinc.h:161
SDLTest_TestCaseSetUpFp testSetUp
#define SDL_Error
uint64_t Uint64
An unsigned 64-bit integer type.
Definition: SDL_stdinc.h:170
const SDLTest_TestCaseReference ** testCases
Uint64 SDLTest_GenerateExecKey(char *runSeed, char *suiteName, char *testName, int iteration)
#define SDL_AddTimer
static void iteration()
#define TEST_RESULT_FAILED
void SDLTest_LogError(SDL_PRINTF_FORMAT_STRING const char *fmt,...) SDL_PRINTF_VARARG_FUNC(1)
Prints given message with a timestamp in the TEST category and the ERROR priority.
Definition: SDL_test_log.c:88
void SDLTest_RandomInitTime(SDLTest_RandomContext *rndContext)
Initialize random number generator based on current system time.
SDL_TimerID SDLTest_SetTestTimeout(int timeout, void(*callback)())
Set timeout handler for test.
void SDL_free(void *mem)
void SDLTest_Md5Update(SDLTest_Md5Context *mdContext, unsigned char *inBuf, unsigned int inLen)
update digest from variable length data
Definition: SDL_test_md5.c:131
void SDLTest_Md5Init(SDLTest_Md5Context *mdContext)
initialize the context
Definition: SDL_test_md5.c:110
#define TEST_RESULT_SETUP_FAILURE
static Uint32 callback(Uint32 interval, void *param)
Definition: testtimer.c:34
#define TEST_STARTED
#define TEST_RESULT_NO_ASSERT
unsigned int SDLTest_Random(SDLTest_RandomContext *rndContext)
Initialize random number generator based on current system time.
int SDLTest_AssertSummaryToTestResult()
Converts the current assert summary state to a test result.
#define NULL
Definition: begin_code.h:143
SDL_bool
Definition: SDL_stdinc.h:130
int SDLTest_GetFuzzerInvocationCount()
#define SDL_INIT_TIMER
Definition: SDL.h:75
#define TEST_RESULT_PASSED
void SDLTest_ResetAssertSummary()
Resets the assert summary counters to zero.
#define SDL_strlen
GLbitfield GLuint64 timeout
void SDLTest_Md5Final(SDLTest_Md5Context *mdContext)
complete digest computation
Definition: SDL_test_md5.c:180
#define SDLTEST_LOG_SUMMARY_FORMAT
GLuint buffer
#define SDL_PRIu64
Definition: SDL_stdinc.h:192
int SDLTest_RunTest(SDLTest_TestSuiteReference *testSuite, SDLTest_TestCaseReference *testCase, Uint64 execKey, SDL_bool forceTestRun)
Execute a test using the given execution key.
GLuint counter
GLuint GLfloat x0
void SDLTest_Log(SDL_PRINTF_FORMAT_STRING const char *fmt,...) SDL_PRINTF_VARARG_FUNC(1)
Prints given message with a timestamp in the TEST category and INFO priority.
Definition: SDL_test_log.c:70
#define SDL_snprintf
#define SDL_malloc
void SDLTest_FuzzerInit(Uint64 execKey)
#define SDL_strcmp
#define SDL_WasInit
#define TEST_RESULT_SKIPPED
GLuint GLsizei GLsizei * length
SDLTest_TestCaseFp testCase
#define TEST_SKIPPED
void SDLTest_LogAssertSummary()
Logs summary of all assertions (total, pass, fail) since last reset as INFO or ERROR.
#define SDL_RemoveTimer
#define SDL_memset
int SDL_TimerID
Definition: SDL_timer.h:86
GLint GLint GLint GLint GLint GLint GLint GLbitfield GLenum filter
unsigned char digest[16]
Definition: SDL_test_md5.h:75
SDLTest_TestSuiteReference * testSuites[]