Adding elementary support for fuzzing.

New options: --seed [VALUE] --exec-key [EXEC-KEY] --iterations VALUE

test/test-automation/Makefile.am

@@ -1,16 +1,20 @@
 ACLOCAL_AMFLAGS = -I acinclude -I build-scripts
 
 SUBDIRS = testdummy testrect testplatform testaudio testsurface
+runnerdir = .
 
 bin_PROGRAMS = runner
 runner_SOURCES = runner.c support.c
+##nobase_runner_HEADERS = fuzzer.h logger.h plain_logger.h  xml_logger.h xml.h
 runner_CLAGS = -W -Wall -Wextra -g `sdl-config --cflags` -DSDL_NO_COMPAT
 runner_LDADD = libtest.la
 runner_LDFLAGS = `sdl-config --libs`
+## -I .libs/libtest.so
 
 lib_LTLIBRARIES = libtest.la
 libtest_la_SOURCES = SDL_test.c logger_helpers.c plain_logger.c xml_logger.c xml.c \
-	common/common.c common/img_blit.c common/img_blitblend.c common/img_face.c common/img_primitives.c common/img_primitivesblend.c
+	common/common.c common/img_blit.c common/img_blitblend.c common/img_face.c common/img_primitives.c common/img_primitivesblend.c \
+	fuzzer/utl_crc32.c fuzzer/utl_md5.c fuzzer/utl_random.c fuzzer/fuzzer.c
 libtest_la_CLAGS = -fPIC -g
 libtest_la_LDFLAGS = `sdl-config --libs`
 

test/test-automation/SDL_test.c

@@ -23,6 +23,7 @@
 #include <time.h>
 
 #include "logger.h"
+#include "fuzzer/fuzzer.h"
 
 #include "SDL_test.h"
 
@@ -36,8 +37,13 @@ int _testAssertsFailed;
 int _testAssertsPassed;
 
 void
-_InitTestEnvironment()
+_InitTestEnvironment(const int execKey)
 {
+	// The execKey gets corrupted while passing arguments
+	// hence the global variable to circumvent the problem
+	InitFuzzer(globalExecKey);
+
+
 	_testReturnValue = TEST_RESULT_PASS;
 	_testAssertsFailed = 0;
 	_testAssertsPassed = 0;
@@ -53,6 +59,8 @@ _QuitTestEnvironment()
 		_testReturnValue = TEST_RESULT_NO_ASSERT;
 	}
 
+	DeinitFuzzer();
+
 	return _testReturnValue;
 }
 
@@ -61,6 +69,7 @@ _CountFailedAsserts() {
 	return _testAssertsFailed;
 }
 
+
 void
 AssertEquals(int expected, int actual, char *message, ...)
 {

test/test-automation/SDL_test.h

@@ -26,6 +26,7 @@
 #include "common/common.h"
 #include "common/images.h"
 
+#include "fuzzer/fuzzer.h"
 
 extern int _testReturnValue;
 extern int _testAssertsFailed;
@@ -68,7 +69,7 @@ typedef struct TestCaseReference {
  *  Initialized the test environment such as asserts. Must be called at
  *  the beginning of every test case, before doing anything else.
  */
-void _InitTestEnvironment();
+void _InitTestEnvironment(const int execKey);
 
 /*!
  *  Deinitializes the test environment and

test/test-automation/logger.h

@@ -37,7 +37,7 @@ typedef	void (*SuiteEndedFp)(int testsPassed, int testsFailed, int testsSkipped,
 		                time_t endTime, double totalRuntime);
 
 typedef	void (*TestStartedFp)(const char *testName, const char *suiteName,
-                              const char *testDescription, time_t startTime);
+                              const char *testDescription, int execKey, time_t startTime);
 typedef	void (*TestEndedFp)(const char *testName, const char *suiteName, int testResult,
                             time_t endTime, double totalRuntime);
 
@@ -67,4 +67,9 @@ extern AssertWithValuesFp AssertWithValues;
 extern AssertSummaryFp AssertSummary;
 extern LogFp Log;
 
+extern int globalExecKey;
+//! Run seed for harness
+extern const char *runSeed;
+
+
 #endif

test/test-automation/plain_logger.c

@@ -80,9 +80,10 @@ PlainSuiteEnded(int testsPassed, int testsFailed, int testsSkipped,
 }
 
 void
-PlainTestStarted(const char *testName, const char *suiteName, const char *testDescription, time_t startTime)
+PlainTestStarted(const char *testName, const char *suiteName,
+				const char *testDescription, int execKey, time_t startTime)
 {
-	Output(indentLevel++, "Executing test: %s (in %s)", testName, suiteName);
+	Output(indentLevel++, "Executing test: %s (in %s). Execution key: %d", testName, suiteName, execKey);
 }
 
 void

test/test-automation/plain_logger.h

@@ -55,10 +55,11 @@ void PlainSuiteEnded(int testsPassed, int testsFailed, int testsSkipped,
  * \param testName Name of the test that'll be executed
  * \param suiteName Name of the suite of the test
  * \param testDescription Description of the test
+ * \param execKey Execution key for fuzzing
  * \param startTime When the test started to execute
  */
 void PlainTestStarted(const char *testName, const char *suiteName,
-                      const char *testDescription, time_t startTime);
+                      const char *testDescription, int execKey, time_t startTime);
 
 /*!
  * Prints information about the test test that was just executed

test/test-automation/runner.c

@@ -28,6 +28,9 @@
 
 #include <sys/types.h>
 
+#include "fuzzer/fuzzer.h"
+
+
 #include "config.h"
 
 #include "SDL_test.h"
@@ -82,12 +85,24 @@ char testcase_name_substring[NAME_BUFFER_SIZE];
 
 //! Name for user-supplied XSL style sheet name
 char xsl_stylesheet_name[NAME_BUFFER_SIZE];
-
+//! User-suppled timeout value for tests
 int universal_timeout = -1;
 
 //! Default directory of the test suites
 #define DEFAULT_TEST_DIRECTORY "tests/"
 
+int globalExecKey = -1;
+const char *runSeed = "seed";
+
+int userExecKey = 0;
+
+//! How man time a test will be invocated
+int testInvocationCount = 1;
+
+// \todo move this upper!! (and add comments)
+int totalTestFailureCount = 0, totalTestPassCount = 0, totalTestSkipCount = 0;
+int testFailureCount = 0, testPassCount = 0, testSkipCount = 0;
+
 
 /*!
  * Holds information about test suite such as it's name
@@ -674,8 +689,12 @@ CheckTestRequirements(TestCase *testCase)
  * \param test result
  */
 int
-RunTest(TestCase *testCase)
+RunTest(TestCase *testCase, const int execKey)
 {
+	if(!testCase) {
+		return -1;
+	}
+
 	int runnable = CheckTestRequirements(testCase);
 	if(runnable != 1) {
 		return TEST_RESULT_SKIPPED;
@@ -719,15 +738,15 @@ RunTest(TestCase *testCase)
  * \return The return value of the test. Zero means success, non-zero failure.
  */
 int
-ExecuteTest(TestCase *testItem) {
+ExecuteTest(TestCase *testItem, const int execKey) {
 	int retVal = -1;
 
 	if(execute_inproc) {
-		retVal = RunTest(testItem);
+		retVal = RunTest(testItem, execKey);
 	} else {
 		int childpid = fork();
 		if(childpid == 0) {
-			exit(RunTest(testItem));
+			exit(RunTest(testItem, execKey));
 		} else {
 			int stat_lock = -1;
 			int child = wait(&stat_lock);
@@ -736,6 +755,20 @@ ExecuteTest(TestCase *testItem) {
 		}
 	}
 
+	if(retVal == TEST_RESULT_SKIPPED) {
+		testSkipCount++;
+		totalTestSkipCount++;
+	}
+	else if(retVal) {
+		totalTestFailureCount++;
+		testFailureCount++;
+	}
+	else {
+		totalTestPassCount++;
+		testPassCount++;
+	}
+
+	// return the value for logger
 	return retVal;
 }
 
@@ -890,6 +923,40 @@ ParseOptions(int argc, char *argv[])
 
     	  universal_timeout = atoi(timeoutString);
       }
+      else if(SDL_strcmp(arg, "--seed") == 0) {
+    	  if( (i + 1) < argc)  {
+    		  runSeed = argv[++i];
+    	  }  else {
+    		  printf("runner: seed value is missing\n");
+    		  PrintUsage();
+    		  exit(1);
+    	  }
+    	  //!Ê\todo should the seed be copied to a buffer?
+      }
+      else if(SDL_strcmp(arg, "--iterations") == 0) {
+    	  char *iterationsString = NULL;
+    	  if( (i + 1) < argc)  {
+    		  iterationsString = argv[++i];
+    	  }  else {
+    		  printf("runner: iterations value is missing\n");
+    		  PrintUsage();
+    		  exit(1);
+    	  }
+
+    	  testInvocationCount = atoi(iterationsString);
+      }
+      else if(SDL_strcmp(arg, "--exec-key") == 0) {
+    	  char *execKeyString = NULL;
+    	  if( (i + 1) < argc)  {
+    		  execKeyString = argv[++i];
+    	  }  else {
+    		  printf("runner: execkey value is missing\n");
+    		  PrintUsage();
+    		  exit(1);
+    	  }
+
+    	  userExecKey = atoi(execKeyString);
+      }
       else if(SDL_strcmp(arg, "--test") == 0 || SDL_strcmp(arg, "-t") == 0) {
     	  only_selected_test = 1;
     	  char *testName = NULL;
@@ -976,10 +1043,12 @@ main(int argc, char *argv[])
 {
 	ParseOptions(argc, argv);
 
+	CRC32_CTX crcContext;
+	utl_crc32Init(&crcContext);
+
 	// print: Testing against SDL version fuu (rev: bar) if verbose == true
 
-	int totalTestFailureCount = 0, totalTestPassCount = 0, totalTestSkipCount = 0;
+
-	int testFailureCount = 0, testPassCount = 0, testSkipCount = 0;
 	char *testSuiteName = NULL;
 	int suiteCounter = 0;
 
@@ -1021,54 +1090,56 @@ main(int argc, char *argv[])
 	char *currentSuiteName = NULL;
 	int suiteStartTime = SDL_GetTicks();
 
+	int notFirstSuite = 0;
+	int startNewSuite = 1;
 	TestCase *testItem = NULL;
 	for(testItem = testCases; testItem; testItem = testItem->next) {
-		if(currentSuiteName == NULL) {
+		if(currentSuiteName && strncmp(currentSuiteName, testItem->suiteName, NAME_BUFFER_SIZE) != 0) {
-			currentSuiteName = testItem->suiteName;
+			startNewSuite = 1;
-			SuiteStarted(currentSuiteName, time(0));
-
-			testFailureCount = testPassCount = testSkipCount = 0;
-
-			suiteCounter++;
 		}
-		else if(strncmp(currentSuiteName, testItem->suiteName, NAME_BUFFER_SIZE) != 0) {
+
+		if(startNewSuite) {
+			if(notFirstSuite) {
 				const double suiteRuntime = (SDL_GetTicks() - suiteStartTime) / 1000.0f;
 
 				SuiteEnded(testPassCount, testFailureCount, testSkipCount, time(0),
 							suiteRuntime);
+			}
 
 			suiteStartTime = SDL_GetTicks();
 
 			currentSuiteName = testItem->suiteName;
 			SuiteStarted(currentSuiteName, time(0));
-
 			testFailureCount = testPassCount = testSkipCount = 0;
-
 			suiteCounter++;
+
+			startNewSuite = 0;
+			notFirstSuite = 1;
+		}
+
+		int currentIteration = testInvocationCount;
+		while(currentIteration > 0) {
+			if(userExecKey != 0) {
+				globalExecKey = userExecKey;
+			} else {
+				const int execKey = GenerateExecKey(crcContext, runSeed, testItem->suiteName,
+											  testItem->testName, currentIteration);
+				globalExecKey = execKey;
 			}
 
 			TestStarted(testItem->testName, testItem->suiteName,
-                    testItem->description, time(0));
+						testItem->description, globalExecKey, time(0));
 
 			const Uint32 testTimeStart = SDL_GetTicks();
 
-		int retVal = ExecuteTest(testItem);
+			int retVal = ExecuteTest(testItem, globalExecKey);
-		if(retVal == 3) {
-			testSkipCount++;
-			totalTestSkipCount++;
-		}
-		else if(retVal) {
-			totalTestFailureCount++;
-			testFailureCount++;
-		}
-		else {
-			totalTestPassCount++;
-			testPassCount++;
-		}
 
 			const double testTotalRuntime = (SDL_GetTicks() - testTimeStart) / 1000.0f;
 
 			TestEnded(testItem->testName, testItem->suiteName, retVal, time(0), testTotalRuntime);
+
+			currentIteration--;
+		}
 	}
 
 	if(currentSuiteName) {
@@ -1082,11 +1153,13 @@ main(int argc, char *argv[])
 	const Uint32 endTicks = SDL_GetTicks();
 	const double totalRunTime = (endTicks - startTicks) / 1000.0f;
 
-	RunEnded(totalTestPassCount + totalTestFailureCount, suiteCounter,
+	RunEnded(totalTestPassCount + totalTestFailureCount + totalTestSkipCount, suiteCounter,
 			 totalTestPassCount, totalTestFailureCount, totalTestSkipCount, time(0), totalRunTime);
 
 	// Some SDL subsystem might be init'ed so shut them down
 	SDL_Quit();
 
+	utl_crc32Done(&crcContext);
+
 	return (totalTestFailureCount ? 1 : 0);
 }

test/test-automation/testdummy/testdummy.c

@@ -29,6 +29,7 @@
 #include <SDL/SDL.h>
 
 #include "../SDL_test.h"
+//#include "fuzzer/fuzzer.h"
 
 /* Test case references */
 static const TestCaseReference test1 =
@@ -89,6 +90,18 @@ void
 dummycase1(void *arg)
 {
 	AssertEquals(5, 5, "Assert message");
+
+	for(; 0 ;) {
+		int min = 50;
+		int max = 69;
+		int random =  RandomPositiveIntegerInRange(min, max);
+		if(random < min || random > max ) {
+			AssertFail("Generated incorrect integer");
+		}
+		Log(0, "%d", random);
+	}
+
+	//Log(0, "Random: %s", RandomAsciiString());
 }
 
 void

test/test-automation/testrect/testrect.c

@@ -12,9 +12,14 @@
 static const TestCaseReference test1 =
 		(TestCaseReference){ "rect_testIntersectRectAndLine", "description", TEST_ENABLED, 0, 0 };
 
+static const TestCaseReference test2 =
+		(TestCaseReference){ "rect_testIntersectRectAndLineFuzzed", "Tests rect to line intersection with fuzzed values", TEST_ENABLED, 0, 0 };
+
+
+
 /* Test suite */
 extern const TestCaseReference *testSuite[] =  {
-	&test1, NULL
+	&test1, &test2, NULL
 };
 
 TestCaseReference **QueryTestSuite() {
@@ -23,6 +28,9 @@ TestCaseReference **QueryTestSuite() {
 
 /*!
  * \brief Tests SDL_IntersectRectAndLine()
+ *
+ * \sa
+ * http://wiki.libsdl.org/moin.cgi/SDL_IntersectRectAndLine
  */
 int rect_testIntersectRectAndLine (void *arg)
 {
@@ -131,3 +139,29 @@ int rect_testIntersectRectAndLine (void *arg)
         "diagonal line to upper right was incorrectly clipped: %d,%d - %d,%d",
         x1, y1, x2, y2);
 }
+
+/*!
+ * \brief Tests SDL_IntersectRectAndLine()
+ *
+ * \sa
+ * http://wiki.libsdl.org/moin.cgi/SDL_IntersectRectAndLine
+ */
+int rect_testIntersectRectAndLineFuzzed(void *arg)
+{
+	SDL_Rect rect = { 0, 0, RandomInteger(), RandomInteger() };
+    int x1, y1;
+    int x2, y2;
+    SDL_bool clipped;
+
+    x1 = -RandomInteger();
+    y1 = RandomInteger();
+    x2 = -RandomInteger();
+    y2 = RandomInteger();
+    clipped = SDL_IntersectRectAndLine(&rect, &x1, &y1, &x2, &y2);
+
+    AssertTrue( !clipped,
+                /*&& x1 == -10 && y1 == 0 && x2 == -10 && y2 == 31, */
+                "line outside to the left was incorrectly clipped: %d,%d - %d,%d",
+                x1, y1, x2, y2);
+}
+

test/test-automation/xml_logger.c

@@ -35,6 +35,7 @@ const char *documentRoot = "testlog";
 const char *parametersElementName = "parameters";
 const char *parameterElementName = "parameter";
 const char *startTimeElementName = "startTime";
+const char *execKeyElementName = "executionKey";
 const char *numSuitesElementName = "numSuites";
 const char *numTestElementName = "numTests";
 const char *numPassedTestsElementName = "numPassedTests";
@@ -310,7 +311,7 @@ XMLSuiteEnded(int testsPassed, int testsFailed, int testsSkipped,
 
 void
 XMLTestStarted(const char *testName, const char *suiteName,
-			  const char *testDescription, time_t startTime)
+			  const char *testDescription, int execKey, time_t startTime)
 {
 	char * output = XMLOpenElement(testElementName);
 	XMLOutputter(indentLevel++, YES, output);
@@ -335,6 +336,17 @@ XMLTestStarted(const char *testName, const char *suiteName,
 	output = XMLCloseElement(descriptionElementName);
 	XMLOutputter(--indentLevel, YES, output);
 
+	// log exec key
+	output = XMLOpenElement(execKeyElementName);
+	XMLOutputter(indentLevel++, NO, output);
+
+	output = XMLAddContent(IntToString(execKey));
+	XMLOutputter(indentLevel, NO, output);
+
+	output = XMLCloseElement(execKeyElementName);
+	XMLOutputter(--indentLevel, YES, output);
+
+	// log start time
 	output = XMLOpenElement(startTimeElementName);
 	XMLOutputter(indentLevel++, NO, output);
 

test/test-automation/xml_logger.h

@@ -53,9 +53,11 @@ void XMLSuiteEnded(int testsPassed, int testsFailed, int testsSkipped,
  * \param testName Name of the test that'll be executed
  * \param suiteName Name of the suite of the test
  * \param testDescription Description of the test
+ * \param execKey Execution key for fuzzing
  * \param startTime When the test started to execute
  */
-void XMLTestStarted(const char *testName, const char *suiteName, const char *testDescription, time_t startTime);
+void XMLTestStarted(const char *testName, const char *suiteName,
+					const char *testDescription, int execKey, time_t startTime);
 
 /*!
  * Prints information about the test test that was just executed in XML