pandory500/SDL-mirror
2
0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

Adding elementary support for fuzzing.

Browse source 
New options: --seed [VALUE] --exec-key [EXEC-KEY] --iterations VALUE
This commit is contained in:
Markus Kauppila 2011-07-24 18:21:53 +03:00
parent f1e162b811
commit afbf90dc78
11 changed files with 207 additions and 52 deletions
Download patch file Download diff file Expand all files Collapse all files

6
test/test-automation/Makefile.am
View file

@ -1,16 +1,20 @@
ACLOCAL_AMFLAGS = -I acinclude -I build-scripts
SUBDIRS = testdummy testrect testplatform testaudio testsurface
runnerdir = .
bin_PROGRAMS = runner
runner_SOURCES = runner.c support.c
##nobase_runner_HEADERS = fuzzer.h logger.h plain_logger.h xml_logger.h xml.h
runner_CLAGS = -W -Wall -Wextra -g `sdl-config --cflags` -DSDL_NO_COMPAT
runner_LDADD = libtest.la
runner_LDFLAGS = `sdl-config --libs`
## -I .libs/libtest.so
lib_LTLIBRARIES = libtest.la
libtest_la_SOURCES = SDL_test.c logger_helpers.c plain_logger.c xml_logger.c xml.c \
common/common.c common/img_blit.c common/img_blitblend.c common/img_face.c common/img_primitives.c common/img_primitivesblend.c
common/common.c common/img_blit.c common/img_blitblend.c common/img_face.c common/img_primitives.c common/img_primitivesblend.c \
fuzzer/utl_crc32.c fuzzer/utl_md5.c fuzzer/utl_random.c fuzzer/fuzzer.c
libtest_la_CLAGS = -fPIC -g
libtest_la_LDFLAGS = `sdl-config --libs`

11
test/test-automation/SDL_test.c
View file

@ -23,6 +23,7 @@
#include <time.h>
#include "logger.h"
#include "fuzzer/fuzzer.h"
#include "SDL_test.h"
@ -36,8 +37,13 @@ int _testAssertsFailed;
int _testAssertsPassed;
void
_InitTestEnvironment()
_InitTestEnvironment(const int execKey)
{
// The execKey gets corrupted while passing arguments
// hence the global variable to circumvent the problem
InitFuzzer(globalExecKey);
_testReturnValue = TEST_RESULT_PASS;
_testAssertsFailed = 0;
_testAssertsPassed = 0;
@ -53,6 +59,8 @@ _QuitTestEnvironment()
_testReturnValue = TEST_RESULT_NO_ASSERT;
}
DeinitFuzzer();
return _testReturnValue;
}
@ -61,6 +69,7 @@ _CountFailedAsserts() {
return _testAssertsFailed;
}
void
AssertEquals(int expected, int actual, char *message, ...)
{

3
test/test-automation/SDL_test.h
View file

@ -26,6 +26,7 @@
#include "common/common.h"
#include "common/images.h"
#include "fuzzer/fuzzer.h"
extern int _testReturnValue;
extern int _testAssertsFailed;
@ -68,7 +69,7 @@ typedef struct TestCaseReference {
* Initialized the test environment such as asserts. Must be called at
* the beginning of every test case, before doing anything else.
*/
void _InitTestEnvironment();
void _InitTestEnvironment(const int execKey);
/*!
* Deinitializes the test environment and

7
test/test-automation/logger.h
View file

@ -37,7 +37,7 @@ typedef void (*SuiteEndedFp)(int testsPassed, int testsFailed, int testsSkipped,
time_t endTime, double totalRuntime);
typedef void (*TestStartedFp)(const char *testName, const char *suiteName,
const char *testDescription, time_t startTime);
const char *testDescription, int execKey, time_t startTime);
typedef void (*TestEndedFp)(const char *testName, const char *suiteName, int testResult,
time_t endTime, double totalRuntime);
@ -67,4 +67,9 @@ extern AssertWithValuesFp AssertWithValues;
extern AssertSummaryFp AssertSummary;
extern LogFp Log;
extern int globalExecKey;
//! Run seed for harness
extern const char *runSeed;
#endif

5
test/test-automation/plain_logger.c
View file

@ -80,9 +80,10 @@ PlainSuiteEnded(int testsPassed, int testsFailed, int testsSkipped,
}
void
PlainTestStarted(const char *testName, const char *suiteName, const char *testDescription, time_t startTime)
PlainTestStarted(const char *testName, const char *suiteName,
const char *testDescription, int execKey, time_t startTime)
{
Output(indentLevel++, "Executing test: %s (in %s)", testName, suiteName);
Output(indentLevel++, "Executing test: %s (in %s). Execution key: %d", testName, suiteName, execKey);
}
void

3
test/test-automation/plain_logger.h
View file

@ -55,10 +55,11 @@ void PlainSuiteEnded(int testsPassed, int testsFailed, int testsSkipped,
* \param testName Name of the test that'll be executed
* \param suiteName Name of the suite of the test
* \param testDescription Description of the test
* \param execKey Execution key for fuzzing
* \param startTime When the test started to execute
*/
void PlainTestStarted(const char *testName, const char *suiteName,
const char *testDescription, time_t startTime);
const char *testDescription, int execKey, time_t startTime);
/*!
* Prints information about the test test that was just executed

153
test/test-automation/runner.c
View file

@ -28,6 +28,9 @@
#include <sys/types.h>
#include "fuzzer/fuzzer.h"
#include "config.h"
#include "SDL_test.h"
@ -82,12 +85,24 @@ char testcase_name_substring[NAME_BUFFER_SIZE];
//! Name for user-supplied XSL style sheet name
char xsl_stylesheet_name[NAME_BUFFER_SIZE];
//! User-suppled timeout value for tests
int universal_timeout = -1;
//! Default directory of the test suites
#define DEFAULT_TEST_DIRECTORY "tests/"
int globalExecKey = -1;
const char *runSeed = "seed";
int userExecKey = 0;
//! How man time a test will be invocated
int testInvocationCount = 1;
// \todo move this upper!! (and add comments)
int totalTestFailureCount = 0, totalTestPassCount = 0, totalTestSkipCount = 0;
int testFailureCount = 0, testPassCount = 0, testSkipCount = 0;
/*!
* Holds information about test suite such as it's name
@ -674,8 +689,12 @@ CheckTestRequirements(TestCase *testCase)
* \param test result
*/
int
RunTest(TestCase *testCase)
RunTest(TestCase *testCase, const int execKey)
{
if(!testCase) {
return -1;
}
int runnable = CheckTestRequirements(testCase);
if(runnable != 1) {
return TEST_RESULT_SKIPPED;
@ -719,15 +738,15 @@ RunTest(TestCase *testCase)
* \return The return value of the test. Zero means success, non-zero failure.
*/
int
ExecuteTest(TestCase *testItem) {
ExecuteTest(TestCase *testItem, const int execKey) {
int retVal = -1;
if(execute_inproc) {
retVal = RunTest(testItem);
retVal = RunTest(testItem, execKey);
} else {
int childpid = fork();
if(childpid == 0) {
exit(RunTest(testItem));
exit(RunTest(testItem, execKey));
} else {
int stat_lock = -1;
int child = wait(&stat_lock);
@ -736,6 +755,20 @@ ExecuteTest(TestCase *testItem) {
}
}
if(retVal == TEST_RESULT_SKIPPED) {
testSkipCount++;
totalTestSkipCount++;
}
else if(retVal) {
totalTestFailureCount++;
testFailureCount++;
}
else {
totalTestPassCount++;
testPassCount++;
}
// return the value for logger
return retVal;
}
@ -890,6 +923,40 @@ ParseOptions(int argc, char *argv[])
universal_timeout = atoi(timeoutString);
}
else if(SDL_strcmp(arg, "--seed") == 0) {
if( (i + 1) < argc) {
runSeed = argv[++i];
} else {
printf("runner: seed value is missing\n");
PrintUsage();
exit(1);
}
//!Ê\todo should the seed be copied to a buffer?
}
else if(SDL_strcmp(arg, "--iterations") == 0) {
char *iterationsString = NULL;
if( (i + 1) < argc) {
iterationsString = argv[++i];
} else {
printf("runner: iterations value is missing\n");
PrintUsage();
exit(1);
}
testInvocationCount = atoi(iterationsString);
}
else if(SDL_strcmp(arg, "--exec-key") == 0) {
char *execKeyString = NULL;
if( (i + 1) < argc) {
execKeyString = argv[++i];
} else {
printf("runner: execkey value is missing\n");
PrintUsage();
exit(1);
}
userExecKey = atoi(execKeyString);
}
else if(SDL_strcmp(arg, "--test") == 0 || SDL_strcmp(arg, "-t") == 0) {
only_selected_test = 1;
char *testName = NULL;
@ -976,10 +1043,12 @@ main(int argc, char *argv[])
{
ParseOptions(argc, argv);
CRC32_CTX crcContext;
utl_crc32Init(&crcContext);
// print: Testing against SDL version fuu (rev: bar) if verbose == true
int totalTestFailureCount = 0, totalTestPassCount = 0, totalTestSkipCount = 0;
int testFailureCount = 0, testPassCount = 0, testSkipCount = 0;
char *testSuiteName = NULL;
int suiteCounter = 0;
@ -1021,54 +1090,56 @@ main(int argc, char *argv[])
char *currentSuiteName = NULL;
int suiteStartTime = SDL_GetTicks();
int notFirstSuite = 0;
int startNewSuite = 1;
TestCase *testItem = NULL;
for(testItem = testCases; testItem; testItem = testItem->next) {
if(currentSuiteName == NULL) {
currentSuiteName = testItem->suiteName;
SuiteStarted(currentSuiteName, time(0));
testFailureCount = testPassCount = testSkipCount = 0;
suiteCounter++;
if(currentSuiteName && strncmp(currentSuiteName, testItem->suiteName, NAME_BUFFER_SIZE) != 0) {
startNewSuite = 1;
}
else if(strncmp(currentSuiteName, testItem->suiteName, NAME_BUFFER_SIZE) != 0) {
const double suiteRuntime = (SDL_GetTicks() - suiteStartTime) / 1000.0f;
SuiteEnded(testPassCount, testFailureCount, testSkipCount, time(0),
suiteRuntime);
if(startNewSuite) {
if(notFirstSuite) {
const double suiteRuntime = (SDL_GetTicks() - suiteStartTime) / 1000.0f;
SuiteEnded(testPassCount, testFailureCount, testSkipCount, time(0),
suiteRuntime);
}
suiteStartTime = SDL_GetTicks();
currentSuiteName = testItem->suiteName;
SuiteStarted(currentSuiteName, time(0));
testFailureCount = testPassCount = testSkipCount = 0;
suiteCounter++;
startNewSuite = 0;
notFirstSuite = 1;
}
TestStarted(testItem->testName, testItem->suiteName,
testItem->description, time(0));
int currentIteration = testInvocationCount;
while(currentIteration > 0) {
if(userExecKey != 0) {
globalExecKey = userExecKey;
} else {
const int execKey = GenerateExecKey(crcContext, runSeed, testItem->suiteName,
testItem->testName, currentIteration);
globalExecKey = execKey;
}
const Uint32 testTimeStart = SDL_GetTicks();
TestStarted(testItem->testName, testItem->suiteName,
testItem->description, globalExecKey, time(0));
int retVal = ExecuteTest(testItem);
if(retVal == 3) {
testSkipCount++;
totalTestSkipCount++;
const Uint32 testTimeStart = SDL_GetTicks();
int retVal = ExecuteTest(testItem, globalExecKey);
const double testTotalRuntime = (SDL_GetTicks() - testTimeStart) / 1000.0f;
TestEnded(testItem->testName, testItem->suiteName, retVal, time(0), testTotalRuntime);
currentIteration--;
}
else if(retVal) {
totalTestFailureCount++;
testFailureCount++;
}
else {
totalTestPassCount++;
testPassCount++;
}
const double testTotalRuntime = (SDL_GetTicks() - testTimeStart) / 1000.0f;
TestEnded(testItem->testName, testItem->suiteName, retVal, time(0), testTotalRuntime);
}
if(currentSuiteName) {
@ -1082,11 +1153,13 @@ main(int argc, char *argv[])
const Uint32 endTicks = SDL_GetTicks();
const double totalRunTime = (endTicks - startTicks) / 1000.0f;
RunEnded(totalTestPassCount + totalTestFailureCount, suiteCounter,
RunEnded(totalTestPassCount + totalTestFailureCount + totalTestSkipCount, suiteCounter,
totalTestPassCount, totalTestFailureCount, totalTestSkipCount, time(0), totalRunTime);
// Some SDL subsystem might be init'ed so shut them down
SDL_Quit();
utl_crc32Done(&crcContext);
return (totalTestFailureCount ? 1 : 0);
}

13
test/test-automation/testdummy/testdummy.c
View file

@ -29,6 +29,7 @@
#include <SDL/SDL.h>
#include "../SDL_test.h"
//#include "fuzzer/fuzzer.h"
/* Test case references */
static const TestCaseReference test1 =
@ -89,6 +90,18 @@ void
dummycase1(void *arg)
{
AssertEquals(5, 5, "Assert message");
for(; 0 ;) {
int min = 50;
int max = 69;
int random = RandomPositiveIntegerInRange(min, max);
if(random < min || random > max ) {
AssertFail("Generated incorrect integer");
}
Log(0, "%d", random);
}
//Log(0, "Random: %s", RandomAsciiString());
}
void

36
test/test-automation/testrect/testrect.c
View file

@ -12,9 +12,14 @@
static const TestCaseReference test1 =
(TestCaseReference){ "rect_testIntersectRectAndLine", "description", TEST_ENABLED, 0, 0 };
static const TestCaseReference test2 =
(TestCaseReference){ "rect_testIntersectRectAndLineFuzzed", "Tests rect to line intersection with fuzzed values", TEST_ENABLED, 0, 0 };
/* Test suite */
extern const TestCaseReference *testSuite[] = {
&test1, NULL
&test1, &test2, NULL
};
TestCaseReference **QueryTestSuite() {
@ -23,6 +28,9 @@ TestCaseReference **QueryTestSuite() {
/*!
* \brief Tests SDL_IntersectRectAndLine()
*
* \sa
* http://wiki.libsdl.org/moin.cgi/SDL_IntersectRectAndLine
*/
int rect_testIntersectRectAndLine (void *arg)
{
@ -131,3 +139,29 @@ int rect_testIntersectRectAndLine (void *arg)
"diagonal line to upper right was incorrectly clipped: %d,%d - %d,%d",
x1, y1, x2, y2);
}
/*!
* \brief Tests SDL_IntersectRectAndLine()
*
* \sa
* http://wiki.libsdl.org/moin.cgi/SDL_IntersectRectAndLine
*/
int rect_testIntersectRectAndLineFuzzed(void *arg)
{
SDL_Rect rect = { 0, 0, RandomInteger(), RandomInteger() };
int x1, y1;
int x2, y2;
SDL_bool clipped;
x1 = -RandomInteger();
y1 = RandomInteger();
x2 = -RandomInteger();
y2 = RandomInteger();
clipped = SDL_IntersectRectAndLine(&rect, &x1, &y1, &x2, &y2);
AssertTrue( !clipped,
/*&& x1 == -10 && y1 == 0 && x2 == -10 && y2 == 31, */
"line outside to the left was incorrectly clipped: %d,%d - %d,%d",
x1, y1, x2, y2);
}

14
test/test-automation/xml_logger.c
View file

@ -35,6 +35,7 @@ const char *documentRoot = "testlog";
const char *parametersElementName = "parameters";
const char *parameterElementName = "parameter";
const char *startTimeElementName = "startTime";
const char *execKeyElementName = "executionKey";
const char *numSuitesElementName = "numSuites";
const char *numTestElementName = "numTests";
const char *numPassedTestsElementName = "numPassedTests";
@ -310,7 +311,7 @@ XMLSuiteEnded(int testsPassed, int testsFailed, int testsSkipped,
void
XMLTestStarted(const char *testName, const char *suiteName,
const char *testDescription, time_t startTime)
const char *testDescription, int execKey, time_t startTime)
{
char * output = XMLOpenElement(testElementName);
XMLOutputter(indentLevel++, YES, output);
@ -335,6 +336,17 @@ XMLTestStarted(const char *testName, const char *suiteName,
output = XMLCloseElement(descriptionElementName);
XMLOutputter(--indentLevel, YES, output);
// log exec key
output = XMLOpenElement(execKeyElementName);
XMLOutputter(indentLevel++, NO, output);
output = XMLAddContent(IntToString(execKey));
XMLOutputter(indentLevel, NO, output);
output = XMLCloseElement(execKeyElementName);
XMLOutputter(--indentLevel, YES, output);
// log start time
output = XMLOpenElement(startTimeElementName);
XMLOutputter(indentLevel++, NO, output);

4
test/test-automation/xml_logger.h
View file

@ -53,9 +53,11 @@ void XMLSuiteEnded(int testsPassed, int testsFailed, int testsSkipped,
* \param testName Name of the test that'll be executed
* \param suiteName Name of the suite of the test
* \param testDescription Description of the test
* \param execKey Execution key for fuzzing
* \param startTime When the test started to execute
*/
void XMLTestStarted(const char *testName, const char *suiteName, const char *testDescription, time_t startTime);
void XMLTestStarted(const char *testName, const char *suiteName,
const char *testDescription, int execKey, time_t startTime);
/*!
* Prints information about the test test that was just executed in XML