diff --git a/test/test-automation/Makefile.am b/test/test-automation/Makefile.am index 0367f1969..e9782236f 100644 --- a/test/test-automation/Makefile.am +++ b/test/test-automation/Makefile.am @@ -3,15 +3,10 @@ ACLOCAL_AMFLAGS = -I acinclude -I build-scripts SUBDIRS = testdummy testrect testplatform bin_PROGRAMS = runner -runner_SOURCES = runner.c SDL_test.c +runner_SOURCES = runner.c SDL_test.c logger.c xml_logger.c plain_logger.c xml.c runner_CLAGS = -W -Wall -Wextra -g `sdl-config --cflags` -DSDL_NO_COMPAT runner_LDFLAGS = `sdl-config --libs` -bin_PROGRAMS = logger -logger_SOURCES = xml_logger.c xml.c plain_logger.c logger.c -logger_CLAGS = -W -Wall -Wextra -g `sdl-config --cflags` -DSDL_NO_COMPAT -logger_LDFLAGS = `sdl-config --libs` - install: install-tests install-tests: -mkdir tests diff --git a/test/test-automation/SDL_test.c b/test/test-automation/SDL_test.c index 18d02d98a..589ae9198 100644 --- a/test/test-automation/SDL_test.c +++ b/test/test-automation/SDL_test.c @@ -24,6 +24,8 @@ #include /* printf/fprintf */ #include /* va_list */ +#include "logger.h" + #include "SDL_test.h" /*! \brief return value of test case. Non-zero value means that the test failed */ @@ -36,8 +38,16 @@ int _testAssertsFailed; int _testAssertsPassed; void -_TestCaseInit() +_TestCaseInit(const int enable_xml_logging) { + // setup logging functions + // rather afwul way to do it, but function pointers didn't work + if(enable_xml_logging) { + SetupXMLLogger(); + } else { + SetupPlainLogger(); + } + _testReturnValue = 0; _testAssertsFailed = 0; _testAssertsPassed = 0; @@ -46,8 +56,9 @@ _TestCaseInit() int _TestCaseQuit() { - //! \todo make the test fail, if it does not contain any asserts - printf("Asserts: passed %d, failed %d\n", _testAssertsPassed, _testAssertsFailed); + //printf("Asserts: passed %d, failed %d\n", _testAssertsPassed, _testAssertsFailed); + AssertSummary(_testAssertsFailed + _testAssertsPassed, + _testAssertsFailed, _testAssertsPassed); if(_testAssertsFailed == 0 && _testAssertsPassed == 0) { _testReturnValue = 2; @@ -66,11 +77,15 @@ AssertEquals(Uint32 expected, Uint32 actual, char* message, ...) va_start( args, message ); SDL_vsnprintf( buf, sizeof(buf), message, args ); va_end( args ); - printf("AssertEquals failed: expected %d, got %d; %s\n", expected, actual, buf); + //printf("AssertEquals failed: expected %d, got %d; %s\n", expected, actual, buf); + Assert("AssertEquals", 0, buf, 0); + _testReturnValue = 1; _testAssertsFailed++; } else { - printf("AssertEquals passed\n"); + //printf("AssertEquals passed\n"); + Assert("AssertEquals", 1, "AssertEquals passed", 0); + _testAssertsPassed++; } } @@ -86,12 +101,15 @@ AssertTrue(int condition, char *message, ...) SDL_vsnprintf( buf, sizeof(buf), message, args ); va_end( args ); - printf("AssertTrue failed: %s\n", buf); + //printf("AssertTrue failed: %s\n", buf); + Assert("AssertTrue", 0, buf, 0); + _testReturnValue = 1; _testAssertsFailed++; } else { - printf("AssertTrue passed\n"); - _testAssertsPassed++; + //printf("AssertTrue passed\n"); + Assert("AssertTrue", 1, "AssertTrue passed", 0); + _testAssertsPassed++; } } @@ -105,7 +123,8 @@ AssertPass(char *message, ...) SDL_vsnprintf( buf, sizeof(buf), message, args ); va_end( args ); - printf("AssertPass: %s\n", buf); + //printf("AssertPass: %s\n", buf); + Assert("AssertPass", 1, buf, 0); _testAssertsPassed++; } @@ -120,7 +139,8 @@ AssertFail(char *message, ...) SDL_vsnprintf( buf, sizeof(buf), message, args ); va_end( args ); - printf("AssertFail: %s\n", buf); + //printf("AssertFail: %s\n", buf); + Assert("AssertFail", 0, buf, 0); _testAssertsFailed++; } diff --git a/test/test-automation/SDL_test.h b/test/test-automation/SDL_test.h index f2e387e50..af1b5a1ec 100644 --- a/test/test-automation/SDL_test.h +++ b/test/test-automation/SDL_test.h @@ -23,6 +23,8 @@ #include +#include "logger.h" + extern int _testReturnValue; extern int _testAssertsFailed; extern int _testAssertsPassed; @@ -52,7 +54,7 @@ typedef struct TestCaseReference { * the beginning of every test case, before doing * anything else. */ -void _TestCaseInit(); +void _TestCaseInit(const int enable_xml_logging); /*! \fn _TestCaseQuit * Deinitializes and exits the test case diff --git a/test/test-automation/logger.c b/test/test-automation/logger.c index a7969eb6f..a082e40ba 100644 --- a/test/test-automation/logger.c +++ b/test/test-automation/logger.c @@ -18,8 +18,45 @@ SuiteEndedFp SuiteEnded = 0; TestStartedFp TestStarted = 0; TestEndedFp TestEnded = 0; AssertFp Assert = 0; +AssertSummaryFp AssertSummary = 0; LogFp Log = 0; +int +SetupXMLLogger() +{ + RunStarted = XMLRunStarted; + RunEnded = XMLRunEnded; + + SuiteStarted = XMLSuiteStarted; + SuiteEnded = XMLSuiteEnded; + + TestStarted = XMLTestStarted; + TestEnded = XMLTestEnded; + + Assert = XMLAssert; + AssertSummary = XMLAssertSummary; + + Log = XMLLog; +} + +int +SetupPlainLogger() +{ + RunStarted = PlainRunStarted; + RunEnded = PlainRunEnded; + + SuiteStarted = PlainSuiteStarted; + SuiteEnded = PlainSuiteEnded; + + TestStarted = PlainTestStarted; + TestEnded = PlainTestEnded; + + Assert = PlainAssert; + AssertSummary = PlainAssertSummary; + + Log = PlainLog; +} + /*! * Prints the given message to stderr. Function adds nesting * to the output. @@ -39,6 +76,7 @@ LogGenericOutput(const char *message, ...) fflush(stderr); } +#if 0 /*! * Test app for logging functionality */ @@ -48,29 +86,9 @@ main(int argc, char *argv[]) int xml_enabled = 1; if(xml_enabled) { - RunStarted = XMLRunStarted; - RunEnded = XMLRunEnded; - - SuiteStarted = XMLSuiteStarted; - SuiteEnded = XMLSuiteEnded; - - TestStarted = XMLTestStarted; - TestEnded = XMLTestEnded; - - Assert = XMLAssert; - Log = XMLLog; + SetupXMLLogger(); } else { - RunStarted = PlainRunStarted; - RunEnded = PlainRunEnded; - - SuiteStarted = PlainSuiteStarted; - SuiteEnded = PlainSuiteEnded; - - TestStarted = PlainTestStarted; - TestEnded = PlainTestEnded; - - Assert = PlainAssert; - Log = PlainLog; + SetupPlainLogger(); } RunStarted(LogGenericOutput, "some_here&here", 0); @@ -84,3 +102,4 @@ main(int argc, char *argv[]) return 0; } +#endif diff --git a/test/test-automation/logger.h b/test/test-automation/logger.h index 9a570143a..b4efe0333 100644 --- a/test/test-automation/logger.h +++ b/test/test-automation/logger.h @@ -26,7 +26,6 @@ // Function pointer to function which handles to output typedef int (*LogOutputFp)(const char *, ...); - /*! * Generic logger interface * @@ -42,12 +41,29 @@ typedef void (*SuiteEndedFp)(int testsPassed, int testsFailed, int testsSkipped, typedef void (*TestStartedFp)(const char *testName, const char *suiteName, const char *testDescription, time_t startTime); typedef void (*TestEndedFp)(const char *testName, const char *suiteName, int testResult, - int numAsserts, time_t endTime, time_t totalRuntime); + time_t endTime, time_t totalRuntime); +/*! + * Note: for assertResult, non-zero == pass, zero == failure + * + */ typedef void (*AssertFp)(const char *assertName, int assertResult, const char *assertMessage, time_t eventTime); +typedef void (*AssertSummaryFp)(int numAsserts, int numAssertsFailed, int numAssertsPass); + typedef void (*LogFp)(const char *logMessage, time_t eventTime); +int LogGenericOutput(const char *message, ...); + +extern RunStartedFp RunStarted; +extern RunEndedFp RunEnded; +extern SuiteStartedFp SuiteStarted; +extern SuiteEndedFp SuiteEnded; +extern TestStartedFp TestStarted; +extern TestEndedFp TestEnded; +extern AssertFp Assert; +extern AssertSummaryFp AssertSummary; +extern LogFp Log; #endif diff --git a/test/test-automation/plain_logger.c b/test/test-automation/plain_logger.c index 0f5902f1b..03c6a7b90 100644 --- a/test/test-automation/plain_logger.c +++ b/test/test-automation/plain_logger.c @@ -13,39 +13,44 @@ void PlainRunStarted(LogOutputFp outputFn, const char *runnerParameters, time_t eventTime) { logger = outputFn; + logger("Test run started"); + logger("Given command line options: %s", "add options"); } void PlainRunEnded(int testCount, int suiteCount, int testPassCount, int testFailCount, time_t endTime, time_t totalRuntime) { - // \todo add total number of tests, suites, pass/failure test count + logger("Ran %d tests in %0.5f seconds.", testCount, totalRuntime); + + logger("%d tests passed", testPassCount); + logger("%d tests failed", testFailCount); } void PlainSuiteStarted(const char *suiteName, time_t eventTime) { - logger("Executing tests in %s\n", suiteName); + logger("Executing tests in %s", suiteName); } void PlainSuiteEnded(int testsPassed, int testsFailed, int testsSkipped, double endTime, time_t totalRuntime) { - logger("Suite executed. %d passed, %d failed and %d skipped\n", testsPassed, testsFailed, testsSkipped); + logger("Suite executed. %d passed, %d failed and %d skipped", testsPassed, testsFailed, testsSkipped); } void PlainTestStarted(const char *testName, const char *suiteName, const char *testDescription, time_t startTime) { + logger("test %s (in %s) started", testName, suiteName); } void PlainTestEnded(const char *testName, const char *suiteName, - int testResult, int numAsserts, time_t endTime, time_t totalRuntime) + int testResult, time_t endTime, time_t totalRuntime) { - logger("Asserts:%d\n", numAsserts); - logger("%s: ok\n", testName); + logger("%s: ok", testName); } void @@ -53,12 +58,19 @@ PlainAssert(const char *assertName, int assertResult, const char *assertMessage, time_t eventTime) { const char *result = (assertResult) ? "passed" : "failed"; - logger("%s %d: %s\n", assertName, assertResult, assertMessage); + logger("%s %d: %s", assertName, assertResult, assertMessage); +} + +void +PlainAssertSummary(int numAsserts, int numAssertsFailed, int numAssertsPass) +{ + logger("Asserts:%d", numAsserts); } void PlainLog(const char *logMessage, time_t eventTime) { + logger("%s %d", logMessage, eventTime); } #endif diff --git a/test/test-automation/plain_logger.h b/test/test-automation/plain_logger.h index 4bfb9226e..d534867a0 100644 --- a/test/test-automation/plain_logger.h +++ b/test/test-automation/plain_logger.h @@ -17,11 +17,15 @@ void PlainTestStarted(const char *testName, const char *suiteName, const char *testDescription, time_t startTime); void PlainTestEnded(const char *testName, const char *suiteName, - int testResult, int numAsserts, time_t endTime, time_t totalRuntime); + int testResult, time_t endTime, time_t totalRuntime); + void PlainAssert(const char *assertName, int assertResult, const char *assertMessage, time_t eventTime); +void PlainAssertSummary(int numAsserts, int numAssertsFailed, int numAssertsPass); + + void PlainLog(const char *logMessage, time_t eventTime); #endif diff --git a/test/test-automation/runner.c b/test/test-automation/runner.c index a2640d23f..35075b7ee 100644 --- a/test/test-automation/runner.c +++ b/test/test-automation/runner.c @@ -29,12 +29,12 @@ #include #include "SDL_test.h" - +#include "logger.h" //!< Function pointer to a test case function typedef void (*TestCaseFp)(void *arg); //!< Function pointer to a test case init function -typedef void (*TestCaseInitFp)(void); +typedef void (*TestCaseInitFp)(const int); //!< Function pointer to a test case quit function typedef int (*TestCaseQuitFp)(void); @@ -49,6 +49,8 @@ static int only_selected_test = 0; static int only_selected_suite = 0; //!< Flag for executing only tests that contain certain string in their name static int only_tests_with_string = 0; +//!< Flag for enabling XML logging +static int xml_enabled = 0; //!< Size of the test and suite name buffers @@ -86,6 +88,10 @@ typedef struct TestCaseItem { char *testName; char *suiteName; + char *description; + long requirements; + long timeout; + TestCaseInitFp testCaseInit; TestCaseFp testCase; TestCaseQuitFp testCaseQuit; @@ -213,14 +219,24 @@ LoadTestCases(TestSuiteReference *suites) item->testCase = testCase; item->testCaseQuit = testCaseQuit; + // copy suite name int length = strlen(suiteReference->name) + 1; item->suiteName = SDL_malloc(length); strcpy(item->suiteName, suiteReference->name); + // copy test name length = strlen(testReference->name) + 1; item->testName = SDL_malloc(length); strcpy(item->testName, testReference->name); + // copy test description + length = strlen(testReference->description) + 1; + item->description = SDL_malloc(length); + strcpy(item->testName, testReference->name); + + item->requirements = testReference->requirements; + item->timeout = testReference->timeout; + // prepend the list item->next = testCases; testCases = item; @@ -247,6 +263,7 @@ UnloadTestCases(TestCase *testCases) while(ref) { SDL_free(ref->testName); SDL_free(ref->suiteName); + SDL_free(ref->description); TestCase *temp = ref->next; SDL_free(ref); @@ -482,7 +499,7 @@ int ExecuteTest(TestCase *testItem) { int retVal = 1; if(execute_inproc) { - testItem->testCaseInit(); + testItem->testCaseInit(xml_enabled); testItem->testCase(0x0); @@ -490,7 +507,7 @@ ExecuteTest(TestCase *testItem) { } else { int childpid = fork(); if(childpid == 0) { - testItem->testCaseInit(); + testItem->testCaseInit(xml_enabled); testItem->testCase(0x0); @@ -517,6 +534,7 @@ printUsage() { printf("Options:\n"); printf(" --in-proc Executes tests in-process\n"); printf(" --show-tests Prints out all the executable tests\n"); + printf(" --xml Enables XML logger\n"); printf(" -t --test TEST Executes only tests with given name\n"); printf(" -ts --name-contains SUBSTR Executes only tests that have given\n"); printf(" substring in test name\n"); @@ -545,6 +563,9 @@ ParseOptions(int argc, char *argv[]) else if(SDL_strcmp(arg, "--show-tests") == 0) { only_print_tests = 1; } + else if(SDL_strcmp(arg, "--xml") == 0) { + xml_enabled = 1; + } else if(SDL_strcmp(arg, "--test") == 0 || SDL_strcmp(arg, "-t") == 0) { only_selected_test = 1; char *testName = NULL; @@ -625,6 +646,11 @@ main(int argc, char *argv[]) #else char *extension = "dylib"; #endif + if(xml_enabled) { + SetupXMLLogger(); + } else { + SetupPlainLogger(); + } const Uint32 startTicks = SDL_GetTicks(); @@ -637,29 +663,49 @@ main(int argc, char *argv[]) if(only_print_tests) { TestCase *testItem = NULL; for(testItem = testCases; testItem; testItem = testItem->next) { + //! \todo This should be handled by the logging system? printf("%s (in %s)\n", testItem->testName, testItem->suiteName); } return 0; } + RunStarted(LogGenericOutput, NULL, 0); + + char *currentSuiteName = NULL; + TestCase *testItem = NULL; for(testItem = testCases; testItem; testItem = testItem->next) { + if(currentSuiteName == NULL) { + currentSuiteName = testItem->suiteName; + SuiteStarted(currentSuiteName, 0); + } + + TestStarted(testItem->testName, testItem->suiteName, + testItem->description, 0); + int retVal = ExecuteTest(testItem); if(retVal) { failureCount++; if(retVal == 2) { - printf("%s (in %s): FAILED -> No asserts\n", testItem->testName, testItem->suiteName); + //printf("%s (in %s): FAILED -> No asserts\n", testItem->testName, testItem->suiteName); } else { - printf("%s (in %s): FAILED\n", testItem->testName, testItem->suiteName); + //printf("%s (in %s): FAILED\n", testItem->testName, testItem->suiteName); } } else { passCount++; - printf("%s (in %s): ok\n", testItem->testName, testItem->suiteName); + //printf("%s (in %s): ok\n", testItem->testName, testItem->suiteName); } - printf("\n"); + TestEnded(testItem->testName, testItem->suiteName, retVal, 0, 0); + + if(strncmp(currentSuiteName, testItem->suiteName, 100) != 0) { + SuiteEnded(0, 0, 0, 0.0f, 0); + + currentSuiteName = testItem->suiteName; + SuiteStarted(currentSuiteName, 0); + } } UnloadTestCases(testCases); @@ -667,10 +713,14 @@ main(int argc, char *argv[]) const Uint32 endTicks = SDL_GetTicks(); + RunEnded(passCount + failureCount, 1 /*add suiteCount */, + passCount, failureCount, 0, 0); + /* printf("Ran %d tests in %0.5f seconds.\n", (passCount + failureCount), (endTicks-startTicks)/1000.0f); printf("%d tests passed\n", passCount); printf("%d tests failed\n", failureCount); + */ return 0; } diff --git a/test/test-automation/xml.c b/test/test-automation/xml.c index 59f45b942..dc821e942 100644 --- a/test/test-automation/xml.c +++ b/test/test-automation/xml.c @@ -55,7 +55,7 @@ AddOpenTag(const char *tag) const int tagSize = SDL_strlen(tag) + 1; openTag->tag = SDL_malloc(tagSize); - strncpy(openTag->tag, tag, tagSize); + strncpy((char *)openTag->tag, (char *)tag, tagSize); openTag->next = openTags; @@ -86,7 +86,7 @@ RemoveOpenTag(const char *tag) // It prevents opening and ending tag mismatch if(SDL_strcmp(tempTag, tag) == 0) { TagList *openTag = openTags; - SDL_free(openTag->tag); + SDL_free((char *)openTag->tag); /* int counter = 0; @@ -175,7 +175,7 @@ const char *EscapeString(const char *string) { * \return Lower-case version of the given string */ char * -ToLowerCase(char *string) +ToLowerCase(const char *string) { const int size = SDL_strlen(string); char *ret = SDL_malloc(size + 1); diff --git a/test/test-automation/xml_logger.c b/test/test-automation/xml_logger.c index 52ed9cb2f..6e4ba7269 100644 --- a/test/test-automation/xml_logger.c +++ b/test/test-automation/xml_logger.c @@ -18,6 +18,9 @@ 3. This notice may not be removed or altered from any source distribution. */ +#include +#include + #include #include "xml.h" @@ -32,19 +35,19 @@ XMLRunStarted(LogOutputFp outputFn, const char *runnerParameters, time_t eventTi { logger = outputFn; - char *output = XMLOpenDocument("teSTtlog"); + char *output = XMLOpenDocument("testlog"); logger(output); SDL_free(output); - output = XMLOpenElement("paRameters"); + output = XMLOpenElement("parameters"); logger(output); SDL_free(output); - output = XMLAddContent(runnerParameters); + output = XMLAddContent("Add: runner parameter"); logger(output); SDL_free(output); - output = XMLCloseElement("Parameters"); + output = XMLCloseElement("parameters"); logger(output); SDL_free(output); } @@ -53,7 +56,7 @@ void XMLRunEnded(int testCount, int suiteCount, int testPassCount, int testFailCount, time_t endTime, time_t totalRuntime) { - char *output = XMLCloseDocument("testlOg"); + char *output = XMLCloseDocument("testlog"); logger(output); SDL_free(output); } @@ -65,12 +68,12 @@ XMLSuiteStarted(const char *suiteName, time_t eventTime) logger(output); SDL_free(output); - output = XMLOpenElement("EVENTTime"); + output = XMLOpenElement("eventtime"); logger(output); SDL_free(output); //XMLAddContent(evenTime); - output = XMLCloseElement("eventTIME"); + output = XMLCloseElement("eventtime"); logger(output); SDL_free(output); } @@ -130,9 +133,31 @@ XMLTestStarted(const char *testName, const char *suiteName, const char *testDesc void XMLTestEnded(const char *testName, const char *suiteName, - int testResult, int numAsserts, time_t endTime, time_t totalRuntime) + int testResult, time_t endTime, time_t totalRuntime) { - char *output = XMLCloseElement("test"); + char *output = XMLOpenElement("result"); + logger(output); + SDL_free(output); + + if(testResult) { + if(testResult == 2) { + output = XMLAddContent("failed -> no assert"); + } else { + output = XMLAddContent("failed"); + } + logger(output); + SDL_free(output); + } else { + output = XMLAddContent("passed"); + logger(output); + SDL_free(output); + + } + output = XMLCloseElement("result"); + logger(output); + SDL_free(output); + + output = XMLCloseElement("test"); logger(output); SDL_free(output); } @@ -153,7 +178,7 @@ XMLAssert(const char *assertName, int assertResult, const char *assertMessage, logger(output); SDL_free(output); - output = XMLOpenElement("result"); + output = XMLCloseElement("result"); logger(output); SDL_free(output); @@ -162,6 +187,57 @@ XMLAssert(const char *assertName, int assertResult, const char *assertMessage, SDL_free(output); } +void +XMLAssertSummary(int numAsserts, int numAssertsFailed, int numAssertsPass) +{ + char *output = XMLOpenElement("assertSummary"); + logger(output); + SDL_free(output); + + output = XMLOpenElement("assertCount"); + logger(output); + SDL_free(output); + + //XMLAddContent() \todo add string conversion + + output = XMLCloseElement("assertCount"); + logger(output); + SDL_free(output); + + output = XMLOpenElement("assertsPassed"); + logger(output); + SDL_free(output); + + const int bufferSize = sizeof(int) * 8 + 1; + //char buffer[bufferSize]; + char *buffer = SDL_malloc(bufferSize); + memset(buffer, 'a', bufferSize); + + //SDL_vsnprintf(buffer, bufferSize, "%d", numAssertsPass); + snprintf(buffer, sizeof(buffer), "%d", numAssertsPass); + buffer[3] = 'a'; + //printf("DEBUG |Ê%s == %d of size %d", buffer, numAssertsPass, bufferSize); + XMLAddContent(buffer); + + output = XMLCloseElement("assertsPassed"); + logger(output); + SDL_free(output); + + output = XMLOpenElement("assertsFailed"); + logger(output); + SDL_free(output); + + //XMLAddContent() \todo add string conversion + + output = XMLCloseElement("assertsFailed"); + logger(output); + SDL_free(output); + + output = XMLCloseElement("assertSummary"); + logger(output); + SDL_free(output); +} + void XMLLog(const char *logMessage, time_t eventTime) { diff --git a/test/test-automation/xml_logger.h b/test/test-automation/xml_logger.h index bd9c5d135..42ec4cfc8 100644 --- a/test/test-automation/xml_logger.h +++ b/test/test-automation/xml_logger.h @@ -16,11 +16,14 @@ void XMLSuiteEnded(int testsPassed, int testsFailed, int testsSkipped, void XMLTestStarted(const char *testName, const char *suiteName, const char *testDescription, time_t startTime); void XMLTestEnded(const char *testName, const char *suiteName, - int testResult, int numAsserts, time_t endTime, time_t totalRuntime); + int testResult, time_t endTime, time_t totalRuntime); void XMLAssert(const char *assertName, int assertResult, const char *assertMessage, time_t eventTime); +void XMLAssertSummary(int numAsserts, int numAssertsFailed, int numAssertsPass); + + void XMLLog(const char *logMessage, time_t eventTime); #endif