diff --git a/test/test-automation/logger.h b/test/test-automation/logger.h index e98fc1666..b75fb697a 100644 --- a/test/test-automation/logger.h +++ b/test/test-automation/logger.h @@ -23,14 +23,11 @@ #include -// Function pointer to function which handles to output -typedef int (*LogOutputFp)(const char *, ...); - /*! * Generic logger interface * */ -typedef void (*RunStartedFp)(LogOutputFp outputFn, const char *runnerParameters, time_t eventTime); +typedef void (*RunStartedFp)(int parameterCount, char *runnerParameters[], time_t eventTime); typedef void (*RunEndedFp)(int testCount, int suiteCount, int testPassCount, int testFailCount, time_t endTime, time_t totalRuntime); @@ -54,7 +51,6 @@ typedef void (*AssertSummaryFp)(int numAsserts, int numAssertsFailed, int numAss typedef void (*LogFp)(const char *logMessage, time_t eventTime); -int Output(const char *message, ...); extern RunStartedFp RunStarted; extern RunEndedFp RunEnded; diff --git a/test/test-automation/plain_logger.c b/test/test-automation/plain_logger.c index fa66762f5..9a8c6851c 100644 --- a/test/test-automation/plain_logger.c +++ b/test/test-automation/plain_logger.c @@ -27,7 +27,7 @@ Output(const char *message, ...) } void -PlainRunStarted(const char *runnerParameters, time_t eventTime) +PlainRunStarted(int parameterCount, char *runnerParameters[], time_t eventTime) { Output("Test run started"); Output("Given command line options: %s", "add options"); diff --git a/test/test-automation/plain_logger.h b/test/test-automation/plain_logger.h index a4bf422e7..3ce12d38d 100644 --- a/test/test-automation/plain_logger.h +++ b/test/test-automation/plain_logger.h @@ -3,7 +3,7 @@ #include "logger.h" -void PlainRunStarted(const char *runnerParameters, time_t eventTime); +void PlainRunStarted(int parameterCount, char *runnerParameters[], time_t eventTime); void PlainRunEnded(int testCount, int suiteCount, int testPassCount, int testFailCount, time_t endTime, time_t totalRuntime); diff --git a/test/test-automation/runner.c b/test/test-automation/runner.c index bf95091ee..82500784e 100644 --- a/test/test-automation/runner.c +++ b/test/test-automation/runner.c @@ -220,19 +220,19 @@ LoadTestCases(TestSuiteReference *suites) item->testCaseQuit = testCaseQuit; // copy suite name - int length = strlen(suiteReference->name) + 1; + int length = SDL_strlen(suiteReference->name) + 1; item->suiteName = SDL_malloc(length); - strcpy(item->suiteName, suiteReference->name); + strncpy(item->suiteName, suiteReference->name, length); // copy test name - length = strlen(testReference->name) + 1; + length = SDL_strlen(testReference->name) + 1; item->testName = SDL_malloc(length); - strcpy(item->testName, testReference->name); + strncpy(item->testName, testReference->name, length); // copy test description - length = strlen(testReference->description) + 1; + length = SDL_strlen(testReference->description) + 1; item->description = SDL_malloc(length); - strcpy(item->testName, testReference->name); + strncpy(item->description, testReference->description, length); item->requirements = testReference->requirements; item->timeout = testReference->timeout; @@ -637,7 +637,8 @@ main(int argc, char *argv[]) // print: Testing against SDL version fuu (rev: bar) if verbose == true - int failureCount = 0, passCount = 0; + int totalTestfailureCount = 0, totalTestPassCount = 0; + int testFailureCount = 0, testPassCount = 0, testSkipCount = 0; char *testSuiteName = NULL; int suiteCounter = 0; @@ -670,7 +671,7 @@ main(int argc, char *argv[]) return 0; } - RunStarted(Output, NULL, 0); + RunStarted(argc, argv, 0); char *currentSuiteName = NULL; @@ -679,50 +680,50 @@ main(int argc, char *argv[]) if(currentSuiteName == NULL) { currentSuiteName = testItem->suiteName; SuiteStarted(currentSuiteName, 0); + + testFailureCount = testPassCount = 0; + + suiteCounter++; + } + else if(strncmp(currentSuiteName, testItem->suiteName, NAME_BUFFER_SIZE) != 0) { + SuiteEnded(testPassCount, testFailureCount, testSkipCount, 0.0f, 0); + + currentSuiteName = testItem->suiteName; + SuiteStarted(currentSuiteName, 0); + + testFailureCount = testPassCount = 0; + + suiteCounter++; } TestStarted(testItem->testName, testItem->suiteName, testItem->description, 0); int retVal = ExecuteTest(testItem); - if(retVal) { - failureCount++; - if(retVal == 2) { - //printf("%s (in %s): FAILED -> No asserts\n", testItem->testName, testItem->suiteName); - } else { - //printf("%s (in %s): FAILED\n", testItem->testName, testItem->suiteName); - } + totalTestfailureCount++; + testFailureCount++; } else { - passCount++; - //printf("%s (in %s): ok\n", testItem->testName, testItem->suiteName); + totalTestPassCount++; + testPassCount++; } TestEnded(testItem->testName, testItem->suiteName, retVal, 0, 0); - - if(strncmp(currentSuiteName, testItem->suiteName, 100) != 0) { - SuiteEnded(0, 0, 0, 0.0f, 0); - - currentSuiteName = testItem->suiteName; - SuiteStarted(currentSuiteName, 0); - } } - SuiteEnded(0, 0, 0, 0.0f, 0); + if(currentSuiteName) { + // \todo if no test are run, this will case incorrect nesting with + // xml output + SuiteEnded(testPassCount, testFailureCount, testSkipCount, 0.0f, 0); + } UnloadTestCases(testCases); UnloadTestSuites(suites); const Uint32 endTicks = SDL_GetTicks(); - RunEnded(passCount + failureCount, 1 /*add suiteCount */, - passCount, failureCount, 0, 0); - /* - printf("Ran %d tests in %0.5f seconds.\n", (passCount + failureCount), (endTicks-startTicks)/1000.0f); - - printf("%d tests passed\n", passCount); - printf("%d tests failed\n", failureCount); - */ + RunEnded(totalTestPassCount + totalTestfailureCount, suiteCounter, + totalTestPassCount, totalTestfailureCount, 0, 0); return 0; } diff --git a/test/test-automation/xml.c b/test/test-automation/xml.c index 5ae886ecc..029da17a0 100644 --- a/test/test-automation/xml.c +++ b/test/test-automation/xml.c @@ -26,9 +26,6 @@ #include "xml.h" -/*! Points the function which handles the output */ -static LogOutputFp logger = 0; - /*! * Defines structure used for "counting" open XML-tags */ @@ -88,13 +85,6 @@ RemoveOpenTag(const char *tag) TagList *openTag = openTags; SDL_free((char *)openTag->tag); - /* - int counter = 0; - for(; counter < strlen(buffer); ++counter) { - buffer[counter] = tolower(buffer[counter]); - } - */ - openTags = openTags->next; SDL_free(openTag); } else { @@ -188,7 +178,6 @@ ToLowerCase(const char *string) strncpy(ret, string, size); ret[size] = '\0'; - // turn the tag to lower case for case-insensitive comparation int counter = 0; for(; counter < size; ++counter) { ret[counter] = tolower(ret[counter]); @@ -283,6 +272,9 @@ XMLCloseElement(const char *tag) char *ret = SDL_malloc(bufferSize); memset(ret, 0, bufferSize); + // \todo check that element we're trying is actually open, + // otherwise it'll case nesting problems + // Close the open tags with proper nesting. Closes tags until it finds // the given tag which is the last tag that will be closed TagList *openTag = openTags; diff --git a/test/test-automation/xml_logger.c b/test/test-automation/xml_logger.c index dd7d99ec3..fbca5f9dd 100644 --- a/test/test-automation/xml_logger.c +++ b/test/test-automation/xml_logger.c @@ -60,10 +60,10 @@ static int prevEOL = YES; * \param the XML element itself * */ -void XMLOutputter(const int il, int EOL, const char *message) { +void XMLOutputter(const int currentIdentLevel, int EOL, const char *message) { int ident = 0; - for( ; ident < il && prevEOL; ++ident) { - printf(" "); + for( ; ident < currentIdentLevel && prevEOL; ++ident) { + printf("\t"); } prevEOL = EOL; @@ -73,24 +73,51 @@ void XMLOutputter(const int il, int EOL, const char *message) { } else { printf("%s", message); } + + fflush(stdout); } void -XMLRunStarted(const char *runnerParameters, time_t eventTime) +XMLRunStarted(int parameterCount, char *runnerParameters[], time_t eventTime) { char *output = XMLOpenDocument("testlog"); XMLOutputter(indentLevel++, YES, output); SDL_free(output); output = XMLOpenElement("parameters"); + XMLOutputter(indentLevel++, YES, output); + SDL_free(output); + + int counter = 0; + for(counter = 0; counter < parameterCount; counter++) { + char *parameter = runnerParameters[counter]; + + output = XMLOpenElement("parameter"); + XMLOutputter(indentLevel++, NO, output); + SDL_free(output); + + output = XMLAddContent(parameter); + XMLOutputter(indentLevel, NO, output); + SDL_free(output); + + output = XMLCloseElement("parameter"); + XMLOutputter(--indentLevel, YES, output); + SDL_free(output); + } + + output = XMLCloseElement("parameters"); + XMLOutputter(--indentLevel, YES, output); + SDL_free(output); + + output = XMLOpenElement("eventTime"); XMLOutputter(indentLevel++, NO, output); SDL_free(output); - output = XMLAddContent("Add: runner parameter"); + output = XMLAddContent(IntToString(eventTime)); XMLOutputter(indentLevel, NO, output); SDL_free(output); - output = XMLCloseElement("parameters"); + output = XMLCloseElement("eventTime"); XMLOutputter(--indentLevel, YES, output); SDL_free(output); } @@ -99,7 +126,88 @@ void XMLRunEnded(int testCount, int suiteCount, int testPassCount, int testFailCount, time_t endTime, time_t totalRuntime) { - char *output = XMLCloseDocument("testlog"); + // log suite count + char *output = XMLOpenElement("numSuites"); + XMLOutputter(indentLevel++, NO, output); + SDL_free(output); + + output = XMLAddContent(IntToString(suiteCount)); + XMLOutputter(indentLevel, NO, output); + SDL_free(output); + + output = XMLCloseElement("numSuites"); + XMLOutputter(--indentLevel, YES, output); + SDL_free(output); + + // log test count + output = XMLOpenElement("numTest"); + XMLOutputter(indentLevel++, NO, output); + SDL_free(output); + + output = XMLAddContent(IntToString(testCount)); + XMLOutputter(indentLevel, NO, output); + SDL_free(output); + + output = XMLCloseElement("numTest"); + XMLOutputter(--indentLevel, YES, output); + SDL_free(output); + + // log passed test count + output = XMLOpenElement("numPassedTests"); + XMLOutputter(indentLevel++, NO, output); + SDL_free(output); + + output = XMLAddContent(IntToString(testPassCount)); + XMLOutputter(indentLevel, NO, output); + SDL_free(output); + + output = XMLCloseElement("numPassedTests"); + XMLOutputter(--indentLevel, YES, output); + SDL_free(output); + + // log failed test count + output = XMLOpenElement("numFailedTests"); + XMLOutputter(indentLevel++, NO, output); + SDL_free(output); + + output = XMLAddContent(IntToString(testFailCount)); + XMLOutputter(indentLevel, NO, output); + SDL_free(output); + + output = XMLCloseElement("numFailedTests"); + XMLOutputter(--indentLevel, YES, output); + SDL_free(output); + + + // log end timte + output = XMLOpenElement("endTime"); + XMLOutputter(indentLevel++, NO, output); + SDL_free(output); + + output = XMLAddContent(IntToString(endTime)); + XMLOutputter(indentLevel, NO, output); + SDL_free(output); + + output = XMLCloseElement("endTime"); + XMLOutputter(--indentLevel, YES, output); + SDL_free(output); + + // log total runtime + output = XMLOpenElement("totalRuntime"); + XMLOutputter(indentLevel++, NO, output); + SDL_free(output); + + output = XMLAddContent(IntToString(totalRuntime)); + XMLOutputter(indentLevel, NO, output); + SDL_free(output); + + output = XMLCloseElement("totalRuntime"); + XMLOutputter(--indentLevel, YES, output); + SDL_free(output); + + + + output = XMLCloseDocument("testlog"); XMLOutputter(--indentLevel, YES, output); SDL_free(output); } @@ -128,11 +236,75 @@ void XMLSuiteEnded(int testsPassed, int testsFailed, int testsSkipped, double endTime, time_t totalRuntime) { - char *output = XMLCloseElement("suite"); + // log tests passed + char *output = XMLOpenElement("testsPassed"); + XMLOutputter(indentLevel++, NO, output); + SDL_free(output); + + output = XMLAddContent(IntToString(testsPassed)); + XMLOutputter(indentLevel, NO, output); + SDL_free(output); + + output = XMLCloseElement("testsPassed"); XMLOutputter(--indentLevel, YES, output); SDL_free(output); - //! \todo endTime and totalRuntiem + // log tests failed + output = XMLOpenElement("testsFailed"); + XMLOutputter(indentLevel++, NO, output); + SDL_free(output); + + output = XMLAddContent(IntToString(testsFailed)); + XMLOutputter(indentLevel, NO, output); + SDL_free(output); + + output = XMLCloseElement("testsFailed"); + XMLOutputter(--indentLevel, YES, output); + SDL_free(output); + + // log tests skipped + output = XMLOpenElement("testsSkipped"); + XMLOutputter(indentLevel++, NO, output); + SDL_free(output); + + output = XMLAddContent(IntToString(testsSkipped)); + XMLOutputter(indentLevel, NO, output); + SDL_free(output); + + output = XMLCloseElement("testsSkipped"); + XMLOutputter(--indentLevel, YES, output); + SDL_free(output); + + // log tests skipped + output = XMLOpenElement("endTime"); + XMLOutputter(indentLevel++, NO, output); + SDL_free(output); + + output = XMLAddContent(IntToString(endTime)); + XMLOutputter(indentLevel, NO, output); + SDL_free(output); + + output = XMLCloseElement("endTime"); + XMLOutputter(--indentLevel, YES, output); + SDL_free(output); + + // log total runtime + output = XMLOpenElement("totalRuntime"); + XMLOutputter(indentLevel++, NO, output); + SDL_free(output); + + output = XMLAddContent(IntToString(totalRuntime)); + XMLOutputter(indentLevel, NO, output); + SDL_free(output); + + output = XMLCloseElement("totalRuntime"); + XMLOutputter(--indentLevel, YES, output); + SDL_free(output); + + + output = XMLCloseElement("suite"); + XMLOutputter(--indentLevel, YES, output); + SDL_free(output); } void @@ -156,7 +328,6 @@ XMLTestStarted(const char *testName, const char *suiteName, const char *testDesc XMLOutputter(--indentLevel, YES, output); SDL_free(output); - output = XMLOpenElement("description"); XMLOutputter(indentLevel++, NO, output); SDL_free(output); diff --git a/test/test-automation/xml_logger.h b/test/test-automation/xml_logger.h index ff59d79bd..8793fff19 100644 --- a/test/test-automation/xml_logger.h +++ b/test/test-automation/xml_logger.h @@ -3,7 +3,7 @@ #include "logger.h" -void XMLRunStarted(const char *runnerParameters, time_t eventTime); +void XMLRunStarted(int parameterCount, char *runnerParameters[], time_t eventTime); void XMLRunEnded(int testCount, int suiteCount, int testPassCount, int testFailCount, time_t endTime, time_t totalRuntime);