pandory500/SDL-mirror
2
0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

Harness kills hung tests (won't work with --in-proc option).

Browse source 
Added result description to test logs (tells why test failed,
such as exceeding its timeout).
This commit is contained in:
Markus Kauppila 2011-07-13 19:51:25 +03:00
parent 4790fd63e3
commit 7de8abe0c4
7 changed files with 110 additions and 30 deletions
Download patch file Download diff file Expand all files Collapse all files

10
test/test-automation/SDL_test.c
View file

@ -38,7 +38,7 @@ int _testAssertsPassed;
void
_InitTestEnvironment()
{
_testReturnValue = 0;
_testReturnValue = TEST_RESULT_PASS;
_testAssertsFailed = 0;
_testAssertsPassed = 0;
}
@ -50,7 +50,7 @@ _QuitTestEnvironment()
_testAssertsFailed, _testAssertsPassed, time(0));
if(_testAssertsFailed == 0 && _testAssertsPassed == 0) {
_testReturnValue = 2;
_testReturnValue = TEST_RESULT_NO_ASSERT;
}
return _testReturnValue;
@ -75,7 +75,7 @@ AssertEquals(int expected, int actual, char *message, ...)
if(expected != expected) {
AssertWithValues("AssertEquals", 0, buf, actual, expected, time(0));
_testReturnValue = 1;
_testReturnValue = TEST_RESULT_FAILURE;
_testAssertsFailed++;
} else {
AssertWithValues("AssertEquals", 1, buf,
@ -97,7 +97,7 @@ AssertTrue(int condition, char *message, ...)
if (!condition) {
Assert("AssertTrue", 0, buf, time(0));
_testReturnValue = 1;
_testReturnValue = TEST_RESULT_FAILURE;
_testAssertsFailed++;
} else {
Assert("AssertTrue", 1, buf, time(0));
@ -133,7 +133,7 @@ AssertFail(char *message, ...)
Assert("AssertFail", 0, buf, time(0));
_testReturnValue = 1;
_testReturnValue = TEST_RESULT_FAILURE;
_testAssertsFailed++;
}

7
test/test-automation/SDL_test.h
View file

@ -39,6 +39,13 @@ extern AssertFp testAssert;
#define TEST_ENABLED 1
#define TEST_DISABLED 0
#define TEST_RESULT_PASS 0
#define TEST_RESULT_FAILURE 1
#define TEST_RESULT_NO_ASSERT 2
#define TEST_RESULT_SKIPPED 3
#define TEST_RESULT_KILLED 4
#define TEST_RESULT_SETUP_FAILURE 5
/*!
* Holds information about a test case
*/

29
test/test-automation/plain_logger.c
View file

@ -8,6 +8,7 @@
#include "logger_helpers.h"
#include "plain_logger.h"
#include "SDL_test.h"
static int indentLevel;
@ -88,17 +89,25 @@ void
PlainTestEnded(const char *testName, const char *suiteName,
int testResult, time_t endTime, double totalRuntime)
{
if(testResult) {
if(testResult == 2) {
Output(--indentLevel, "%s: failed -> no assert", testName);
}
else if(testResult == 3) {
Output(--indentLevel, "%s: skipped", testName);
} else {
switch(testResult) {
case TEST_RESULT_PASS:
Output(--indentLevel, "%s: ok", testName);
break;
case TEST_RESULT_FAILURE:
Output(--indentLevel, "%s: failed", testName);
}
} else {
Output(--indentLevel, "%s: ok", testName);
break;
case TEST_RESULT_NO_ASSERT:
Output(--indentLevel, "%s: failed -> no assert", testName);
break;
case TEST_RESULT_SKIPPED:
Output(--indentLevel, "%s: skipped", testName);
break;
case TEST_RESULT_KILLED:
Output(--indentLevel, "%s: killed, exceeded timeout", testName);
break;
case TEST_RESULT_SETUP_FAILURE:
Output(--indentLevel, "%s: killed, setup failure", testName);
break;
}
}

26
test/test-automation/runner.c
View file

@ -600,7 +600,7 @@ RunTest(TestCase *testItem) {
int cntFailedAsserts = testItem->countFailedAsserts();
if(cntFailedAsserts != 0) {
return 3;
return TEST_RESULT_SETUP_FAILURE;
}
testItem->testCase(0x0);
@ -612,6 +612,22 @@ RunTest(TestCase *testItem) {
return testItem->quitTestEnvironment();
}
/*!
* Kills test that hungs. Test hungs when its execution
* takes longer than timeout specified for it.
*
* When test will be killed SIG_ALRM will be triggered and
* it'll call this function which kills the test process.
*
* Note: if runner is executed with --in-proc then hung tests
* can't be killed
*
* \param signum
*/
void KillHungTest(int signum) {
exit(TEST_RESULT_KILLED);
}
/*!
* Executes a test case. Loads the test, executes it and
* returns the tests return value to the caller.
@ -621,13 +637,18 @@ RunTest(TestCase *testItem) {
*/
int
ExecuteTest(TestCase *testItem) {
int retVal = 1;
int retVal = -1;
if(execute_inproc) {
retVal = RunTest(testItem);
} else {
int childpid = fork();
if(childpid == 0) {
if(testItem->timeout > 0) {
signal(SIGALRM, KillHungTest);
alarm((unsigned int) testItem->timeout);
}
exit(RunTest(testItem));
} else {
int stat_lock = -1;
@ -641,6 +662,7 @@ ExecuteTest(TestCase *testItem) {
}
/*!
* If using out-of-proc execution of tests. This function
* will handle the return value of the child process

5
test/test-automation/style.xsl
View file

@ -209,6 +209,11 @@ div, h1 {
<xsl:value-of select="result"/>
</xsl:attribute><xsl:value-of select="result"/>
</span>
<xsl:if test="resultDescription != ''">
<span xml:space="preserve">
(<xsl:value-of select="resultDescription"/>)
</span>
</xsl:if>
(Total runtime: <xsl:value-of select="totalRuntime"/> seconds)<br/>
Description: <span class="description"> <xsl:value-of select="description"/> </span><br/>
<span class="switch show-asserts" uid="{generate-id(assertSummary)}">[Show Assert Summary]</span><br/>

3
test/test-automation/testdummy/testdummy.c
View file

@ -32,7 +32,7 @@
/* Test case references */
static const TestCaseReference test1 =
(TestCaseReference){ "dummycase1", "description", TEST_ENABLED, 0, 0};
(TestCaseReference){ "dummycase1", "description", TEST_ENABLED, 0, 4};
static const TestCaseReference test2 =
(TestCaseReference){ "dummycase2", "description", TEST_ENABLED, 0, 0};
@ -89,6 +89,7 @@ void
dummycase1(void *arg)
{
//AssertEquals(5, 5, "Assert message");
while(1);
}
void

60
test/test-automation/xml_logger.c
View file

@ -26,6 +26,7 @@
#include "xml.h"
#include "logger_helpers.h"
#include "SDL_test.h"
#include "xml_logger.h"
@ -49,6 +50,7 @@ const char *testElementName = "test";
const char *nameElementName = "name";
const char *descriptionElementName = "description";
const char *resultElementName = "result";
const char *resultDescriptionElementName = "resultDescription";
const char *assertElementName = "assert";
const char *messageElementName = "message";
const char *timeElementName = "time";
@ -347,27 +349,61 @@ void
XMLTestEnded(const char *testName, const char *suiteName,
int testResult, time_t endTime, double totalRuntime)
{
// Log test result
char *output = XMLOpenElement(resultElementName);
XMLOutputter(indentLevel++, NO, output);
if(testResult) {
if(testResult == 2) {
output = XMLAddContent("failed. No assert");
}
else if(testResult == 3) {
output = XMLAddContent("skipped");
} else {
switch(testResult) {
case TEST_RESULT_PASS:
output = XMLAddContent("passed");
break;
case TEST_RESULT_FAILURE:
output = XMLAddContent("failed");
}
XMLOutputter(indentLevel, NO, output);
} else {
output = XMLAddContent("passed");
XMLOutputter(indentLevel, NO, output);
break;
case TEST_RESULT_NO_ASSERT:
output = XMLAddContent("failed");
break;
case TEST_RESULT_SKIPPED:
output = XMLAddContent("skipped");
break;
case TEST_RESULT_KILLED:
output = XMLAddContent("failed");
break;
case TEST_RESULT_SETUP_FAILURE:
output = XMLAddContent("failed");
break;
}
XMLOutputter(indentLevel, NO, output);
output = XMLCloseElement(resultElementName);
XMLOutputter(--indentLevel, YES, output);
// Log description of test result. Why the test failed,
// if there's some specific reason
output = XMLOpenElement(resultDescriptionElementName);
XMLOutputter(indentLevel++, NO, output);
switch(testResult) {
case TEST_RESULT_PASS:
case TEST_RESULT_FAILURE:
case TEST_RESULT_SKIPPED:
output = XMLAddContent("");
break;
case TEST_RESULT_NO_ASSERT:
output = XMLAddContent("No assert");
break;
case TEST_RESULT_KILLED:
output = XMLAddContent("Timeout exceeded");
break;
case TEST_RESULT_SETUP_FAILURE:
output = XMLAddContent("Setup failure, couldn't be executed");
break;
}
XMLOutputter(indentLevel, NO, output);
output = XMLCloseElement(resultDescriptionElementName);
XMLOutputter(--indentLevel, YES, output);
// log total runtime
output = XMLOpenElement(endTimeElementName);
XMLOutputter(indentLevel++, NO, output);