diff --git a/test/test-automation/SDL_test.c b/test/test-automation/SDL_test.c
index e9811199f..0564d3017 100644
--- a/test/test-automation/SDL_test.c
+++ b/test/test-automation/SDL_test.c
@@ -38,7 +38,7 @@ int _testAssertsPassed;
void
_InitTestEnvironment()
{
- _testReturnValue = 0;
+ _testReturnValue = TEST_RESULT_PASS;
_testAssertsFailed = 0;
_testAssertsPassed = 0;
}
@@ -50,7 +50,7 @@ _QuitTestEnvironment()
_testAssertsFailed, _testAssertsPassed, time(0));
if(_testAssertsFailed == 0 && _testAssertsPassed == 0) {
- _testReturnValue = 2;
+ _testReturnValue = TEST_RESULT_NO_ASSERT;
}
return _testReturnValue;
@@ -75,7 +75,7 @@ AssertEquals(int expected, int actual, char *message, ...)
if(expected != expected) {
AssertWithValues("AssertEquals", 0, buf, actual, expected, time(0));
- _testReturnValue = 1;
+ _testReturnValue = TEST_RESULT_FAILURE;
_testAssertsFailed++;
} else {
AssertWithValues("AssertEquals", 1, buf,
@@ -97,7 +97,7 @@ AssertTrue(int condition, char *message, ...)
if (!condition) {
Assert("AssertTrue", 0, buf, time(0));
- _testReturnValue = 1;
+ _testReturnValue = TEST_RESULT_FAILURE;
_testAssertsFailed++;
} else {
Assert("AssertTrue", 1, buf, time(0));
@@ -133,7 +133,7 @@ AssertFail(char *message, ...)
Assert("AssertFail", 0, buf, time(0));
- _testReturnValue = 1;
+ _testReturnValue = TEST_RESULT_FAILURE;
_testAssertsFailed++;
}
diff --git a/test/test-automation/SDL_test.h b/test/test-automation/SDL_test.h
index 041670f79..e56c2a8d9 100644
--- a/test/test-automation/SDL_test.h
+++ b/test/test-automation/SDL_test.h
@@ -39,6 +39,13 @@ extern AssertFp testAssert;
#define TEST_ENABLED 1
#define TEST_DISABLED 0
+#define TEST_RESULT_PASS 0
+#define TEST_RESULT_FAILURE 1
+#define TEST_RESULT_NO_ASSERT 2
+#define TEST_RESULT_SKIPPED 3
+#define TEST_RESULT_KILLED 4
+#define TEST_RESULT_SETUP_FAILURE 5
+
/*!
* Holds information about a test case
*/
diff --git a/test/test-automation/plain_logger.c b/test/test-automation/plain_logger.c
index 5cf4d2667..44997141b 100644
--- a/test/test-automation/plain_logger.c
+++ b/test/test-automation/plain_logger.c
@@ -8,6 +8,7 @@
#include "logger_helpers.h"
#include "plain_logger.h"
+#include "SDL_test.h"
static int indentLevel;
@@ -88,17 +89,25 @@ void
PlainTestEnded(const char *testName, const char *suiteName,
int testResult, time_t endTime, double totalRuntime)
{
- if(testResult) {
- if(testResult == 2) {
- Output(--indentLevel, "%s: failed -> no assert", testName);
- }
- else if(testResult == 3) {
- Output(--indentLevel, "%s: skipped", testName);
- } else {
+ switch(testResult) {
+ case TEST_RESULT_PASS:
+ Output(--indentLevel, "%s: ok", testName);
+ break;
+ case TEST_RESULT_FAILURE:
Output(--indentLevel, "%s: failed", testName);
- }
- } else {
- Output(--indentLevel, "%s: ok", testName);
+ break;
+ case TEST_RESULT_NO_ASSERT:
+ Output(--indentLevel, "%s: failed -> no assert", testName);
+ break;
+ case TEST_RESULT_SKIPPED:
+ Output(--indentLevel, "%s: skipped", testName);
+ break;
+ case TEST_RESULT_KILLED:
+ Output(--indentLevel, "%s: killed, exceeded timeout", testName);
+ break;
+ case TEST_RESULT_SETUP_FAILURE:
+ Output(--indentLevel, "%s: killed, setup failure", testName);
+ break;
}
}
diff --git a/test/test-automation/runner.c b/test/test-automation/runner.c
index a989e01c7..72d98504b 100644
--- a/test/test-automation/runner.c
+++ b/test/test-automation/runner.c
@@ -600,7 +600,7 @@ RunTest(TestCase *testItem) {
int cntFailedAsserts = testItem->countFailedAsserts();
if(cntFailedAsserts != 0) {
- return 3;
+ return TEST_RESULT_SETUP_FAILURE;
}
testItem->testCase(0x0);
@@ -612,6 +612,22 @@ RunTest(TestCase *testItem) {
return testItem->quitTestEnvironment();
}
+/*!
+ * Kills test that hungs. Test hungs when its execution
+ * takes longer than timeout specified for it.
+ *
+ * When test will be killed SIG_ALRM will be triggered and
+ * it'll call this function which kills the test process.
+ *
+ * Note: if runner is executed with --in-proc then hung tests
+ * can't be killed
+ *
+ * \param signum
+ */
+void KillHungTest(int signum) {
+ exit(TEST_RESULT_KILLED);
+}
+
/*!
* Executes a test case. Loads the test, executes it and
* returns the tests return value to the caller.
@@ -621,13 +637,18 @@ RunTest(TestCase *testItem) {
*/
int
ExecuteTest(TestCase *testItem) {
- int retVal = 1;
+ int retVal = -1;
if(execute_inproc) {
retVal = RunTest(testItem);
} else {
int childpid = fork();
if(childpid == 0) {
+ if(testItem->timeout > 0) {
+ signal(SIGALRM, KillHungTest);
+ alarm((unsigned int) testItem->timeout);
+ }
+
exit(RunTest(testItem));
} else {
int stat_lock = -1;
@@ -641,6 +662,7 @@ ExecuteTest(TestCase *testItem) {
}
+
/*!
* If using out-of-proc execution of tests. This function
* will handle the return value of the child process
diff --git a/test/test-automation/style.xsl b/test/test-automation/style.xsl
index 367fa4d7f..69515f4f7 100644
--- a/test/test-automation/style.xsl
+++ b/test/test-automation/style.xsl
@@ -209,6 +209,11 @@ div, h1 {
+
+
+ ()
+
+
(Total runtime: seconds)
Description:
[Show Assert Summary]
diff --git a/test/test-automation/testdummy/testdummy.c b/test/test-automation/testdummy/testdummy.c
index 26779e4d1..dd8e864a6 100644
--- a/test/test-automation/testdummy/testdummy.c
+++ b/test/test-automation/testdummy/testdummy.c
@@ -32,7 +32,7 @@
/* Test case references */
static const TestCaseReference test1 =
- (TestCaseReference){ "dummycase1", "description", TEST_ENABLED, 0, 0};
+ (TestCaseReference){ "dummycase1", "description", TEST_ENABLED, 0, 4};
static const TestCaseReference test2 =
(TestCaseReference){ "dummycase2", "description", TEST_ENABLED, 0, 0};
@@ -89,6 +89,7 @@ void
dummycase1(void *arg)
{
//AssertEquals(5, 5, "Assert message");
+ while(1);
}
void
diff --git a/test/test-automation/xml_logger.c b/test/test-automation/xml_logger.c
index 3481b588c..03aae4a11 100644
--- a/test/test-automation/xml_logger.c
+++ b/test/test-automation/xml_logger.c
@@ -26,6 +26,7 @@
#include "xml.h"
#include "logger_helpers.h"
+#include "SDL_test.h"
#include "xml_logger.h"
@@ -49,6 +50,7 @@ const char *testElementName = "test";
const char *nameElementName = "name";
const char *descriptionElementName = "description";
const char *resultElementName = "result";
+const char *resultDescriptionElementName = "resultDescription";
const char *assertElementName = "assert";
const char *messageElementName = "message";
const char *timeElementName = "time";
@@ -347,27 +349,61 @@ void
XMLTestEnded(const char *testName, const char *suiteName,
int testResult, time_t endTime, double totalRuntime)
{
+ // Log test result
char *output = XMLOpenElement(resultElementName);
XMLOutputter(indentLevel++, NO, output);
- if(testResult) {
- if(testResult == 2) {
- output = XMLAddContent("failed. No assert");
- }
- else if(testResult == 3) {
- output = XMLAddContent("skipped");
- } else {
+ switch(testResult) {
+ case TEST_RESULT_PASS:
+ output = XMLAddContent("passed");
+ break;
+ case TEST_RESULT_FAILURE:
output = XMLAddContent("failed");
- }
- XMLOutputter(indentLevel, NO, output);
- } else {
- output = XMLAddContent("passed");
- XMLOutputter(indentLevel, NO, output);
+ break;
+ case TEST_RESULT_NO_ASSERT:
+ output = XMLAddContent("failed");
+ break;
+ case TEST_RESULT_SKIPPED:
+ output = XMLAddContent("skipped");
+ break;
+ case TEST_RESULT_KILLED:
+ output = XMLAddContent("failed");
+ break;
+ case TEST_RESULT_SETUP_FAILURE:
+ output = XMLAddContent("failed");
+ break;
}
+ XMLOutputter(indentLevel, NO, output);
output = XMLCloseElement(resultElementName);
XMLOutputter(--indentLevel, YES, output);
+ // Log description of test result. Why the test failed,
+ // if there's some specific reason
+ output = XMLOpenElement(resultDescriptionElementName);
+ XMLOutputter(indentLevel++, NO, output);
+
+ switch(testResult) {
+ case TEST_RESULT_PASS:
+ case TEST_RESULT_FAILURE:
+ case TEST_RESULT_SKIPPED:
+ output = XMLAddContent("");
+ break;
+ case TEST_RESULT_NO_ASSERT:
+ output = XMLAddContent("No assert");
+ break;
+ case TEST_RESULT_KILLED:
+ output = XMLAddContent("Timeout exceeded");
+ break;
+ case TEST_RESULT_SETUP_FAILURE:
+ output = XMLAddContent("Setup failure, couldn't be executed");
+ break;
+ }
+ XMLOutputter(indentLevel, NO, output);
+
+ output = XMLCloseElement(resultDescriptionElementName);
+ XMLOutputter(--indentLevel, YES, output);
+
// log total runtime
output = XMLOpenElement(endTimeElementName);
XMLOutputter(indentLevel++, NO, output);