Add log, assert and harness (partial) to test lib

include/SDL_log.h

@@ -59,8 +59,9 @@ extern "C" {
  *  \brief The predefined log categories
  *
  *  By default the application category is enabled at the INFO level,
- *  the assert category is enabled at the WARN level, and all other
+ *  the assert category is enabled at the WARN level, test is enabled
- *  categories are enabled at the CRITICAL level.
+ *  at the VERBOSE level and all other categories are enabled at the 
+ *  CRITICAL level.
  */
 enum
 {
@@ -72,6 +73,7 @@ enum
     SDL_LOG_CATEGORY_VIDEO,
     SDL_LOG_CATEGORY_RENDER,
     SDL_LOG_CATEGORY_INPUT,
+    SDL_LOG_CATEGORY_TEST,
 
     /* Reserved for future SDL library use */
     SDL_LOG_CATEGORY_RESERVED1,

include/SDL_test.h

@@ -36,6 +36,9 @@
 #include "SDL_test_fuzzer.h"
 #include "SDL_test_crc32.h"
 #include "SDL_test_md5.h"
+#include "SDL_test_log.h"
+#include "SDL_test_assert.h"
+#include "SDL_test_harness.h"
 
 #include "begin_code.h"
 /* Set up for C function definitions, even when using C++ */

include/SDL_test_assert.h

@@ -0,0 +1,83 @@
+/*
+  Simple DirectMedia Layer
+  Copyright (C) 1997-2012 Sam Lantinga <slouken@libsdl.org>
+
+  This software is provided 'as-is', without any express or implied
+  warranty.  In no event will the authors be held liable for any damages
+  arising from the use of this software.
+
+  Permission is granted to anyone to use this software for any purpose,
+  including commercial applications, and to alter it and redistribute it
+  freely, subject to the following restrictions:
+
+  1. The origin of this software must not be misrepresented; you must not
+     claim that you wrote the original software. If you use this software
+     in a product, an acknowledgment in the product documentation would be
+     appreciated but is not required.
+  2. Altered source versions must be plainly marked as such, and must not be
+     misrepresented as being the original software.
+  3. This notice may not be removed or altered from any source distribution.
+*/
+
+/**
+ *  \file SDL_test_assert.h
+ *  
+ *  Include file for SDL test framework.
+ *
+ *  This code is a part of the SDL2_test library, not the main SDL library.
+ */
+
+/* 
+ *
+ * Assert API for test code and test cases
+ *
+ */
+
+#ifndef _SDL_test_assert_h
+#define _SDL_test_assert_h
+
+#include "begin_code.h"
+/* Set up for C function definitions, even when using C++ */
+#ifdef __cplusplus
+/* *INDENT-OFF* */
+extern "C" {
+/* *INDENT-ON* */
+#endif
+
+/**
+ * \brief Assert that logs and break execution flow on failures.
+ *
+ * \param assertCondition Evaluated condition or variable to assert; fail (==0) or pass (!=0).
+ * \param assertDescription Message to log with the assert describing it.
+ */
+void SDLTest_Assert(int assertCondition, char *assertDescription);
+
+/**
+ * \brief Assert for test cases that logs but does not break execution flow on failures.
+ *
+ * \param assertCondition Evaluated condition or variable to assert; fail (==0) or pass (!=0).
+ * \param assertDescription Message to log with the assert describing it.
+ */
+void SDLTest_AssertCheck(int assertCondition, char *assertDescription);
+
+/**
+ * \brief Resets the assert summary counters to zero.
+ */
+void SDLTest_ResetAssertSummary();
+
+/**
+ * \brief Logs summary of all assertions (total, pass, fail) since last reset as INFO or ERROR.
+ *
+ */
+void SDLTest_LogAssertSummary();
+
+#ifdef __cplusplus
+/* *INDENT-OFF* */
+}
+/* *INDENT-ON* */
+#endif
+#include "close_code.h"
+
+#endif /* _SDL_test_assert_h */
+
+/* vi: set ts=4 sw=4 expandtab: */

include/SDL_test_harness.h

@@ -0,0 +1,116 @@
+/*
+  Simple DirectMedia Layer
+  Copyright (C) 1997-2012 Sam Lantinga <slouken@libsdl.org>
+
+  This software is provided 'as-is', without any express or implied
+  warranty.  In no event will the authors be held liable for any damages
+  arising from the use of this software.
+
+  Permission is granted to anyone to use this software for any purpose,
+  including commercial applications, and to alter it and redistribute it
+  freely, subject to the following restrictions:
+
+  1. The origin of this software must not be misrepresented; you must not
+     claim that you wrote the original software. If you use this software
+     in a product, an acknowledgment in the product documentation would be
+     appreciated but is not required.
+  2. Altered source versions must be plainly marked as such, and must not be
+     misrepresented as being the original software.
+  3. This notice may not be removed or altered from any source distribution.
+*/
+
+/**
+ *  \file SDL_test_harness.h
+ *  
+ *  Include file for SDL test framework.
+ *
+ *  This code is a part of the SDL2_test library, not the main SDL library.
+ */
+
+/*
+  Defines types for test case definitions and the test execution harness API.
+  
+  Based on original GSOC code by Markus Kauppila <markus.kauppila@gmail.com>
+*/
+
+#ifndef _SDL_test_harness_h
+#define _SDL_test_harness_h
+
+#include "begin_code.h"
+/* Set up for C function definitions, even when using C++ */
+#ifdef __cplusplus
+/* *INDENT-OFF* */
+extern "C" {
+/* *INDENT-ON* */
+#endif
+
+
+//! Definitions for test case structures
+#define TEST_ENABLED  1
+#define TEST_DISABLED 0
+
+//! Definitions of assert results
+#define ASSERT_PASS		1
+#define ASSERT_FAIL		0
+
+//! Definition of all the possible test return values of the test case method
+#define TEST_ABORTED		-1
+#define TEST_COMPLETED		 0
+#define TEST_SKIPPED		 1
+
+//! Definition of all the possible test results for the harness
+#define TEST_RESULT_PASSED			0
+#define TEST_RESULT_FAILED			1
+#define TEST_RESULT_NO_ASSERT			2
+#define TEST_RESULT_SKIPPED			3
+#define TEST_RESULT_KILLED			4
+#define TEST_RESULT_SETUP_FAILURE		5
+
+//!< Function pointer to a test case setup function (run before every test)
+typedef void (*SDLTest_TestCaseSetUpFp)(void *arg);
+
+//!< Function pointer to a test case function
+typedef void (*SDLTest_TestCaseFp)(void *arg);
+
+//!< Function pointer to a test case teardown function (run after every test)
+typedef void  (*SDLTest_TestCaseTearDownFp)(void *arg);
+
+/**
+ * Holds information about a single test case.
+ */
+typedef struct SDLTest_TestCaseReference {
+	/*!< Func2Stress */
+	SDLTest_TestCaseFp testCase;
+	/*!< Short name (or function name) "Func2Stress" */
+	char *name;
+	/*!< Long name or full description "This test pushes func2() to the limit." */
+	char *description;
+	/*!< Set to TEST_ENABLED or TEST_DISABLED (test won't be run) */
+	int enabled;
+} SDLTest_TestCaseReference;
+
+/**
+ * Holds information about a test suite (multiple test cases).
+ */
+typedef struct TestSuiteReference {
+	/*!< "PlatformSuite" */
+	char *name;
+	/*!< The function that is run before each test. NULL skips. */
+	SDLTest_TestCaseSetUpFp testSetUp;
+	/*!< The test cases that are run as part of the suite. Last item should be NULL. */
+	const SDLTest_TestCaseReference **testCases;
+	/*!< The function that is run after each test. NULL skips. */
+	SDLTest_TestCaseTearDownFp testTearDown;
+} TestSuiteReference;
+
+/* Ends C function definitions when using C++ */
+#ifdef __cplusplus
+/* *INDENT-OFF* */
+}
+/* *INDENT-ON* */
+#endif
+#include "close_code.h"
+
+#endif /* _SDL_test_harness_h */
+
+/* vi: set ts=4 sw=4 expandtab: */

include/SDL_test_log.h

@@ -0,0 +1,71 @@
+/*
+  Simple DirectMedia Layer
+  Copyright (C) 1997-2012 Sam Lantinga <slouken@libsdl.org>
+
+  This software is provided 'as-is', without any express or implied
+  warranty.  In no event will the authors be held liable for any damages
+  arising from the use of this software.
+
+  Permission is granted to anyone to use this software for any purpose,
+  including commercial applications, and to alter it and redistribute it
+  freely, subject to the following restrictions:
+
+  1. The origin of this software must not be misrepresented; you must not
+     claim that you wrote the original software. If you use this software
+     in a product, an acknowledgment in the product documentation would be
+     appreciated but is not required.
+  2. Altered source versions must be plainly marked as such, and must not be
+     misrepresented as being the original software.
+  3. This notice may not be removed or altered from any source distribution.
+*/
+
+/**
+ *  \file SDL_test_log.h
+ *  
+ *  Include file for SDL test framework.
+ *
+ *  This code is a part of the SDL2_test library, not the main SDL library.
+ */
+
+/* 
+ *
+ *  Wrapper to log in the TEST category
+ *
+ */
+
+#ifndef _SDL_test_log_h
+#define _SDL_test_log_h
+
+#include "begin_code.h"
+/* Set up for C function definitions, even when using C++ */
+#ifdef __cplusplus
+/* *INDENT-OFF* */
+extern "C" {
+/* *INDENT-ON* */
+#endif
+
+/**
+ * \brief Prints given message with a timestamp in the TEST category and INFO priority.
+ *
+ * \param fmt Message to be logged
+ */
+void SDLTest_Log(char *fmt, ...);
+
+/**
+ * \brief Prints given message with a timestamp in the TEST category and the ERROR priority.
+ *
+ * \param fmt Message to be logged
+ */
+void SDLTest_LogError(char *fmt, ...);
+
+/* Ends C function definitions when using C++ */
+#ifdef __cplusplus
+/* *INDENT-OFF* */
+}
+/* *INDENT-ON* */
+#endif
+#include "close_code.h"
+
+#endif /* _SDL_test_log_h */
+
+/* vi: set ts=4 sw=4 expandtab: */

src/SDL_log.c

@@ -37,6 +37,7 @@
 #define DEFAULT_PRIORITY                SDL_LOG_PRIORITY_CRITICAL
 #define DEFAULT_ASSERT_PRIORITY         SDL_LOG_PRIORITY_WARN
 #define DEFAULT_APPLICATION_PRIORITY    SDL_LOG_PRIORITY_INFO
+#define DEFAULT_TEST_PRIORITY           SDL_LOG_PRIORITY_VERBOSE
 
 typedef struct SDL_LogLevel
 {
@@ -54,6 +55,7 @@ static SDL_LogLevel *SDL_loglevels;
 static SDL_LogPriority SDL_default_priority = DEFAULT_PRIORITY;
 static SDL_LogPriority SDL_assert_priority = DEFAULT_ASSERT_PRIORITY;
 static SDL_LogPriority SDL_application_priority = DEFAULT_APPLICATION_PRIORITY;
+static SDL_LogPriority SDL_test_priority = DEFAULT_TEST_PRIORITY;
 static SDL_LogOutputFunction SDL_log_function = SDL_LogOutput;
 static void *SDL_log_userdata = NULL;
 
@@ -135,7 +137,9 @@ SDL_LogGetPriority(int category)
         }
     }
 
-    if (category == SDL_LOG_CATEGORY_APPLICATION) {
+    if (category == SDL_LOG_CATEGORY_TEST) {
+        return SDL_test_priority;    
+    } else if (category == SDL_LOG_CATEGORY_APPLICATION) {
         return SDL_application_priority;
     } else if (category == SDL_LOG_CATEGORY_ASSERT) {
         return SDL_assert_priority;
@@ -158,6 +162,7 @@ SDL_LogResetPriorities(void)
     SDL_default_priority = DEFAULT_PRIORITY;
     SDL_assert_priority = DEFAULT_ASSERT_PRIORITY;
     SDL_application_priority = DEFAULT_APPLICATION_PRIORITY;
+    SDL_test_priority = DEFAULT_TEST_PRIORITY;
 }
 
 void

src/test/SDL_test_assert.c

@@ -0,0 +1,96 @@
+/*
+  Simple DirectMedia Layer
+  Copyright (C) 1997-2012 Sam Lantinga <slouken@libsdl.org>
+
+  This software is provided 'as-is', without any express or implied
+  warranty.  In no event will the authors be held liable for any damages
+  arising from the use of this software.
+
+  Permission is granted to anyone to use this software for any purpose,
+  including commercial applications, and to alter it and redistribute it
+  freely, subject to the following restrictions:
+
+  1. The origin of this software must not be misrepresented; you must not
+     claim that you wrote the original software. If you use this software
+     in a product, an acknowledgment in the product documentation would be
+     appreciated but is not required.
+  2. Altered source versions must be plainly marked as such, and must not be
+     misrepresented as being the original software.
+  3. This notice may not be removed or altered from any source distribution.
+*/
+
+/*
+
+ Used by the test framework and test cases. 
+
+*/
+
+#include "SDL_config.h"
+
+#include "SDL_test.h"
+
+/*! \brief counts the failed asserts */
+static Uint32 SDLTest_testAssertsFailed = 0;
+
+/*! \brief counts the passed asserts */
+static Uint32 SDLTest_testAssertsPassed = 0;
+
+/* Assert check message format */
+const char *SDLTest_AssertCheckFmt = "Assert %s: %s";
+
+/* Assert summary message format */
+const char *SDLTest_AssertSummaryFmt = "Assert Summary: Total=%d Passed=%d Failed=%d";
+
+/*
+ *  Assert that logs and break execution flow on failures (i.e. for harness errors).
+ */
+void SDLTest_Assert(int assertCondition, char *assertDescription)
+{
+	SDLTest_AssertCheck(assertCondition, assertDescription);
+	SDL_assert((assertCondition));
+}
+
+/*
+ * Assert that logs but does not break execution flow on failures (i.e. for test cases).
+ */
+void SDLTest_AssertCheck(int assertCondition, char *assertDescription)
+{
+	char *fmt = (char *)SDLTest_AssertCheckFmt;
+	if (assertCondition)
+	{
+		SDLTest_testAssertsPassed++;
+		SDLTest_Log(fmt, "Passed", assertDescription);
+	} 
+	else 
+	{
+		SDLTest_testAssertsFailed++;
+		SDLTest_LogError(fmt, "Failed", assertDescription);
+	}
+}
+
+/*
+ * Resets the assert summary counters to zero.
+ */
+void SDLTest_ResetAssertSummary()
+{
+	SDLTest_testAssertsPassed = 0;
+	SDLTest_testAssertsFailed = 0;
+}
+
+/*
+ * Logs summary of all assertions (total, pass, fail) since last reset 
+ * as INFO (failed==0) or ERROR (failed > 0).
+ */
+void SDLTest_LogAssertSummary()
+{
+	char *fmt = (char *)SDLTest_AssertSummaryFmt;
+	Uint32 totalAsserts = SDLTest_testAssertsPassed + SDLTest_testAssertsFailed;
+	if (SDLTest_testAssertsFailed == 0)
+	{
+		SDLTest_Log(fmt, totalAsserts, SDLTest_testAssertsPassed, SDLTest_testAssertsFailed);
+	} 
+	else 
+	{
+		SDLTest_LogError(fmt, totalAsserts, SDLTest_testAssertsPassed, SDLTest_testAssertsFailed);
+	}
+}

src/test/SDL_test_harness.c

@@ -0,0 +1,142 @@
+/*
+  Simple DirectMedia Layer
+  Copyright (C) 1997-2012 Sam Lantinga <slouken@libsdl.org>
+
+  This software is provided 'as-is', without any express or implied
+  warranty.  In no event will the authors be held liable for any damages
+  arising from the use of this software.
+
+  Permission is granted to anyone to use this software for any purpose,
+  including commercial applications, and to alter it and redistribute it
+  freely, subject to the following restrictions:
+
+  1. The origin of this software must not be misrepresented; you must not
+     claim that you wrote the original software. If you use this software
+     in a product, an acknowledgment in the product documentation would be
+     appreciated but is not required.
+  2. Altered source versions must be plainly marked as such, and must not be
+     misrepresented as being the original software.
+  3. This notice may not be removed or altered from any source distribution.
+*/
+
+#include "SDL_config.h"
+
+#include "SDL_test.h"
+
+// TODO: port over harness
+
+/**
+ * Generates a random run seed string for the harness. The generated seed
+ * will contain alphanumeric characters (0-9A-Z).
+ *
+ * Note: The returned string needs to be deallocated by the caller.
+ *
+ * \param length The length of the seed string to generate
+ *
+ * \returns The generated seed string
+ */
+char *
+SDLTest_GenerateRunSeed(const int length)
+{
+	char *seed = NULL;
+	SDLTest_RandomContext randomContext;
+	int counter;
+
+	// Sanity check input
+	if (length <= 0) {
+		SDLTest_LogError("The length of the harness seed must be >0.");
+		return NULL;
+	}
+
+	// Allocate output buffer
+	seed = (char *)SDL_malloc((length + 1) * sizeof(char));
+	if (seed == NULL) {
+		SDLTest_LogError("SDL_malloc for run seed output buffer failed.");
+		return NULL;
+	}
+
+	// Generate a random string of alphanumeric characters
+	SDLTest_RandomInitTime(&randomContext);
+	for (counter = 0; counter < length - 1; ++counter) {
+		unsigned int number = SDLTest_Random(&randomContext);
+		char ch = (char) (number % (91 - 48)) + 48;
+		if (ch >= 58 && ch <= 64) {
+			ch = 65;
+		}
+		seed[counter] = ch;
+	}
+	seed[counter] = '\0';
+
+	return seed;
+}
+
+/**
+ * Generates an execution key for the fuzzer.
+ *
+ * \param runSeed		The run seed to use
+ * \param suiteName		The name of the test suite
+ * \param testName		The name of the test
+ * \param iteration		The iteration count
+ *
+ * \returns The generated execution key to initialize the fuzzer with.
+ *
+ */
+Uint64
+SDLTest_GenerateExecKey(char *runSeed, char *suiteName, char *testName, int iteration)
+{
+	SDLTest_Md5Context md5Context;
+	Uint64 *keys;
+	char iterationString[16];
+	Uint32 runSeedLength;
+	Uint32 suiteNameLength;
+	Uint32 testNameLength;
+	Uint32 iterationStringLength;
+	Uint32 entireStringLength;
+	char *buffer;
+
+	if (runSeed == NULL || strlen(runSeed)==0) {
+		SDLTest_LogError("Invalid runSeed string.");
+		return -1;
+	}
+
+	if (suiteName == NULL || strlen(suiteName)==0) {
+		SDLTest_LogError("Invalid suiteName string.");
+		return -1;
+	}
+
+	if (testName == NULL || strlen(testName)==0) {
+		SDLTest_LogError("Invalid testName string.");
+		return -1;
+	}
+
+	if (iteration <= 0) {
+		SDLTest_LogError("Invalid iteration count.");
+		return -1;
+	}
+
+	// Convert iteration number into a string
+	memset(iterationString, 0, sizeof(iterationString));
+	SDL_snprintf(iterationString, sizeof(iterationString) - 1, "%d", iteration);
+
+	// Combine the parameters into single string
+	runSeedLength = strlen(runSeed);
+	suiteNameLength = strlen(suiteName);
+	testNameLength = strlen(testName);
+	iterationStringLength = strlen(iterationString);
+	entireStringLength  = runSeedLength + suiteNameLength + testNameLength + iterationStringLength + 1;
+	buffer = (char *)SDL_malloc(entireStringLength);
+	if (buffer == NULL) {
+		SDLTest_LogError("SDL_malloc failed to allocate buffer for execKey generation.");
+		return 0;
+	}
+	SDL_snprintf(buffer, entireStringLength, "%s%s%s%d", runSeed, suiteName, testName, iteration);
+
+	// Hash string and use half of the digest as 64bit exec key
+	SDLTest_Md5Init(&md5Context);
+	SDLTest_Md5Update(&md5Context, (unsigned char *)buffer, entireStringLength);
+	SDLTest_Md5Final(&md5Context);
+	SDL_free(buffer);
+	keys = (Uint64 *)md5Context.digest;
+
+	return keys[0];
+}

src/test/SDL_test_log.c

@@ -0,0 +1,100 @@
+/*
+  Simple DirectMedia Layer
+  Copyright (C) 1997-2012 Sam Lantinga <slouken@libsdl.org>
+
+  This software is provided 'as-is', without any express or implied
+  warranty.  In no event will the authors be held liable for any damages
+  arising from the use of this software.
+
+  Permission is granted to anyone to use this software for any purpose,
+  including commercial applications, and to alter it and redistribute it
+  freely, subject to the following restrictions:
+
+  1. The origin of this software must not be misrepresented; you must not
+     claim that you wrote the original software. If you use this software
+     in a product, an acknowledgment in the product documentation would be
+     appreciated but is not required.
+  2. Altered source versions must be plainly marked as such, and must not be
+     misrepresented as being the original software.
+  3. This notice may not be removed or altered from any source distribution.
+*/
+
+/*
+
+ Used by the test framework and test cases. 
+
+*/
+
+#include "SDL_config.h"
+
+#include <stdarg.h> /* va_list */
+#include <time.h>
+
+#include "SDL_test.h"
+
+/* 
+ * Note: Maximum size of SDLTest log message is less than SDLs limit 
+ * to ensure we can fit additional information such as the timestamp. 
+ */
+#define SDLTEST_MAX_LOGMESSAGE_LENGTH	3584
+
+/*!
+ * Converts unix timestamp to its ascii representation in localtime
+ *
+ * Note: Uses a static buffer internally, so the return value
+ * isn't valid after the next call of this function. If you
+ * want to retain the return value, make a copy of it.
+ *
+ * \param timestamp A Timestamp, i.e. time(0)
+ *
+ * \return Ascii representation of the timestamp in localtime
+ */
+char *SDLTest_TimestampToString(const time_t timestamp) 
+{
+	time_t copy;
+	static char buffer[256];
+	struct tm *local;
+
+	memset(buffer, 0, sizeof(buffer));\
+	copy = timestamp;
+	local = localtime(&copy);
+	strftime(buffer, sizeof(buffer), "%a %Y-%m-%d %H:%M:%S %Z", local);
+
+	return buffer;
+}
+
+/*
+ * Prints given message with a timestamp in the TEST category and INFO priority.
+ */
+void SDLTest_Log(char *fmt, ...)
+{
+	va_list list;
+	char logMessage[SDLTEST_MAX_LOGMESSAGE_LENGTH];
+
+	// Print log message into a buffer
+	memset(logMessage, 0, SDLTEST_MAX_LOGMESSAGE_LENGTH);
+	va_start(list, fmt);
+	SDL_vsnprintf(logMessage, SDLTEST_MAX_LOGMESSAGE_LENGTH - 1, fmt, list);
+	va_end(list);
+
+	// Log with timestamp and newline
+	SDL_LogMessage(SDL_LOG_CATEGORY_TEST, SDL_LOG_PRIORITY_INFO, "%s: %s\n", SDLTest_TimestampToString(time(0)), logMessage);
+}
+
+/*
+ * Prints given message with a timestamp in the TEST category and the ERROR priority.
+ */
+void SDLTest_LogError(char *fmt, ...)
+{
+	va_list list;
+	char logMessage[SDLTEST_MAX_LOGMESSAGE_LENGTH];
+
+	// Print log message into a buffer
+	memset(logMessage, 0, SDLTEST_MAX_LOGMESSAGE_LENGTH);
+	va_start(list, fmt);
+	SDL_vsnprintf(logMessage, SDLTEST_MAX_LOGMESSAGE_LENGTH - 1, fmt, list);
+	va_end(list);
+
+	// Log with timestamp and newline
+	SDL_LogMessage(SDL_LOG_CATEGORY_TEST, SDL_LOG_PRIORITY_ERROR, "%s: %s\n", SDLTest_TimestampToString(time(0)), logMessage);
+}