unit-tests: Implement testing framework without "check"

src/libstrongswan/tests/Makefile.am

@@ -3,7 +3,7 @@ TESTS = test_runner
 check_PROGRAMS = $(TESTS)
 
 test_runner_SOURCES = \
-  test_runner.c test_runner.h test_suite.h \
+  test_runner.c test_runner.h test_suite.c test_suite.h \
   suites/test_linked_list.c \
   suites/test_enumerator.c \
   suites/test_linked_list_enumerator.c \
@@ -28,11 +28,9 @@ test_runner_CFLAGS = \
   -I$(top_srcdir)/src/libstrongswan \
   -DPLUGINDIR=\""$(top_builddir)/src/libstrongswan/plugins\"" \
   -DPLUGINS=\""${s_plugins}\"" \
-  @COVERAGE_CFLAGS@ \
+  @COVERAGE_CFLAGS@
-  @CHECK_CFLAGS@
 
 test_runner_LDFLAGS = @COVERAGE_LDFLAGS@
 test_runner_LDADD = \
   $(top_builddir)/src/libstrongswan/libstrongswan.la \
-  $(PTHREADLIB) \
+  $(PTHREADLIB)
-  @CHECK_LIBS@

src/libstrongswan/tests/test_runner.c

@@ -1,6 +1,8 @@
 /*
  * Copyright (C) 2013 Tobias Brunner
  * Hochschule fuer Technik Rapperswil
+ * Copyright (C) 2013 Martin Willi
+ * Copyright (C) 2013 revosec AG
  *
  * This program is free software; you can redistribute it and/or modify it
  * under the terms of the GNU General Public License as published by the
@@ -13,14 +15,20 @@
  * for more details.
  */
 
-#include <unistd.h>
-
 #include "test_runner.h"
 
 #include <library.h>
 #include <plugins/plugin_feature.h>
+#include <collections/array.h>
 
 #include <dirent.h>
+#include <unistd.h>
+#include <limits.h>
+
+/**
+ * Get a tty color escape character for stderr
+ */
+#define TTY(color) tty_escape_get(2, TTY_FG_##color)
 
 /**
  * Load plugins from builddir
@@ -43,17 +51,147 @@ static bool load_plugins()
 	return lib->plugins->load(lib->plugins, PLUGINS);
 }
 
-int main()
+/**
+ * Check if a specific feature is available, return falg if so
+ */
+static int check_feature(plugin_feature_t feature, int flag)
 {
-	SRunner *sr;
+	if (lib->plugins->has_feature(lib->plugins, feature))
-	int nf;
+	{
+		return flag;
+	}
+	return 0;
+}
 
-	/* test cases are forked and there is no cleanup, so disable leak detective.
+/**
-	 * if test_suite.h is included leak detective is enabled in test cases */
+ * Load all available test suites
-	setenv("LEAK_DETECTIVE_DISABLE", "1", 1);
+ */
-	/* redirect all output to stderr (to redirect make's stdout to /dev/null) */
+static array_t *load_suites()
-	dup2(2, 1);
+{
+	array_t *suites;
+	enum {
+		OTEST_RSA = (1<<0),
+		OTEST_ECDSA = (1<<1),
+	} otest = 0;
 
+	library_init(NULL);
+	if (!load_plugins())
+	{
+		library_deinit();
+		return NULL;
+	}
+	lib->plugins->status(lib->plugins, LEVEL_CTRL);
+
+	/* we have to build the test suite array without leak detective, so
+	 * separate plugin checks and suite creation */
+	otest |= check_feature(PLUGIN_DEPENDS(PRIVKEY_GEN, KEY_RSA), OTEST_RSA);
+	otest |= check_feature(PLUGIN_DEPENDS(PRIVKEY_GEN, KEY_ECDSA), OTEST_ECDSA);
+
+	library_deinit();
+
+	suites = array_create(0, 0);
+
+	array_insert(suites, -1, bio_reader_suite_create());
+	array_insert(suites, -1, bio_writer_suite_create());
+	array_insert(suites, -1, chunk_suite_create());
+	array_insert(suites, -1, enum_suite_create());
+	array_insert(suites, -1, enumerator_suite_create());
+	array_insert(suites, -1, linked_list_suite_create());
+	array_insert(suites, -1, linked_list_enumerator_suite_create());
+	array_insert(suites, -1, hashtable_suite_create());
+	array_insert(suites, -1, array_suite_create());
+	array_insert(suites, -1, identification_suite_create());
+	array_insert(suites, -1, threading_suite_create());
+	array_insert(suites, -1, utils_suite_create());
+	array_insert(suites, -1, host_suite_create());
+	array_insert(suites, -1, vectors_suite_create());
+	array_insert(suites, -1, pen_suite_create());
+	array_insert(suites, -1, asn1_suite_create());
+	array_insert(suites, -1, printf_suite_create());
+	if (otest & OTEST_RSA)
+	{
+		array_insert(suites, -1, rsa_suite_create());
+	}
+	if (otest & OTEST_ECDSA)
+	{
+		array_insert(suites, -1, ecdsa_suite_create());
+	}
+
+	return suites;
+}
+
+/**
+ * Unload and destroy test suites and associated data
+ */
+static void unload_suites(array_t *suites)
+{
+	test_suite_t *suite;
+	test_case_t *tcase;
+
+	while (array_remove(suites, 0, &suite))
+	{
+		while (array_remove(suite->tcases, 0, &tcase))
+		{
+			array_destroy(tcase->functions);
+			array_destroy(tcase->fixtures);
+		}
+		free(suite);
+	}
+	array_destroy(suites);
+}
+
+/**
+ * Run a single test function, return FALSE on failure
+ */
+static bool run_test(test_function_t *tfun, int i)
+{
+	if (test_restore_point())
+	{
+		tfun->cb(i);
+		return TRUE;
+	}
+	return FALSE;
+}
+
+/**
+ * Invoke fixture setup/teardown
+ */
+static bool call_fixture(test_case_t *tcase, bool up)
+{
+	enumerator_t *enumerator;
+	test_fixture_t *fixture;
+	bool failure = FALSE;
+
+	enumerator = array_create_enumerator(tcase->fixtures);
+	while (enumerator->enumerate(enumerator, &fixture))
+	{
+		if (test_restore_point())
+		{
+			if (up)
+			{
+				fixture->setup();
+			}
+			else
+			{
+				fixture->teardown();
+			}
+		}
+		else
+		{
+			failure = TRUE;
+			break;
+		}
+	}
+	enumerator->destroy(enumerator);
+
+	return !failure;
+}
+
+/**
+ * Test initialization, initializes libstrongswan for the next run
+ */
+static bool pre_test()
+{
 	library_init(NULL);
 
 	/* use non-blocking RNG to generate keys fast */
@@ -62,47 +200,295 @@ int main()
 			lib->settings->get_str(lib->settings,
 				"libstrongswan.plugins.random.urandom", "/dev/urandom"));
 
+	if (lib->leak_detective)
+	{
+		/* disable leak reports during testing */
+		lib->leak_detective->set_report_cb(lib->leak_detective,
+										   NULL, NULL, NULL);
+	}
 	if (!load_plugins())
 	{
 		library_deinit();
-		return EXIT_FAILURE;
+		return FALSE;
 	}
-	lib->plugins->status(lib->plugins, LEVEL_CTRL);
 
-	sr = srunner_create(NULL);
+	dbg_default_set_level(LEVEL_SILENT);
-	srunner_add_suite(sr, bio_reader_suite_create());
+	return TRUE;
-	srunner_add_suite(sr, bio_writer_suite_create());
+}
-	srunner_add_suite(sr, chunk_suite_create());
+
-	srunner_add_suite(sr, enum_suite_create());
+/**
-	srunner_add_suite(sr, enumerator_suite_create());
+ * Failure description
-	srunner_add_suite(sr, linked_list_suite_create());
+ */
-	srunner_add_suite(sr, linked_list_enumerator_suite_create());
+typedef struct {
-	srunner_add_suite(sr, hashtable_suite_create());
+	char *name;
-	srunner_add_suite(sr, array_suite_create());
+	char msg[512 - sizeof(char*) - 2 * sizeof(int)];
-	srunner_add_suite(sr, identification_suite_create());
+	const char *file;
-	srunner_add_suite(sr, threading_suite_create());
+	int line;
-	srunner_add_suite(sr, utils_suite_create());
+	int i;
-	srunner_add_suite(sr, host_suite_create());
+	backtrace_t *bt;
-	srunner_add_suite(sr, vectors_suite_create());
+} failure_t;
-	srunner_add_suite(sr, printf_suite_create());
+
-	srunner_add_suite(sr, pen_suite_create());
+/**
-	srunner_add_suite(sr, asn1_suite_create());
+ * Data passed to leak report callbacks
-	if (lib->plugins->has_feature(lib->plugins,
+ */
-								  PLUGIN_DEPENDS(PRIVKEY_GEN, KEY_RSA)))
+typedef struct {
+	array_t *failures;
+	char *name;
+	int i;
+	int leaks;
+} report_data_t;
+
+/**
+ * Leak report callback, build failures from leaks
+ */
+static void report_leaks(report_data_t *data, int count, size_t bytes,
+						 backtrace_t *bt, bool detailed)
+{
+	failure_t failure = {
+		.name = data->name,
+		.i = data->i,
+		.bt = bt->clone(bt),
+	};
+
+	snprintf(failure.msg, sizeof(failure.msg),
+			 "Leak detected: %d allocations using %zu bytes", count, bytes);
+
+	array_insert(data->failures, -1, &failure);
+}
+
+/**
+ * Leak summary callback, check if any leaks found
+ */
+static void sum_leaks(report_data_t *data, int count, size_t bytes,
+					  int whitelisted)
+{
+	data->leaks = count;
+}
+
+/**
+ * Do library cleanup and optionally check for memory leaks
+ */
+static bool post_test(bool check_leaks, array_t *failures, char *name, int i)
+{
+	report_data_t data = {
+		.failures = failures,
+		.name = name,
+		.i = i,
+	};
+
+	if (check_leaks && lib->leak_detective)
 	{
-		srunner_add_suite(sr, rsa_suite_create());
+		lib->leak_detective->set_report_cb(lib->leak_detective,
+								(leak_detective_report_cb_t)report_leaks,
+								(leak_detective_summary_cb_t)sum_leaks, &data);
 	}
-	if (lib->plugins->has_feature(lib->plugins,
-								  PLUGIN_DEPENDS(PRIVKEY_GEN, KEY_ECDSA)))
-	{
-		srunner_add_suite(sr, ecdsa_suite_create());
-	}
-
-	srunner_run_all(sr, CK_NORMAL);
-	nf = srunner_ntests_failed(sr);
-
-	srunner_free(sr);
 	library_deinit();
 
-	return (nf == 0) ? EXIT_SUCCESS : EXIT_FAILURE;
+	return data.leaks != 0;
+}
+
+/**
+ * Collect failure information, add failure_t to array
+ */
+static void collect_failure_info(array_t *failures, char *name, int i)
+{
+	failure_t failure = {
+		.name = name,
+		.i = i,
+		.bt = test_failure_backtrace(),
+	};
+
+	failure.line = test_failure_get(failure.msg, sizeof(failure.msg),
+									&failure.file);
+
+	array_insert(failures, -1, &failure);
+}
+
+/**
+ * Print array of collected failure_t to stderr
+ */
+static void print_failures(array_t *failures)
+{
+	failure_t failure;
+
+	while (array_remove(failures, 0, &failure))
+	{
+		fprintf(stderr, "      %sFailure in '%s': %s (",
+				TTY(RED), failure.name, failure.msg);
+		if (failure.line)
+		{
+			fprintf(stderr, "%s:%d, ", failure.file, failure.line);
+		}
+		fprintf(stderr, "i = %d)%s\n", failure.i, TTY(DEF));
+		if (failure.bt)
+		{
+			failure.bt->log(failure.bt, stderr, TRUE);
+			failure.bt->destroy(failure.bt);
+		}
+	}
+}
+
+/**
+ * Run a single test case with fixtures
+ */
+static bool run_case(test_case_t *tcase)
+{
+	enumerator_t *enumerator;
+	test_function_t *tfun;
+	int passed = 0;
+	array_t *failures;
+
+	failures = array_create(sizeof(failure_t), 0);
+
+	fprintf(stderr, "    Running case '%s': ", tcase->name);
+	fflush(stderr);
+
+	enumerator = array_create_enumerator(tcase->functions);
+	while (enumerator->enumerate(enumerator, &tfun))
+	{
+		int i, rounds = 0;
+
+		for (i = tfun->start; i < tfun->end; i++)
+		{
+			if (pre_test())
+			{
+				bool ok = FALSE, leaks = FALSE;
+
+				test_setup_timeout(tcase->timeout);
+
+				if (call_fixture(tcase, TRUE))
+				{
+					if (run_test(tfun, i))
+					{
+						if (call_fixture(tcase, FALSE))
+						{
+							ok = TRUE;
+						}
+					}
+					else
+					{
+						call_fixture(tcase, FALSE);
+					}
+
+				}
+				leaks = post_test(ok, failures, tfun->name, i);
+
+				test_setup_timeout(0);
+
+				if (ok)
+				{
+					if (!leaks)
+					{
+						rounds++;
+						fprintf(stderr, "%s+%s", TTY(GREEN), TTY(DEF));
+					}
+				}
+				else
+				{
+					collect_failure_info(failures, tfun->name, i);
+				}
+				if (!ok || leaks)
+				{
+					fprintf(stderr, "%s-%s", TTY(RED), TTY(DEF));
+				}
+			}
+			else
+			{
+				fprintf(stderr, "!");
+			}
+		}
+		fflush(stderr);
+		if (rounds == tfun->end - tfun->start)
+		{
+			passed++;
+		}
+	}
+	enumerator->destroy(enumerator);
+
+	fprintf(stderr, "\n");
+
+	print_failures(failures);
+	array_destroy(failures);
+
+	return passed == array_count(tcase->functions);
+}
+
+/**
+ * Run a single test suite
+ */
+static bool run_suite(test_suite_t *suite)
+{
+	enumerator_t *enumerator;
+	test_case_t *tcase;
+	int passed = 0;
+
+	fprintf(stderr, "  Running suite '%s':\n", suite->name);
+
+	enumerator = array_create_enumerator(suite->tcases);
+	while (enumerator->enumerate(enumerator, &tcase))
+	{
+		if (run_case(tcase))
+		{
+			passed++;
+		}
+	}
+	enumerator->destroy(enumerator);
+
+	if (passed == array_count(suite->tcases))
+	{
+		fprintf(stderr, "  %sPassed all %u '%s' test cases%s\n",
+				TTY(GREEN), array_count(suite->tcases), suite->name, TTY(DEF));
+		return TRUE;
+	}
+	fprintf(stderr, "  %sPassed %u/%u '%s' test cases%s\n",
+			TTY(RED), passed, array_count(suite->tcases), suite->name, TTY(DEF));
+	return FALSE;
+}
+
+int main(int argc, char *argv[])
+{
+	array_t *suites;
+	test_suite_t *suite;
+	enumerator_t *enumerator;
+	int passed = 0, result;
+
+	/* redirect all output to stderr (to redirect make's stdout to /dev/null) */
+	dup2(2, 1);
+
+	test_setup_handler();
+
+	suites = load_suites();
+	if (!suites)
+	{
+		return EXIT_FAILURE;
+	}
+
+	fprintf(stderr, "Running %u test suites:\n", array_count(suites));
+
+	enumerator = array_create_enumerator(suites);
+	while (enumerator->enumerate(enumerator, &suite))
+	{
+		if (run_suite(suite))
+		{
+			passed++;
+		}
+	}
+	enumerator->destroy(enumerator);
+
+	if (passed == array_count(suites))
+	{
+		fprintf(stderr, "%sPassed all %u suites%s\n",
+				TTY(GREEN), array_count(suites), TTY(DEF));
+		result = EXIT_SUCCESS;
+	}
+	else
+	{
+		fprintf(stderr, "%sPassed %u of %u suites%s\n",
+				TTY(RED), passed, array_count(suites), TTY(DEF));
+		result = EXIT_FAILURE;
+	}
+
+	unload_suites(suites);
+
+	return result;
 }

src/libstrongswan/tests/test_runner.h

@@ -16,7 +16,7 @@
 #ifndef TEST_RUNNER_H_
 #define TEST_RUNNER_H_
 
-#include <check.h>
+#include <test_suite.h>
 
 Suite *bio_reader_suite_create();
 Suite *bio_writer_suite_create();

src/libstrongswan/tests/test_suite.c

@@ -0,0 +1,240 @@
+/*
+ * Copyright (C) 2013 Martin Willi
+ * Copyright (C) 2013 revosec AG
+ *
+ * This program is free software; you can redistribute it and/or modify it
+ * under the terms of the GNU General Public License as published by the
+ * Free Software Foundation; either version 2 of the License, or (at your
+ * option) any later version.  See <http://www.fsf.org/copyleft/gpl.txt>.
+ *
+ * This program is distributed in the hope that it will be useful, but
+ * WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY
+ * or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
+ * for more details.
+ */
+
+#include "test_suite.h"
+
+#include <signal.h>
+#include <unistd.h>
+
+/**
+ * Failure message buf
+ */
+static char failure_buf[512];
+
+/**
+ * Source file failure occured
+ */
+static const char *failure_file;
+
+/**
+ * Line of source file failure occured
+ */
+static int failure_line;
+
+/**
+ * Backtrace of failure, if any
+ */
+static backtrace_t *failure_backtrace;
+
+/**
+ * Longjump restore point when failing
+ */
+sigjmp_buf test_restore_point_env;
+
+/**
+ * See header.
+ */
+test_suite_t* test_suite_create(const char *name)
+{
+	test_suite_t *suite;
+
+	INIT(suite,
+		.name = name,
+		.tcases = array_create(0, 0),
+	);
+	return suite;
+}
+
+/**
+ * See header.
+ */
+test_case_t* test_case_create(const char *name)
+{
+	test_case_t *tcase;
+
+	INIT(tcase,
+		.name = name,
+		.functions = array_create(sizeof(test_function_t), 0),
+		.fixtures = array_create(sizeof(test_fixture_t), 0),
+		.timeout = TEST_FUNCTION_DEFAULT_TIMEOUT,
+	);
+	return tcase;
+}
+
+/**
+ * See header.
+ */
+void test_case_add_checked_fixture(test_case_t *tcase, test_fixture_cb_t setup,
+								   test_fixture_cb_t teardown)
+{
+	test_fixture_t fixture = {
+		.setup = setup,
+		.teardown = teardown,
+	};
+	array_insert(tcase->fixtures, -1, &fixture);
+}
+
+/**
+ * See header.
+ */
+void test_case_add_test_name(test_case_t *tcase, char *name,
+							 test_function_cb_t cb, int start, int end)
+{
+	test_function_t fun = {
+		.name = name,
+		.cb = cb,
+		.start = start,
+		.end = end,
+	};
+	array_insert(tcase->functions, -1, &fun);
+}
+
+/**
+ * See header.
+ */
+void test_case_set_timeout(test_case_t *tcase, int s)
+{
+	tcase->timeout = s;
+}
+
+/**
+ * See header.
+ */
+void test_suite_add_case(test_suite_t *suite, test_case_t *tcase)
+{
+	array_insert(suite->tcases, -1, tcase);
+}
+
+/**
+ * Let test case fail
+ */
+static inline void test_failure()
+{
+	siglongjmp(test_restore_point_env, 1);
+}
+
+/**
+ * See header.
+ */
+void test_fail_vmsg(const char *file, int line, char *fmt, va_list args)
+{
+	vsnprintf(failure_buf, sizeof(failure_buf), fmt, args);
+	failure_line = line;
+	failure_file = file;
+
+	test_failure();
+}
+
+/**
+ * See header.
+ */
+void test_fail_msg(const char *file, int line, char *fmt, ...)
+{
+	va_list args;
+
+	va_start(args, fmt);
+	vsnprintf(failure_buf, sizeof(failure_buf), fmt, args);
+	failure_line = line;
+	failure_file = file;
+	va_end(args);
+
+	test_failure();
+}
+
+/**
+ * Signal handler catching critical and alarm signals
+ */
+static void test_sighandler(int signal)
+{
+	char *signame;
+	bool old = FALSE;
+
+	switch (signal)
+	{
+		case SIGSEGV:
+			signame = "SIGSEGV";
+			break;
+		case SIGILL:
+			signame = "SIGILL";
+			break;
+		case SIGBUS:
+			signame = "SIGBUS";
+			break;
+		case SIGALRM:
+			signame = "timeout";
+			break;
+		default:
+			signame = "SIG";
+			break;
+	}
+	if (lib->leak_detective)
+	{
+		old = lib->leak_detective->set_state(lib->leak_detective, FALSE);
+	}
+	failure_backtrace = backtrace_create(3);
+	if (lib->leak_detective)
+	{
+		lib->leak_detective->set_state(lib->leak_detective, old);
+	}
+	test_fail_msg(NULL, 0, "%s(%d)", signame, signal);
+}
+
+/**
+ * See header.
+ */
+void test_setup_handler()
+{
+	struct sigaction action;
+
+	action.sa_handler = test_sighandler;
+	action.sa_flags = 0;
+	sigemptyset(&action.sa_mask);
+	sigaction(SIGSEGV, &action, NULL);
+	sigaction(SIGILL, &action, NULL);
+	sigaction(SIGBUS, &action, NULL);
+	sigaction(SIGALRM, &action, NULL);
+}
+
+/**
+ * See header.
+ */
+void test_setup_timeout(int s)
+{
+	alarm(s);
+}
+
+/**
+ * See header.
+ */
+int test_failure_get(char *msg, int len, const char **file)
+{
+	strncpy(msg, failure_buf, len - 1);
+	msg[len - 1] = 0;
+	*file = failure_file;
+	return failure_line;
+}
+
+/**
+ * See header.
+ */
+backtrace_t *test_failure_backtrace()
+{
+	backtrace_t *bt;
+
+	bt = failure_backtrace;
+	failure_backtrace = NULL;
+
+	return bt;
+}

src/libstrongswan/tests/test_suite.h

@@ -1,6 +1,8 @@
 /*
  * Copyright (C) 2013 Tobias Brunner
  * Hochschule fuer Technik Rapperswil
+ * Copyright (C) 2013 Martin Willi
+ * Copyright (C) 2013 revosec AG
  *
  * This program is free software; you can redistribute it and/or modify it
  * under the terms of the GNU General Public License as published by the
@@ -16,95 +18,312 @@
 #ifndef TEST_UTILS_H_
 #define TEST_UTILS_H_
 
-#include <check.h>
+#define _GNU_SOURCE
+#include <setjmp.h>
+
 #include <library.h>
 #include <utils/debug.h>
+#include <utils/backtrace.h>
+#include <collections/array.h>
+
+typedef struct test_suite_t test_suite_t;
+typedef struct test_case_t test_case_t;
+typedef struct test_function_t test_function_t;
+typedef struct test_fixture_t test_fixture_t;
 
 /**
- * Used to mark test cases that use test fixtures.
+ * Default timeout for a single test function
  */
-#define UNIT_TEST_FIXTURE_USED "UNIT_TEST_FIXTURE_USED"
+#define TEST_FUNCTION_DEFAULT_TIMEOUT 2
 
 /**
- * Check for memory leaks and fail if any are encountered.
+ * Test function implementation
  */
-#define CHECK_FOR_LEAKS() do \
+typedef void (*test_function_cb_t)(int);
-{ \
-	if (lib->leak_detective) \
-	{ \
-		if (lib->leak_detective->leaks(lib->leak_detective)) { \
-			lib->leak_detective->report(lib->leak_detective, TRUE); \
-		} \
-		ck_assert_int_eq(lib->leak_detective->leaks(lib->leak_detective), 0); \
-	} \
-} \
-while(0)
 
 /**
- * Extended versions of the START|END_TEST macros that use leak detective.
+ * Fixture for a test case.
+ */
+typedef void (*test_fixture_cb_t)(void);
+
+/**
+ * A test suite; a collection of test cases with fixtures
+ */
+struct test_suite_t {
+	/** name of the test suite */
+	const char *name;
+	/** test cases registered, as test_case_t* */
+	array_t *tcases;
+};
+
+/**
+ * A test case; multiple test functions using the same fixtures
+ */
+struct test_case_t {
+	/** name of the test case */
+	const char *name;
+	/** tests registered, as test_function_t */
+	array_t *functions;
+	/** fixture for tests, as test_fixture_t */
+	array_t *fixtures;
+	/** timeout for each function, in s */
+	int timeout;
+};
+
+/**
+ * A test function, with optional loop setup
+ */
+struct test_function_t {
+	/** name of test function */
+	char *name;
+	/** tests function registered, test_function_t* */
+	test_function_cb_t cb;
+	/** start for loop test */
+	int start;
+	/** end for loop test */
+	int end;
+};
+
+/**
+ * Registered fixture for a test case
+ */
+struct test_fixture_t {
+	test_fixture_cb_t setup;
+	test_fixture_cb_t teardown;
+};
+
+/**
+ * Create a new test suite
  *
- * Since each test case runs in its own fork of the test runner the stuff
+ * @param name		name of the test suite
- * allocated before the test starts is not freed, so leak detective is disabled
+ * @return			test suite
- * by default to prevent false positives.  By enabling it right when the test
+ */
- * starts we at least capture leaks created by the tested objects/functions and
+test_suite_t* test_suite_create(const char *name);
- * the test case itself.  This allows writing test cases for cleanup functions.
+
+/**
+ * Create a new test case
  *
- * To define test fixture with possibly allocated/destroyed memory that is
+ * @param name		name of test case
- * allocated/freed in a test case use the START|END_SETUP|TEARDOWN macros.
+ * @return			test case
  */
-#undef START_TEST
+test_case_t* test_case_create(const char *name);
-#define START_TEST(name) \
-static void name (int _i CK_ATTRIBUTE_UNUSED) \
-{ \
-	tcase_fn_start(""#name, __FILE__, __LINE__); \
-	dbg_default_set_level(LEVEL_SILENT); \
-	if (lib->leak_detective) \
-	{ \
-		lib->leak_detective->set_state(lib->leak_detective, TRUE); \
-	}
 
-#undef END_TEST
+/**
-#define END_TEST \
+ * Add a setup/teardown function to the test case
-	if (!lib->get(lib, UNIT_TEST_FIXTURE_USED)) \
+ *
+ * @param tcase		test case to add a fixture to
+ * @param setup		setup function called before each test
+ * @param teardown	cleanup function called after each test
+ */
+void test_case_add_checked_fixture(test_case_t *tcase, test_fixture_cb_t setup,
+								   test_fixture_cb_t teardown);
+
+/**
+ * Add a test function to a test case, with a name, looped several times
+ *
+ * @param name		name of the test case
+ * @param tcase		test case to add test function to
+ * @param cb		callback function to invoke for test
+ * @param start		start of loop counter
+ * @param end		end of loop counter
+ */
+void test_case_add_test_name(test_case_t *tcase, char *name,
+							 test_function_cb_t cb, int start, int end);
+
+/**
+ * Add a test function to a test case
+ *
+ * @param tcase		test case to add test function to
+ * @param cb		callback function to invoke for test
+ */
+#define test_case_add_test(tcase, cb) \
+	test_case_add_test_name(tcase, #cb, cb, 0, 1)
+
+/**
+ * Add a test function to a test case, looped several times
+ *
+ * @param tcase		test case to add test function to
+ * @param cb		callback function to invoke for test
+ * @param start		start of loop counter
+ * @param end		end of loop counter
+ */
+#define test_case_add_loop_test(tcase, cb, start, end) \
+	test_case_add_test_name(tcase, #cb, cb, start, end)
+
+/**
+ * Set a custom timeout for test functions in a test case
+ *
+ * @param tcase		test case to set timeout for
+ * @param s			test timeout in s
+ */
+void test_case_set_timeout(test_case_t *tcase, int s);
+
+/**
+ * Add a test function to a test case, looped several times
+ *
+ * @param tcase		test case to add test function to
+ * @param cb		callback function to invoke for test
+ * @param start		start of loop counter
+ * @param end		end of loop counter
+ */
+void test_suite_add_case(test_suite_t *suite, test_case_t *tcase);
+
+/**
+ * sigjmp restore point used by test_restore_point
+ */
+extern sigjmp_buf test_restore_point_env;
+
+/**
+ * Set or return from an execution restore point
+ *
+ * This call sets a restore execution point and returns TRUE after it has
+ * been set up. On test failure, the execution is returned to the restore point
+ * and FALSE is returned to indicate test failure.
+ *
+ * @return			TRUE if restore point set, FALSE when restored
+ */
+#define test_restore_point() (sigsetjmp(test_restore_point_env, 1) == 0)
+
+/**
+ * Set up signal handlers for test cases
+ */
+void test_setup_handler();
+
+/**
+ * Set up a timeout to let a test fail
+ *
+ * @param s			timeout, 0 to disable timeout
+ */
+void test_setup_timeout(int s);
+
+/**
+ * Get info about a test failure
+ *
+ * @param msg		buffer receiving failure info
+ * @param len		size of msg buffer
+ * @param file		pointer receiving source code file
+ * @return			source code line number
+ */
+int test_failure_get(char *msg, int len, const char **file);
+
+/**
+ * Get a backtrace for a failure.
+ *
+ * @return			allocated backtrace of test failure, if any
+ */
+backtrace_t *test_failure_backtrace();
+
+/**
+ * Let a test fail and set a message using vprintf style arguments.
+ *
+ * @param file		source code file name
+ * @param line		source code line number
+ * @param fmt		printf format string
+ * @param args		argument list for fmt
+ */
+void test_fail_vmsg(const char *file, int line, char *fmt, va_list args);
+
+/**
+ * Let a test fail and set a message using printf style arguments.
+ *
+ * @param file		source code file name
+ * @param line		source code line number
+ * @param fmt		printf format string
+ * @param ...		arguments for fmt
+ */
+void test_fail_msg(const char *file, int line, char *fmt, ...);
+
+/**
+ * Check if two integers equal, fail test if not
+ *
+ * @param a			first integer
+ * @param b			second integer
+ */
+#define test_int_eq(a, b) \
+({ \
+	typeof(a) _a = a; \
+	typeof(b) _b = b; \
+	if (_a != _b) \
 	{ \
-		CHECK_FOR_LEAKS(); \
+		test_fail_msg(__FILE__, __LINE__, #a " != " #b " (%d != %d)", _a, _b); \
 	} \
-}
+})
 
 /**
- * Define a function to setup a test fixture that can be used with the above
+ * Check if two strings equal, fail test if not
- * macros.
+ *
+ * @param a			first string
+ * @param b			second string
  */
-#define START_SETUP(name) \
+#define test_str_eq(a, b) \
-static void name() \
+({ \
-{ \
+	char* _a = (char*)a; \
-	lib->set(lib, UNIT_TEST_FIXTURE_USED, (void*)TRUE); \
+	char* _b = (char*)b; \
-	if (lib->leak_detective) \
+	if (!_a || !_b || !streq(_a, _b)) \
 	{ \
-		lib->leak_detective->set_state(lib->leak_detective, TRUE); \
+		test_fail_msg(__FILE__, __LINE__, \
-	}
+					  #a " != " #b " (\"%s\" != \"%s\")", _a, _b); \
+	} \
+})
 
 /**
- * End a setup function
+ * Check if a statement evaluates to TRUE, fail test if not
+ *
+ * @param x			statement to evaluate
  */
+#define test_assert(x) \
+({ \
+	if (!(x)) \
+	{ \
+		test_fail_msg(__FILE__, __LINE__, #x); \
+	} \
+})
+
+/**
+ * Check if a statement evaluates to TRUE, fail and print a message if not
+ *
+ * @param x			statement to evaluate
+ * @param fmt		message format string
+ * @param ...		fmt printf arguments
+ */
+#define test_assert_msg(x, fmt, ...) \
+({ \
+	if (!(x)) \
+	{ \
+		test_fail_msg(__FILE__, __LINE__, #x ": " fmt, ##__VA_ARGS__); \
+	} \
+})
+
+
+
+/* "check unit testing" compatibility */
+#define Suite test_suite_t
+#define TCase test_case_t
+#define ck_assert_int_eq test_int_eq
+#define ck_assert test_assert
+#define ck_assert_msg test_assert_msg
+#define ck_assert_str_eq test_str_eq
+#define fail(fmt, ...) test_fail_msg(__FILE__, __LINE__, fmt, ##__VA_ARGS__)
+#define fail_if(x, fmt, ...) \
+({ \
+	if (x) \
+	{ \
+		test_fail_msg(__FILE__, __LINE__, #x ": " fmt, ##__VA_ARGS__); \
+	} \
+})
+#define fail_unless test_assert_msg
+#define suite_create test_suite_create
+#define tcase_create test_case_create
+#define tcase_add_checked_fixture test_case_add_checked_fixture
+#define tcase_add_test test_case_add_test
+#define tcase_add_loop_test test_case_add_loop_test
+#define tcase_set_timeout test_case_set_timeout
+#define suite_add_tcase test_suite_add_case
+#define START_TEST(name) static void name (int _i) {
+#define END_TEST }
+#define START_SETUP(name) static void name() {
 #define END_SETUP }
-
+#define START_TEARDOWN(name) static void name() {
-/**
+#define END_TEARDOWN }
- * Define a function to teardown a test fixture that can be used with the above
- * macros.
- */
-#define START_TEARDOWN(name) \
-static void name() \
-{
-
-/**
- * End a teardown function
- */
-#define END_TEARDOWN \
-	if (lib->get(lib, UNIT_TEST_FIXTURE_USED)) \
-	{ \
-		CHECK_FOR_LEAKS(); \
-	} \
-}
 
 #endif /** TEST_UTILS_H_ */