path: root/test-runner.c

/* This file contains test-runner
 *
 * Copyright (c) 2016 Samsung Electronics Co., Ltd All Rights Reserved
 * Author: Kazimierz Krosman <k.krosman@samsung.com>
 *
 * Licensed under the Apache License, Version 2.0 (the "License");
 * you may not use this file except in compliance with the License.
 * You may obtain a copy of the License at
 *
 * http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
*/

#include <unistd.h>
#include <stdio.h>
#include <stdlib.h>
#include <string.h>
#include <getopt.h>
#include <fcntl.h>
#include <limits.h>
#include <signal.h>
#include <sys/types.h>
#include <sys/stat.h>
#include <sys/select.h>
#include <sys/time.h>
#include <ctype.h>
#include <time.h>
#include <sys/select.h>
#include <errno.h>
#include <sys/types.h>
#include <sys/wait.h>
#include <assert.h>
#include <stdbool.h>

#define MAX_TC_NUM 1024
#define MAX_BUFFER (64*1024)
#define MAX_COMMENT 1024

#define INSTALLED_TESTS_DIR "/usr/lib/dbus/installed-tests/dbus/data"

enum {
	INIT_TEST,
	NEW_STDOUT,
	NEW_STDERR,
	RESULT_CODE,
	RESULT_SIGNAL,
	RESULT_ERROR,
	RESULT_TIMEOUT
};

struct test_result {
	bool is_positive;
	char comment[MAX_COMMENT];
	char result[MAX_COMMENT];
	char name[MAX_COMMENT];
};

struct test_case {
	const char* name;
	const char* description;
};

struct binary {
	const char* path;
	const char* name;
	struct test_case* test_cases;
	int timeout;

	char** (*prepare_args) (const struct binary* b, const char* test_name);
	void (*parse) (const struct binary* b, const char* test_name, char* buffer, int state_change, int state_option);
	int (*init)(void);
	int (*clean)(void);
};

char* get_test_id(char* dest, const struct binary* b, const char* test_name);
void add_test_result(const char* test_id, const char* result, const char* comment, int res);
enum {
	PIPE_READ,
	PIPE_WRITE,
};

void parse_binary_outputs(const struct binary* b, const char* test_name, char* buffer, int state_change, int state_option);
char** prepare_args_for_binary(const struct binary* b, const char* test_name);
char** prepare_args_for_dir_iter(const struct binary* b, const char* test_name);
int init_environment_vars();

static struct test_case desc_2[] = {
	{"", "Testing DBus ability to iterate over directory contents."}, {NULL, NULL}
};
static struct test_case desc_3[] = {
	{"", "Simple manual tcp check."}, {NULL, NULL}
};
static struct test_case desc_4[] = {
	{"", "Test for being disconnected by a corrupt message."}, {NULL, NULL}
};
static struct test_case desc_5[] = {
	{"", "Integration tests for the dbus-daemon."}, {NULL, NULL}
};
static struct test_case desc_6[] = {
	{"", "Checks DBus daemon eavesdropping ability."}, {NULL, NULL}
};
static struct test_case desc_7[] = {
	{"", "Tests passing various ammounts of fds(If supported on given platform)."}, {NULL, NULL}
};
static struct test_case desc_8[] = {
	{"", "Simple sanity-check for loopback through TCP and Unix sockets."}, {NULL, NULL}
};
static struct test_case desc_9[] = {
	{"", "Simple sanity-check for D-Bus message serialization."}, {NULL, NULL}
};
static struct test_case desc_10[] = {
	{"", "Integration tests for monitor-mode D-Bus connections."}, {NULL, NULL}
};
static struct test_case desc_11[] = {
	{"", "Test for _dbus_printf_string_upper_bound."}, {NULL, NULL}
};
static struct test_case desc_12[] = {
	{"", "Test for thread-safe reference-counting."}, {NULL, NULL}
};
static struct test_case desc_13[] = {
	{"", "Test for passing unmodified messages between connections."}, {NULL, NULL}
};
static struct test_case desc_14[] = {
	{"", "Unit tests for systemd activation."}, {NULL, NULL}
};
static struct test_case desc_15[] = {
	{"", "Test for shell commands."}, {NULL, NULL}
};
static struct test_case desc_16[] = {
	{"", "Test dor D-bus syntax validation."}, {NULL, NULL}
};
static struct test_case desc_17[] = {
	{"", "Manual test for syslog support."}, {NULL, NULL}
};
static struct test_case desc_18[] = {
	{"", "Integration tests for the dbus-daemon's uid-based hardening."}, {NULL, NULL}
};

/* This table is used to start binaries */
struct binary tests[] = {
/*path, name, TC_table, timeout in us, prepare_args_handler, parse_function_handler, init_handler, clean_handler*/
	{"/usr/lib/dbus-tests/test-suites/dbus-tests/manual-dir-iter",
		"manual-dir-iter", desc_2, 1000*1000, prepare_args_for_dir_iter, parse_binary_outputs, init_environment_vars, NULL},
	{"/usr/lib/dbus-tests/test-suites/dbus-tests/manual-tcp",
		"manual-tcp", desc_3, 1000*1000, prepare_args_for_binary, parse_binary_outputs, init_environment_vars, NULL},
	{"/usr/lib/dbus-tests/test-suites/dbus-tests/test-corrupt",
		"test-corrupt", desc_4, 10*1000*1000, prepare_args_for_binary, parse_binary_outputs, init_environment_vars, NULL},
	{"/usr/lib/dbus-tests/test-suites/dbus-tests/test-dbus-daemon",
		"test-dbus-daemon", desc_5, 15*1000*1000, prepare_args_for_binary, parse_binary_outputs, init_environment_vars, NULL},
	{"/usr/lib/dbus-tests/test-suites/dbus-tests/test-dbus-daemon-eavesdrop",
		"test-dbus-daemon-eavesdrop", desc_6, 1000*1000, prepare_args_for_binary, parse_binary_outputs, init_environment_vars, NULL},
	{"/usr/lib/dbus-tests/test-suites/dbus-tests/test-fdpass",
		"test-fdpass", desc_7, 3*1000*1000, prepare_args_for_binary, parse_binary_outputs, init_environment_vars, NULL},
	{"/usr/lib/dbus-tests/test-suites/dbus-tests/test-loopback",
		"test-loopback", desc_8, 1000*1000, prepare_args_for_binary, parse_binary_outputs, init_environment_vars, NULL},
	{"/usr/lib/dbus-tests/test-suites/dbus-tests/test-marshal",
		"test-marshal", desc_9, 1000*1000, prepare_args_for_binary, parse_binary_outputs, init_environment_vars, NULL},
	{"/usr/lib/dbus-tests/test-suites/dbus-tests/test-monitor",
		"test-monitor", desc_10, 3*1000*1000, prepare_args_for_binary, parse_binary_outputs, init_environment_vars, NULL},
	{"/usr/lib/dbus-tests/test-suites/dbus-tests/test-printf",
		"test-printf", desc_11, 2*1000*1000, prepare_args_for_binary, parse_binary_outputs, init_environment_vars, NULL},
	{"/usr/lib/dbus-tests/test-suites/dbus-tests/test-refs",
		"test-refs", desc_12, 90*1000*1000, prepare_args_for_binary, parse_binary_outputs, init_environment_vars, NULL},
	{"/usr/lib/dbus-tests/test-suites/dbus-tests/test-relay",
		"test-relay", desc_13, 6*1000*1000, prepare_args_for_binary, parse_binary_outputs, init_environment_vars, NULL},
	{"/usr/lib/dbus-tests/test-suites/dbus-tests/test-sd-activation",
		"test-sd-activation", desc_14, 90*1000*1000, prepare_args_for_binary, parse_binary_outputs, init_environment_vars, NULL},
	{"/usr/lib/dbus-tests/test-suites/dbus-tests/test-shell",
		"test-shell", desc_15, 1000*1000, prepare_args_for_binary, parse_binary_outputs, init_environment_vars, NULL},
	{"/usr/lib/dbus-tests/test-suites/dbus-tests/test-syntax",
		"test-syntax", desc_16, 1000*1000, prepare_args_for_binary, parse_binary_outputs, init_environment_vars, NULL},
	{"/usr/lib/dbus-tests/test-suites/dbus-tests/test-syslog",
		"test-syslog", desc_17, 1000*1000, prepare_args_for_binary, parse_binary_outputs, init_environment_vars, NULL},
	{"/usr/lib/dbus-tests/test-suites/dbus-tests/test-uid-permissions",
		"test-uid-permissions", desc_18, 1000*1000, prepare_args_for_binary, parse_binary_outputs, init_environment_vars, NULL}
};



static const char result_pattern[] = "[r][%0]%1";
static struct state {
	unsigned int i; // index of input buffer
	unsigned int j; // index in results[n]
	unsigned int n; //index in results buffer (0 or 1)
	char results[2][MAX_COMMENT];
} g_state;

static void sm_reset(void)
{
	memset(&g_state, 0, sizeof (g_state));
}

static int sm_update(char* buffer, int i)
{
	int l = strlen(buffer) + 1;
	while (i < l  && g_state.i < sizeof(result_pattern) - 1) {
		if (result_pattern[g_state.i] == '%') {
			g_state.n = result_pattern[g_state.i+1] - '0';
			if (g_state.n > 1) {
				sm_reset();
				i--;
			} else if (isalnum(buffer[i])) {
				g_state.results[g_state.n][g_state.j++] = buffer[i];
			} else if (buffer[i] == result_pattern[g_state.i+2] || buffer[i] == '\n') {
				g_state.results[g_state.n][g_state.j] = 0;
				g_state.i += 3;
				g_state.j = 0;
				if (g_state.n == 1)
					return i;
			} else {
				g_state.i = 0;
				g_state.j = 0;
			}
		} else if (result_pattern[g_state.i] == buffer[i]) {
			g_state.i++;
		} else {
			sm_reset();
		}
		i++;
	}

	if (g_state.i >= sizeof(result_pattern) - 1) {
		g_state.results[g_state.n][g_state.j] = 0;
		g_state.i += 3;
		g_state.j = 0;
		if (g_state.n == 1)
			return i;
	}

	return 0;
}

static const char* sm_get_result(int i)
{
	return g_state.results[i];
}

static char* args[3];
char** prepare_args_for_binary(const struct binary* b, const char* test_name)
{
	args[0] = (char*)b->name;
	if (!test_name[0])
		args[1] = NULL;
	else {
		args[1] = (char*)test_name;
		args[2] = NULL;
	}
	return args;
}

char** prepare_args_for_dir_iter(const struct binary* b, const char* test_name)
{
	static char* args_dir[2];
	args_dir[0] = (char*)b->name;
	args_dir[1] = INSTALLED_TESTS_DIR;
	return args_dir;
}

int init_environment_vars()
{
	return !(putenv("DBUS_TEST_DATA="INSTALLED_TESTS_DIR));
}

void parse_binary_outputs(const struct binary* b, const char* test_name, char* buffer, int state_change, int state_option)
{
	char test_id[MAX_COMMENT];

	switch(state_change) {
	case INIT_TEST:
		break;
	case NEW_STDOUT:
		buffer[state_option] = 0;
		get_test_id(test_id, b, test_name);
		fprintf(stderr, "[stdout][%s]%s\n",test_id, buffer);
		break;
	case NEW_STDERR:
		buffer[state_option] = 0;
		get_test_id(test_id, b, test_name);
		fprintf(stderr, "[stderr][%s]%s\n",test_id, buffer);
		break;
	case RESULT_CODE:
		get_test_id(test_id, b, test_name);
		if (state_option != 0)
			add_test_result(test_id, "FAIL", "", 0);
		else if (state_option == 77)
			add_test_result(test_id, "SKIP", "", 0);
		else
			add_test_result(test_id, "PASS", "", 1);
		break;
	case RESULT_SIGNAL:
		get_test_id(test_id, b, test_name);
		add_test_result(test_id, "FAIL", "Finished by SIGNAL", 0);
		break;
	case RESULT_TIMEOUT:
		get_test_id(test_id, b, test_name);
		add_test_result(test_id, "FAIL", "Test TIMEOUT", 0);
		break;
	}
}

static struct option long_options[] = {
	{"list",        no_argument,       0, 'l'},
	{"run",         required_argument, 0, 'r'},
	{"description", required_argument, 0, 'd'},
	{0,             0,                 0,  0 }
};

static int stdin_pipe[2];
static int stdout_pipe[2];
static int stderr_pipe[2];
static int gravedigger_pipe[2];
static char buffer[MAX_BUFFER];
static const char* requested_tc[MAX_TC_NUM];

char* get_test_id(char* dest, const struct binary* b, const char* test_name)
{
	int len = strlen(b->name);
	memcpy(dest, b->name, len);
	memcpy(dest + len, test_name, strlen(test_name)+1);
	return dest;
}

static void print_description(const char* name, const char* description)
{
	printf("%s;%s\n",name, description);
}

static void print_list(const char* test_name)
{
	unsigned int i;
	char full_name[MAX_COMMENT];
	for (i = 0;i < sizeof(tests)/sizeof(struct binary); i++) {
		int j = 0;
		int l = strlen(tests[i].name);
		memcpy(full_name, tests[i].name, l+1);
		if (test_name && strncmp(test_name, full_name, l) != 0)
			continue;

		while (tests[i].test_cases[j].name) {
			memcpy(full_name + l, tests[i].test_cases[j].name, strlen(tests[i].test_cases[j].name) + 1);
			if (!test_name || strcmp(full_name, test_name) == 0)
				print_description(full_name,tests[i].test_cases[j].description);
			j++;
		}
	}
}

static void stop_binary(const struct binary* b, pid_t pid, const char* test_name, int w_res)
{
	int status = 0;
	int res = 0;
	if (w_res == 0)
		res = waitpid(pid, &status, WNOHANG);
	else
		res = waitpid(pid, &status, 0);

	if (res == 0) {
		//timeouted
		kill(pid, SIGKILL);
		res = waitpid(pid, &status, WNOHANG);
		b->parse(b, test_name, buffer, RESULT_TIMEOUT, res);
	} else if (res < 0) {
		//errno check
		kill(pid, SIGKILL);
		res = waitpid(pid, &status, WNOHANG);
		b->parse(b, test_name, buffer, RESULT_ERROR, res);
	} else if (res > 0) {
		if (WIFEXITED(status)) {
			b->parse(b, test_name, buffer, RESULT_CODE, WEXITSTATUS(status));
		} else if (WIFSIGNALED(status)) {
			b->parse(b, test_name, buffer, RESULT_SIGNAL, WTERMSIG(status));
		} else if (WIFSTOPPED(status)) {
			b->parse(b, test_name, buffer, RESULT_SIGNAL, WSTOPSIG(status));
        } else if (WIFCONTINUED(status)) {
			kill(pid, SIGKILL);
			b->parse(b, test_name, buffer, RESULT_SIGNAL, -1);
		}
	}
}


static void parse_output_with_timeout(const struct binary* b, pid_t pid, const char* test_name)
{
	struct timeval tv;
	fd_set rfds;
	int nfds;
	int res;
	int w_res = 0;
	tv.tv_sec = b->timeout/(1000*1000);
	tv.tv_usec = (b->timeout-tv.tv_sec*1000*1000);
	while (1) {
		FD_ZERO(&rfds);
		if (stdout_pipe[PIPE_READ] > -1) {
			assert(stdout_pipe[PIPE_READ] > -1);
			assert(stdout_pipe[PIPE_READ] < 1024);
			FD_SET(stdout_pipe[PIPE_READ], &rfds);
		}
		if (stderr_pipe[PIPE_READ] > -1) {
			assert(stderr_pipe[PIPE_READ] > -1);
			assert(stderr_pipe[PIPE_READ] < 1024);
			FD_SET(stderr_pipe[PIPE_READ], &rfds);
		}
		FD_SET(gravedigger_pipe[PIPE_READ], &rfds);

		nfds = select(FD_SETSIZE, &rfds, NULL, NULL, &tv);
		if (nfds == -1) {
			if (errno != EINTR) {
				w_res = 0;
				break;
			}
		} else if (nfds > 0) {
			if (stdout_pipe[PIPE_READ] > -1 && FD_ISSET(stdout_pipe[PIPE_READ], &rfds)) {
				res = read(stdout_pipe[PIPE_READ], buffer, MAX_BUFFER-1);
				if (res == 0 || (res < 0 && errno != EINTR)) {
					close (stdout_pipe[PIPE_READ]);
					stdout_pipe[PIPE_READ] = -1;
					continue;
				} else if (res >=0) {
					b->parse(b, test_name, buffer, NEW_STDOUT, res);
				}
			}

			if (stderr_pipe[PIPE_READ] > -1 && FD_ISSET(stderr_pipe[PIPE_READ], &rfds)) {
				res = read(stderr_pipe[PIPE_READ], buffer, MAX_BUFFER-1);
				if (res == 0 || (res < 0 && errno != EINTR)) {
					close (stderr_pipe[PIPE_READ]);
					stderr_pipe[PIPE_READ] = -1;
					continue;
				}
				b->parse(b, test_name, buffer, NEW_STDERR, res);
			}

			if (FD_ISSET(gravedigger_pipe[PIPE_READ], &rfds)) {
				w_res = 1;
				break; //it has ended
			}
		} else {
			//timeout
			w_res = 0;
			break;
		}
	}
	stop_binary(b, pid, test_name, w_res);
}

static int create_child(const char* path, char* const arguments[])
{
	int child;
	int nResult;
	if (pipe(gravedigger_pipe) < 0) {
		perror("allocating pipe for gravedigger failed");
		goto error1;
	}

	if (pipe(stdin_pipe) < 0) {
		perror("allocating pipe for child input redirect failed");
		goto error1;
	}

	if (pipe(stdout_pipe) < 0) {
		perror("allocating pipe for child output redirect failed");
		goto error2;
	}

	if (pipe(stderr_pipe) < 0) {
		perror("allocating pipe for child output redirect failed");
		goto error3;
	}

	child = fork();
	if (!child) {
		char ld_path[512];
		sprintf(ld_path, "/usr/lib/dbus-tests/lib/libdbuspolicy-tests/:");
		// redirect stdin
		if (dup2(stdin_pipe[PIPE_READ], STDIN_FILENO) == -1) {
			perror("redirecting stdin failed");
			return -1;
		}

        // redirect stdout
		if (dup2(stdout_pipe[PIPE_WRITE], STDOUT_FILENO) == -1) {
			perror("redirecting stdout failed");
			return -1;
		}

        // redirect stderr
		if (dup2(stderr_pipe[PIPE_WRITE], STDERR_FILENO) == -1) {
			perror("redirecting stderr failed");
			return -1;
		}

        // all these are for use by parent only
		close(stdin_pipe[PIPE_READ]);
		close(stdin_pipe[PIPE_WRITE]);
		close(stdout_pipe[PIPE_READ]);
		close(stdout_pipe[PIPE_WRITE]);
		close(stderr_pipe[PIPE_READ]);
		close(stderr_pipe[PIPE_WRITE]);
		close(gravedigger_pipe[PIPE_READ]);

		char* ld_path_b = getenv("LD_LIBRARY_PATH");
		if (ld_path_b != NULL)
			memcpy(ld_path + strlen(ld_path), ld_path_b, strlen(ld_path_b)+1);
		setenv("LD_LIBRARY_PATH", ld_path, 1);
		// run child process image
		nResult = execv(path, arguments);

		// if we get here at all, an error occurred, but we are in the child
		// process, so just exit
		perror("exec of the child process  failed");
		exit(nResult);
	} else if (child > 0) {
		// parent continues here

		// close unused file descriptors, these are for child only
		close(stdin_pipe[PIPE_READ]);
		close(stdout_pipe[PIPE_WRITE]);
		close(stderr_pipe[PIPE_WRITE]);
		close(gravedigger_pipe[PIPE_WRITE]);
	} else {
		// failed to create child
		goto error4;
	}

	return child;

error4:
	close(stderr_pipe[PIPE_READ]);
	close(stderr_pipe[PIPE_WRITE]);
error3:
	close(stdout_pipe[PIPE_READ]);
	close(stdout_pipe[PIPE_WRITE]);
error2:
	close(stdin_pipe[PIPE_READ]);
	close(stdin_pipe[PIPE_WRITE]);
error1:
	return -1;
}

static void run_test(const struct binary* b, const char* test_name)
{
	int res = -1;
	char** arg;
	char test_id[MAX_COMMENT];

	assert(b);
	assert(b->name);
	assert(b->path);
	assert(test_name);

	arg = b->prepare_args(b, test_name);

	if (b->init)
		if (!b->init()) {
			add_test_result(get_test_id(test_id, b, test_name), "ERROR", "Cannot init test", 0);
			return;
		}

	res = create_child(b->path, arg);
	if (res > 0)
		parse_output_with_timeout(b, res, test_name);
	else
		add_test_result(get_test_id(test_id, b, test_name), "ERROR", "Cannot start test", 0);

	if (b->clean)
		b->clean();
}

static void parse_run_test(const char* tc) {
	unsigned int i = 0;
	for (i = 0;i < sizeof(tests)/sizeof(struct binary); i++) {
		int len = strlen(tests[i].name);
		if (strncmp(tc, tests[i].name, len) == 0) {
			if (tc[len] == '*' || tc[len] == '\0')
				run_test(&tests[i], "");
            else
				run_test(&tests[i], tc + len);
		}
	}
}

static int parse_option(int argc, char* argv[])
{
	int ch = 0;
	int c = 0;
	while ((ch = getopt_long(argc, argv, "lr:d:", long_options, NULL)) != -1) {
		switch (ch) {
		case 'l':
			print_list(NULL);
			return 1;
		case 'r':
			if (c >= MAX_TC_NUM - 1) //NULL at the end
				return 0;

			if (optarg)
				requested_tc[c++] = optarg;

			break;
		case 'd':
			print_list(optarg);
			return 1;
		}
	}
	return 0;
}

void add_test_result(const char* test_id, const char* result, const char* comment, int res)
{
	printf("%s;%s;%s\n", test_id, result, comment);
	fflush(stdout);
}

int main(int argc, char* argv[])
{
	unsigned int i;
	signal(SIGPIPE, SIG_IGN);
	if (parse_option(argc, argv))
		return 0;

	if (!requested_tc[0]) {
		for (i = 0;i < sizeof(tests)/sizeof(struct binary); i++)
		    run_test(&tests[i], "");
	} else {
		i = 0;
		while(requested_tc[i]) {
		    parse_run_test(requested_tc[i]);
			i++;
		}
	}

	return 0;
}