// Copyright 2014 The Kyua Authors.
// All rights reserved.
//
// Redistribution and use in source and binary forms, with or without
// modification, are permitted provided that the following conditions are
// met:
//
// * Redistributions of source code must retain the above copyright
// notice, this list of conditions and the following disclaimer.
// * Redistributions in binary form must reproduce the above copyright
// notice, this list of conditions and the following disclaimer in the
// documentation and/or other materials provided with the distribution.
// * Neither the name of Google Inc. nor the names of its contributors
// may be used to endorse or promote products derived from this software
// without specific prior written permission.
//
// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
// "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
// LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
// A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
// OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
// LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
// DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
// THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
#include "engine/scheduler.hpp"
extern "C" {
#include <sys/types.h>
#include <signal.h>
#include <unistd.h>
}
#include <cstdlib>
#include <fstream>
#include <iostream>
#include <string>
#include <atf-c++.hpp>
#include "engine/config.hpp"
#include "engine/exceptions.hpp"
#include "model/context.hpp"
#include "model/metadata.hpp"
#include "model/test_case.hpp"
#include "model/test_program.hpp"
#include "model/test_result.hpp"
#include "utils/config/tree.ipp"
#include "utils/datetime.hpp"
#include "utils/defs.hpp"
#include "utils/env.hpp"
#include "utils/format/containers.ipp"
#include "utils/format/macros.hpp"
#include "utils/fs/operations.hpp"
#include "utils/fs/path.hpp"
#include "utils/optional.ipp"
#include "utils/passwd.hpp"
#include "utils/process/status.hpp"
#include "utils/sanity.hpp"
#include "utils/stacktrace.hpp"
#include "utils/stream.hpp"
#include "utils/test_utils.ipp"
#include "utils/text/exceptions.hpp"
#include "utils/text/operations.ipp"
namespace config = utils::config;
namespace datetime = utils::datetime;
namespace fs = utils::fs;
namespace passwd = utils::passwd;
namespace process = utils::process;
namespace scheduler = engine::scheduler;
namespace text = utils::text;
using utils::none;
using utils::optional;
namespace {
/// Checks if a string starts with a prefix.
///
/// \param str The string to be tested.
/// \param prefix The prefix to look for.
///
/// \return True if the string is prefixed as specified.
static bool
starts_with(const std::string& str, const std::string& prefix)
{
return (str.length() >= prefix.length() &&
str.substr(0, prefix.length()) == prefix);
}
/// Strips a prefix from a string and converts the rest to an integer.
///
/// \param str The string to be tested.
/// \param prefix The prefix to strip from the string.
///
/// \return The part of the string after the prefix converted to an integer.
static int
suffix_to_int(const std::string& str, const std::string& prefix)
{
PRE(starts_with(str, prefix));
try {
return text::to_type< int >(str.substr(prefix.length()));
} catch (const text::value_error& error) {
std::cerr << F("Failed: %s\n") % error.what();
std::abort();
}
}
/// Mock interface definition for testing.
///
/// This scheduler interface does not execute external binaries. It is designed
/// to simulate the scheduler of various programs with different exit statuses.
class mock_interface : public scheduler::interface {
/// Executes the subprocess simulating an exec.
///
/// This is just a simple wrapper over _exit(2) because we cannot use
/// std::exit on exit from this mock interface. The reason is that we do
/// not want to invoke any destructors as otherwise we'd clear up the global
/// scheduler state by mistake. This wouldn't be a major problem if it
/// wasn't because doing so deletes on-disk files and we want to leave them
/// in place so that the parent process can test for them!
///
/// \param exit_code Exit code.
void
do_exit(const int exit_code) const UTILS_NORETURN
{
std::cout.flush();
std::cerr.flush();
::_exit(exit_code);
}
/// Executes a test case that creates various files and then fails.
void
exec_create_files_and_fail(void) const UTILS_NORETURN
{
std::cerr << "This should not be clobbered\n";
atf::utils::create_file("first file", "");
atf::utils::create_file("second-file", "");
fs::mkdir_p(fs::path("dir1/dir2"), 0755);
::kill(::getpid(), SIGTERM);
std::abort();
}
/// Executes a test case that deletes all files in the current directory.
///
/// This is intended to validate that the test runs in an empty directory,
/// separate from any control files that the scheduler may have created.
void
exec_delete_all(void) const UTILS_NORETURN
{
const int exit_code = ::system("rm *") == -1
? EXIT_FAILURE : EXIT_SUCCESS;
// Recreate our own cookie.
atf::utils::create_file("exec_test_was_called", "");
do_exit(exit_code);
}
/// Executes a test case that returns a specific exit code.
///
/// \param exit_code Exit status to terminate the program with.
void
exec_exit(const int exit_code) const UTILS_NORETURN
{
do_exit(exit_code);
}
/// Executes a test case that just fails.
void
exec_fail(void) const UTILS_NORETURN
{
std::cerr << "This should not be clobbered\n";
::kill(::getpid(), SIGTERM);
std::abort();
}
/// Executes a test case that prints all input parameters to the functor.
///
/// \param test_program The test program to execute.
/// \param test_case_name Name of the test case to invoke, which must be a
/// number.
/// \param vars User-provided variables to pass to the test program.
void
exec_print_params(const model::test_program& test_program,
const std::string& test_case_name,
const config::properties_map& vars) const
UTILS_NORETURN
{
std::cout << F("Test program: %s\n") % test_program.relative_path();
std::cout << F("Test case: %s\n") % test_case_name;
for (config::properties_map::const_iterator iter = vars.begin();
iter != vars.end(); ++iter) {
std::cout << F("%s=%s\n") % (*iter).first % (*iter).second;
}
std::cerr << F("stderr: %s\n") % test_case_name;
do_exit(EXIT_SUCCESS);
}
public:
/// Executes a test program's list operation.
///
/// This method is intended to be called within a subprocess and is expected
/// to terminate execution either by exec(2)ing the test program or by
/// exiting with a failure.
///
/// \param test_program The test program to execute.
/// \param vars User-provided variables to pass to the test program.
void
exec_list(const model::test_program& test_program,
const config::properties_map& vars)
const UTILS_NORETURN
{
const std::string name = test_program.absolute_path().leaf_name();
std::cerr << name;
std::cerr.flush();
if (name == "check_i_exist") {
if (fs::exists(test_program.absolute_path())) {
std::cout << "found\n";
do_exit(EXIT_SUCCESS);
} else {
std::cout << "not_found\n";
do_exit(EXIT_FAILURE);
}
} else if (name == "empty") {
do_exit(EXIT_SUCCESS);
} else if (name == "misbehave") {
utils::abort_without_coredump();
} else if (name == "timeout") {
std::cout << "sleeping\n";
std::cout.flush();
::sleep(100);
utils::abort_without_coredump();
} else if (name == "vars") {
for (config::properties_map::const_iterator iter = vars.begin();
iter != vars.end(); ++iter) {
std::cout << F("%s_%s\n") % (*iter).first % (*iter).second;
}
do_exit(15);
} else {
std::abort();
}
}
/// Computes the test cases list of a test program.
///
/// \param status The termination status of the subprocess used to execute
/// the exec_test() method or none if the test timed out.
/// \param stdout_path Path to the file containing the stdout of the test.
/// \param stderr_path Path to the file containing the stderr of the test.
///
/// \return A list of test cases.
model::test_cases_map
parse_list(const optional< process::status >& status,
const fs::path& stdout_path,
const fs::path& stderr_path) const
{
const std::string name = utils::read_file(stderr_path);
if (name == "check_i_exist") {
ATF_REQUIRE(status.get().exited());
ATF_REQUIRE_EQ(EXIT_SUCCESS, status.get().exitstatus());
} else if (name == "empty") {
ATF_REQUIRE(status.get().exited());
ATF_REQUIRE_EQ(EXIT_SUCCESS, status.get().exitstatus());
} else if (name == "misbehave") {
throw std::runtime_error("misbehaved in parse_list");
} else if (name == "timeout") {
ATF_REQUIRE(!status);
} else if (name == "vars") {
ATF_REQUIRE(status.get().exited());
ATF_REQUIRE_EQ(15, status.get().exitstatus());
} else {
ATF_FAIL("Invalid stderr contents; got " + name);
}
model::test_cases_map_builder test_cases_builder;
std::ifstream input(stdout_path.c_str());
ATF_REQUIRE(input);
std::string line;
while (std::getline(input, line).good()) {
test_cases_builder.add(line);
}
return test_cases_builder.build();
}
/// Executes a test case of the test program.
///
/// This method is intended to be called within a subprocess and is expected
/// to terminate execution either by exec(2)ing the test program or by
/// exiting with a failure.
///
/// \param test_program The test program to execute.
/// \param test_case_name Name of the test case to invoke.
/// \param vars User-provided variables to pass to the test program.
/// \param control_directory Directory where the interface may place control
/// files.
void
exec_test(const model::test_program& test_program,
const std::string& test_case_name,
const config::properties_map& vars,
const fs::path& control_directory) const
{
const fs::path cookie = control_directory / "exec_test_was_called";
std::ofstream control_file(cookie.c_str());
if (!control_file) {
std::cerr << "Failed to create " << cookie << '\n';
std::abort();
}
control_file << test_case_name;
control_file.close();
if (test_case_name == "check_i_exist") {
do_exit(fs::exists(test_program.absolute_path()) ? 0 : 1);
} else if (starts_with(test_case_name, "cleanup_timeout")) {
exec_exit(EXIT_SUCCESS);
} else if (starts_with(test_case_name, "create_files_and_fail")) {
exec_create_files_and_fail();
} else if (test_case_name == "delete_all") {
exec_delete_all();
} else if (starts_with(test_case_name, "exit ")) {
exec_exit(suffix_to_int(test_case_name, "exit "));
} else if (starts_with(test_case_name, "fail")) {
exec_fail();
} else if (starts_with(test_case_name, "fail_body_fail_cleanup")) {
exec_fail();
} else if (starts_with(test_case_name, "fail_body_pass_cleanup")) {
exec_fail();
} else if (starts_with(test_case_name, "pass_body_fail_cleanup")) {
exec_exit(EXIT_SUCCESS);
} else if (starts_with(test_case_name, "print_params")) {
exec_print_params(test_program, test_case_name, vars);
} else if (starts_with(test_case_name, "skip_body_pass_cleanup")) {
exec_exit(EXIT_SUCCESS);
} else {
std::cerr << "Unknown test case " << test_case_name << '\n';
std::abort();
}
}
/// Executes a test cleanup routine of the test program.
///
/// This method is intended to be called within a subprocess and is expected
/// to terminate execution either by exec(2)ing the test program or by
/// exiting with a failure.
///
/// \param test_case_name Name of the test case to invoke.
void
exec_cleanup(const model::test_program& /* test_program */,
const std::string& test_case_name,
const config::properties_map& /* vars */,
const fs::path& /* control_directory */) const
{
std::cout << "exec_cleanup was called\n";
std::cout.flush();
if (starts_with(test_case_name, "cleanup_timeout")) {
::sleep(100);
std::abort();
} else if (starts_with(test_case_name, "fail_body_fail_cleanup")) {
exec_fail();
} else if (starts_with(test_case_name, "fail_body_pass_cleanup")) {
exec_exit(EXIT_SUCCESS);
} else if (starts_with(test_case_name, "pass_body_fail_cleanup")) {
exec_fail();
} else if (starts_with(test_case_name, "skip_body_pass_cleanup")) {
exec_exit(EXIT_SUCCESS);
} else {
std::cerr << "Should not have been called for a test without "
"a cleanup routine" << '\n';
std::abort();
}
}
/// Computes the result of a test case based on its termination status.
///
/// \param status The termination status of the subprocess used to execute
/// the exec_test() method or none if the test timed out.
/// \param control_directory Path to the directory where the interface may
/// have placed control files.
/// \param stdout_path Path to the file containing the stdout of the test.
/// \param stderr_path Path to the file containing the stderr of the test.
///
/// \return A test result.
model::test_result
compute_result(const optional< process::status >& status,
const fs::path& control_directory,
const fs::path& stdout_path,
const fs::path& stderr_path) const
{
// Do not use any ATF_* macros here. Some of the tests below invoke
// this code in a subprocess, and terminating such subprocess due to a
// failed ATF_* macro yields mysterious failures that are incredibly
// hard to debug. (Case in point: the signal_handling test is racy by
// nature, and the test run by exec_test() above may not have created
// the cookie we expect below. We don't want to "silently" exit if the
// file is not there.)
if (!status) {
return model::test_result(model::test_result_broken,
"Timed out");
}
if (status.get().exited()) {
// Only sanity-check the work directory-related parameters in case
// of a clean exit. In all other cases, there is no guarantee that
// these were ever created.
const fs::path cookie = control_directory / "exec_test_was_called";
if (!atf::utils::file_exists(cookie.str())) {
return model::test_result(
model::test_result_broken,
"compute_result's control_directory does not seem to point "
"to the right location");
}
const std::string test_case_name = utils::read_file(cookie);
if (!atf::utils::file_exists(stdout_path.str())) {
return model::test_result(
model::test_result_broken,
"compute_result's stdout_path does not exist");
}
if (!atf::utils::file_exists(stderr_path.str())) {
return model::test_result(
model::test_result_broken,
"compute_result's stderr_path does not exist");
}
if (test_case_name == "skip_body_pass_cleanup") {
return model::test_result(
model::test_result_skipped,
F("Exit %s") % status.get().exitstatus());
} else {
return model::test_result(
model::test_result_passed,
F("Exit %s") % status.get().exitstatus());
}
} else {
return model::test_result(
model::test_result_failed,
F("Signal %s") % status.get().termsig());
}
}
};
} // anonymous namespace
/// Runs list_tests on the scheduler and returns the results.
///
/// \param test_name The name of the test supported by our exec_list function.
/// \param user_config Optional user settings for the test.
///
/// \return The loaded list of test cases.
static model::test_cases_map
check_integration_list(const char* test_name, const fs::path root,
const config::tree& user_config = engine::empty_config())
{
const model::test_program program = model::test_program_builder(
"mock", fs::path(test_name), root, "the-suite")
.build();
scheduler::scheduler_handle handle = scheduler::setup();
const model::test_cases_map test_cases = handle.list_tests(&program,
user_config);
handle.cleanup();
return test_cases;
}
ATF_TEST_CASE_WITHOUT_HEAD(integration__list_some);
ATF_TEST_CASE_BODY(integration__list_some)
{
config::tree user_config = engine::empty_config();
user_config.set_string("test_suites.the-suite.first", "test");
user_config.set_string("test_suites.the-suite.second", "TEST");
user_config.set_string("test_suites.abc.unused", "unused");
const model::test_cases_map test_cases = check_integration_list(
"vars", fs::path("."), user_config);
const model::test_cases_map exp_test_cases = model::test_cases_map_builder()
.add("first_test").add("second_TEST").build();
ATF_REQUIRE_EQ(exp_test_cases, test_cases);
}
ATF_TEST_CASE_WITHOUT_HEAD(integration__list_check_paths);
ATF_TEST_CASE_BODY(integration__list_check_paths)
{
fs::mkdir_p(fs::path("dir1/dir2/dir3"), 0755);
atf::utils::create_file("dir1/dir2/dir3/check_i_exist", "");
const model::test_cases_map test_cases = check_integration_list(
"dir2/dir3/check_i_exist", fs::path("dir1"));
const model::test_cases_map exp_test_cases = model::test_cases_map_builder()
.add("found").build();
ATF_REQUIRE_EQ(exp_test_cases, test_cases);
}
ATF_TEST_CASE_WITHOUT_HEAD(integration__list_timeout);
ATF_TEST_CASE_BODY(integration__list_timeout)
{
scheduler::list_timeout = datetime::delta(1, 0);
const model::test_cases_map test_cases = check_integration_list(
"timeout", fs::path("."));
const model::test_cases_map exp_test_cases = model::test_cases_map_builder()
.add("sleeping").build();
ATF_REQUIRE_EQ(exp_test_cases, test_cases);
}
ATF_TEST_CASE_WITHOUT_HEAD(integration__list_fail);
ATF_TEST_CASE_BODY(integration__list_fail)
{
const model::test_cases_map test_cases = check_integration_list(
"misbehave", fs::path("."));
ATF_REQUIRE_EQ(1, test_cases.size());
const model::test_case& test_case = test_cases.begin()->second;
ATF_REQUIRE_EQ("__test_cases_list__", test_case.name());
ATF_REQUIRE(test_case.fake_result());
ATF_REQUIRE_EQ(model::test_result(model::test_result_broken,
"misbehaved in parse_list"),
test_case.fake_result().get());
}
ATF_TEST_CASE_WITHOUT_HEAD(integration__list_empty);
ATF_TEST_CASE_BODY(integration__list_empty)
{
const model::test_cases_map test_cases = check_integration_list(
"empty", fs::path("."));
ATF_REQUIRE_EQ(1, test_cases.size());
const model::test_case& test_case = test_cases.begin()->second;
ATF_REQUIRE_EQ("__test_cases_list__", test_case.name());
ATF_REQUIRE(test_case.fake_result());
ATF_REQUIRE_EQ(model::test_result(model::test_result_broken,
"Empty test cases list"),
test_case.fake_result().get());
}
ATF_TEST_CASE_WITHOUT_HEAD(integration__run_one);
ATF_TEST_CASE_BODY(integration__run_one)
{
const model::test_program_ptr program = model::test_program_builder(
"mock", fs::path("the-program"), fs::current_path(), "the-suite")
.add_test_case("exit 41").build_ptr();
const config::tree user_config = engine::empty_config();
scheduler::scheduler_handle handle = scheduler::setup();
const scheduler::exec_handle exec_handle = handle.spawn_test(
program, "exit 41", user_config);
scheduler::result_handle_ptr result_handle = handle.wait_any();
const scheduler::test_result_handle* test_result_handle =
dynamic_cast< const scheduler::test_result_handle* >(
result_handle.get());
ATF_REQUIRE_EQ(exec_handle, result_handle->original_pid());
ATF_REQUIRE_EQ(model::test_result(model::test_result_passed, "Exit 41"),
test_result_handle->test_result());
result_handle->cleanup();
result_handle.reset();
handle.cleanup();
}
ATF_TEST_CASE_WITHOUT_HEAD(integration__run_many);
ATF_TEST_CASE_BODY(integration__run_many)
{
static const std::size_t num_test_programs = 30;
const config::tree user_config = engine::empty_config();
scheduler::scheduler_handle handle = scheduler::setup();
// We mess around with the "current time" below, so make sure the tests do
// not spuriously exceed their deadline by bumping it to a large number.
const model::metadata infinite_timeout = model::metadata_builder()
.set_timeout(datetime::delta(1000000L, 0)).build();
std::size_t total_tests = 0;
std::map< scheduler::exec_handle, model::test_program_ptr >
exp_test_programs;
std::map< scheduler::exec_handle, std::string > exp_test_case_names;
std::map< scheduler::exec_handle, datetime::timestamp > exp_start_times;
std::map< scheduler::exec_handle, int > exp_exit_statuses;
for (std::size_t i = 0; i < num_test_programs; ++i) {
const std::string test_case_0 = F("exit %s") % (i * 3 + 0);
const std::string test_case_1 = F("exit %s") % (i * 3 + 1);
const std::string test_case_2 = F("exit %s") % (i * 3 + 2);
const model::test_program_ptr program = model::test_program_builder(
"mock", fs::path(F("program-%s") % i),
fs::current_path(), "the-suite")
.set_metadata(infinite_timeout)
.add_test_case(test_case_0)
.add_test_case(test_case_1)
.add_test_case(test_case_2)
.build_ptr();
const datetime::timestamp start_time = datetime::timestamp::from_values(
2014, 12, 8, 9, 40, 0, i);
scheduler::exec_handle exec_handle;
datetime::set_mock_now(start_time);
exec_handle = handle.spawn_test(program, test_case_0, user_config);
exp_test_programs.insert(std::make_pair(exec_handle, program));
exp_test_case_names.insert(std::make_pair(exec_handle, test_case_0));
exp_start_times.insert(std::make_pair(exec_handle, start_time));
exp_exit_statuses.insert(std::make_pair(exec_handle, i * 3));
++total_tests;
datetime::set_mock_now(start_time);
exec_handle = handle.spawn_test(program, test_case_1, user_config);
exp_test_programs.insert(std::make_pair(exec_handle, program));
exp_test_case_names.insert(std::make_pair(exec_handle, test_case_1));
exp_start_times.insert(std::make_pair(exec_handle, start_time));
exp_exit_statuses.insert(std::make_pair(exec_handle, i * 3 + 1));
++total_tests;
datetime::set_mock_now(start_time);
exec_handle = handle.spawn_test(program, test_case_2, user_config);
exp_test_programs.insert(std::make_pair(exec_handle, program));
exp_test_case_names.insert(std::make_pair(exec_handle, test_case_2));
exp_start_times.insert(std::make_pair(exec_handle, start_time));
exp_exit_statuses.insert(std::make_pair(exec_handle, i * 3 + 2));
++total_tests;
}
for (std::size_t i = 0; i < total_tests; ++i) {
const datetime::timestamp end_time = datetime::timestamp::from_values(
2014, 12, 8, 9, 50, 10, i);
datetime::set_mock_now(end_time);
scheduler::result_handle_ptr result_handle = handle.wait_any();
const scheduler::test_result_handle* test_result_handle =
dynamic_cast< const scheduler::test_result_handle* >(
result_handle.get());
const scheduler::exec_handle exec_handle =
result_handle->original_pid();
const model::test_program_ptr test_program = exp_test_programs.find(
exec_handle)->second;
const std::string& test_case_name = exp_test_case_names.find(
exec_handle)->second;
const datetime::timestamp& start_time = exp_start_times.find(
exec_handle)->second;
const int exit_status = exp_exit_statuses.find(exec_handle)->second;
ATF_REQUIRE_EQ(model::test_result(model::test_result_passed,
F("Exit %s") % exit_status),
test_result_handle->test_result());
ATF_REQUIRE_EQ(test_program, test_result_handle->test_program());
ATF_REQUIRE_EQ(test_case_name, test_result_handle->test_case_name());
ATF_REQUIRE_EQ(start_time, result_handle->start_time());
ATF_REQUIRE_EQ(end_time, result_handle->end_time());
result_handle->cleanup();
ATF_REQUIRE(!atf::utils::file_exists(
result_handle->stdout_file().str()));
ATF_REQUIRE(!atf::utils::file_exists(
result_handle->stderr_file().str()));
ATF_REQUIRE(!atf::utils::file_exists(
result_handle->work_directory().str()));
result_handle.reset();
}
handle.cleanup();
}
ATF_TEST_CASE_WITHOUT_HEAD(integration__run_check_paths);
ATF_TEST_CASE_BODY(integration__run_check_paths)
{
fs::mkdir_p(fs::path("dir1/dir2/dir3"), 0755);
atf::utils::create_file("dir1/dir2/dir3/program", "");
const model::test_program_ptr program = model::test_program_builder(
"mock", fs::path("dir2/dir3/program"), fs::path("dir1"), "the-suite")
.add_test_case("check_i_exist").build_ptr();
scheduler::scheduler_handle handle = scheduler::setup();
(void)handle.spawn_test(program, "check_i_exist", engine::default_config());
scheduler::result_handle_ptr result_handle = handle.wait_any();
const scheduler::test_result_handle* test_result_handle =
dynamic_cast< const scheduler::test_result_handle* >(
result_handle.get());
ATF_REQUIRE_EQ(model::test_result(model::test_result_passed, "Exit 0"),
test_result_handle->test_result());
result_handle->cleanup();
result_handle.reset();
handle.cleanup();
}
ATF_TEST_CASE_WITHOUT_HEAD(integration__parameters_and_output);
ATF_TEST_CASE_BODY(integration__parameters_and_output)
{
const model::test_program_ptr program = model::test_program_builder(
"mock", fs::path("the-program"), fs::current_path(), "the-suite")
.add_test_case("print_params").build_ptr();
config::tree user_config = engine::empty_config();
user_config.set_string("test_suites.the-suite.one", "first variable");
user_config.set_string("test_suites.the-suite.two", "second variable");
scheduler::scheduler_handle handle = scheduler::setup();
const scheduler::exec_handle exec_handle = handle.spawn_test(
program, "print_params", user_config);
scheduler::result_handle_ptr result_handle = handle.wait_any();
const scheduler::test_result_handle* test_result_handle =
dynamic_cast< const scheduler::test_result_handle* >(
result_handle.get());
ATF_REQUIRE_EQ(exec_handle, result_handle->original_pid());
ATF_REQUIRE_EQ(program, test_result_handle->test_program());
ATF_REQUIRE_EQ("print_params", test_result_handle->test_case_name());
ATF_REQUIRE_EQ(model::test_result(model::test_result_passed, "Exit 0"),
test_result_handle->test_result());
const fs::path stdout_file = result_handle->stdout_file();
ATF_REQUIRE(atf::utils::compare_file(
stdout_file.str(),
"Test program: the-program\n"
"Test case: print_params\n"
"one=first variable\n"
"two=second variable\n"));
const fs::path stderr_file = result_handle->stderr_file();
ATF_REQUIRE(atf::utils::compare_file(
stderr_file.str(), "stderr: print_params\n"));
result_handle->cleanup();
ATF_REQUIRE(!fs::exists(stdout_file));
ATF_REQUIRE(!fs::exists(stderr_file));
result_handle.reset();
handle.cleanup();
}
ATF_TEST_CASE_WITHOUT_HEAD(integration__fake_result);
ATF_TEST_CASE_BODY(integration__fake_result)
{
const model::test_result fake_result(model::test_result_skipped,
"Some fake details");
model::test_cases_map test_cases;
test_cases.insert(model::test_cases_map::value_type(
"__fake__", model::test_case("__fake__", "ABC", fake_result)));
const model::test_program_ptr program(new model::test_program(
"mock", fs::path("the-program"), fs::current_path(), "the-suite",
model::metadata_builder().build(), test_cases));
const config::tree user_config = engine::empty_config();
scheduler::scheduler_handle handle = scheduler::setup();
(void)handle.spawn_test(program, "__fake__", user_config);
scheduler::result_handle_ptr result_handle = handle.wait_any();
const scheduler::test_result_handle* test_result_handle =
dynamic_cast< const scheduler::test_result_handle* >(
result_handle.get());
ATF_REQUIRE_EQ(fake_result, test_result_handle->test_result());
result_handle->cleanup();
result_handle.reset();
handle.cleanup();
}
ATF_TEST_CASE_WITHOUT_HEAD(integration__cleanup__head_skips);
ATF_TEST_CASE_BODY(integration__cleanup__head_skips)
{
const model::test_program_ptr program = model::test_program_builder(
"mock", fs::path("the-program"), fs::current_path(), "the-suite")
.add_test_case("skip_me",
model::metadata_builder()
.add_required_config("variable-that-does-not-exist")
.set_has_cleanup(true)
.build())
.build_ptr();
const config::tree user_config = engine::empty_config();
scheduler::scheduler_handle handle = scheduler::setup();
(void)handle.spawn_test(program, "skip_me", user_config);
scheduler::result_handle_ptr result_handle = handle.wait_any();
const scheduler::test_result_handle* test_result_handle =
dynamic_cast< const scheduler::test_result_handle* >(
result_handle.get());
ATF_REQUIRE_EQ(model::test_result(
model::test_result_skipped,
"Required configuration property "
"'variable-that-does-not-exist' not defined"),
test_result_handle->test_result());
ATF_REQUIRE(!atf::utils::grep_file("exec_cleanup was called",
result_handle->stdout_file().str()));
result_handle->cleanup();
result_handle.reset();
handle.cleanup();
}
/// Runs a test to verify the behavior of cleanup routines.
///
/// \param test_case The name of the test case to invoke.
/// \param exp_result The expected test result of the execution.
static void
do_cleanup_test(const char* test_case,
const model::test_result& exp_result)
{
const model::test_program_ptr program = model::test_program_builder(
"mock", fs::path("the-program"), fs::current_path(), "the-suite")
.add_test_case(test_case)
.set_metadata(model::metadata_builder().set_has_cleanup(true).build())
.build_ptr();
const config::tree user_config = engine::empty_config();
scheduler::scheduler_handle handle = scheduler::setup();
(void)handle.spawn_test(program, test_case, user_config);
scheduler::result_handle_ptr result_handle = handle.wait_any();
const scheduler::test_result_handle* test_result_handle =
dynamic_cast< const scheduler::test_result_handle* >(
result_handle.get());
ATF_REQUIRE_EQ(exp_result, test_result_handle->test_result());
ATF_REQUIRE(atf::utils::compare_file(
result_handle->stdout_file().str(),
"exec_cleanup was called\n"));
result_handle->cleanup();
result_handle.reset();
handle.cleanup();
}
ATF_TEST_CASE_WITHOUT_HEAD(integration__cleanup__body_skips);
ATF_TEST_CASE_BODY(integration__cleanup__body_skips)
{
do_cleanup_test(
"skip_body_pass_cleanup",
model::test_result(model::test_result_skipped, "Exit 0"));
}
ATF_TEST_CASE_WITHOUT_HEAD(integration__cleanup__body_bad__cleanup_ok);
ATF_TEST_CASE_BODY(integration__cleanup__body_bad__cleanup_ok)
{
do_cleanup_test(
"fail_body_pass_cleanup",
model::test_result(model::test_result_failed, "Signal 15"));
}
ATF_TEST_CASE_WITHOUT_HEAD(integration__cleanup__body_ok__cleanup_bad);
ATF_TEST_CASE_BODY(integration__cleanup__body_ok__cleanup_bad)
{
do_cleanup_test(
"pass_body_fail_cleanup",
model::test_result(model::test_result_broken, "Test case cleanup "
"did not terminate successfully"));
}
ATF_TEST_CASE_WITHOUT_HEAD(integration__cleanup__body_bad__cleanup_bad);
ATF_TEST_CASE_BODY(integration__cleanup__body_bad__cleanup_bad)
{
do_cleanup_test(
"fail_body_fail_cleanup",
model::test_result(model::test_result_failed, "Signal 15"));
}
ATF_TEST_CASE_WITHOUT_HEAD(integration__cleanup__timeout);
ATF_TEST_CASE_BODY(integration__cleanup__timeout)
{
scheduler::cleanup_timeout = datetime::delta(1, 0);
do_cleanup_test(
"cleanup_timeout",
model::test_result(model::test_result_broken, "Test case cleanup "
"timed out"));
}
ATF_TEST_CASE_WITHOUT_HEAD(integration__check_requirements);
ATF_TEST_CASE_BODY(integration__check_requirements)
{
const model::test_program_ptr program = model::test_program_builder(
"mock", fs::path("the-program"), fs::current_path(), "the-suite")
.add_test_case("exit 12")
.set_metadata(model::metadata_builder()
.add_required_config("abcde").build())
.build_ptr();
const config::tree user_config = engine::empty_config();
scheduler::scheduler_handle handle = scheduler::setup();
(void)handle.spawn_test(program, "exit 12", user_config);
scheduler::result_handle_ptr result_handle = handle.wait_any();
const scheduler::test_result_handle* test_result_handle =
dynamic_cast< const scheduler::test_result_handle* >(
result_handle.get());
ATF_REQUIRE_EQ(model::test_result(
model::test_result_skipped,
"Required configuration property 'abcde' not defined"),
test_result_handle->test_result());
result_handle->cleanup();
result_handle.reset();
handle.cleanup();
}
ATF_TEST_CASE_WITHOUT_HEAD(integration__stacktrace);
ATF_TEST_CASE_BODY(integration__stacktrace)
{
utils::prepare_coredump_test(this);
const model::test_program_ptr program = model::test_program_builder(
"mock", fs::path("the-program"), fs::current_path(), "the-suite")
.add_test_case("unknown-dumps-core").build_ptr();
const config::tree user_config = engine::empty_config();
scheduler::scheduler_handle handle = scheduler::setup();
(void)handle.spawn_test(program, "unknown-dumps-core", user_config);
scheduler::result_handle_ptr result_handle = handle.wait_any();
const scheduler::test_result_handle* test_result_handle =
dynamic_cast< const scheduler::test_result_handle* >(
result_handle.get());
ATF_REQUIRE_EQ(model::test_result(model::test_result_failed,
F("Signal %s") % SIGABRT),
test_result_handle->test_result());
ATF_REQUIRE(!atf::utils::grep_file("attempting to gather stack trace",
result_handle->stdout_file().str()));
ATF_REQUIRE( atf::utils::grep_file("attempting to gather stack trace",
result_handle->stderr_file().str()));
result_handle->cleanup();
result_handle.reset();
handle.cleanup();
}
/// Runs a test to verify the dumping of the list of existing files on failure.
///
/// \param test_case The name of the test case to invoke.
/// \param exp_stderr Expected contents of stderr.
static void
do_check_list_files_on_failure(const char* test_case, const char* exp_stderr)
{
const model::test_program_ptr program = model::test_program_builder(
"mock", fs::path("the-program"), fs::current_path(), "the-suite")
.add_test_case(test_case).build_ptr();
const config::tree user_config = engine::empty_config();
scheduler::scheduler_handle handle = scheduler::setup();
(void)handle.spawn_test(program, test_case, user_config);
scheduler::result_handle_ptr result_handle = handle.wait_any();
atf::utils::cat_file(result_handle->stdout_file().str(), "child stdout: ");
ATF_REQUIRE(atf::utils::compare_file(result_handle->stdout_file().str(),
""));
atf::utils::cat_file(result_handle->stderr_file().str(), "child stderr: ");
ATF_REQUIRE(atf::utils::compare_file(result_handle->stderr_file().str(),
exp_stderr));
result_handle->cleanup();
result_handle.reset();
handle.cleanup();
}
ATF_TEST_CASE_WITHOUT_HEAD(integration__list_files_on_failure__none);
ATF_TEST_CASE_BODY(integration__list_files_on_failure__none)
{
do_check_list_files_on_failure("fail", "This should not be clobbered\n");
}
ATF_TEST_CASE_WITHOUT_HEAD(integration__list_files_on_failure__some);
ATF_TEST_CASE_BODY(integration__list_files_on_failure__some)
{
do_check_list_files_on_failure(
"create_files_and_fail",
"This should not be clobbered\n"
"Files left in work directory after failure: "
"dir1, first file, second-file\n");
}
ATF_TEST_CASE_WITHOUT_HEAD(integration__prevent_clobbering_control_files);
ATF_TEST_CASE_BODY(integration__prevent_clobbering_control_files)
{
const model::test_program_ptr program = model::test_program_builder(
"mock", fs::path("the-program"), fs::current_path(), "the-suite")
.add_test_case("delete_all").build_ptr();
const config::tree user_config = engine::empty_config();
scheduler::scheduler_handle handle = scheduler::setup();
(void)handle.spawn_test(program, "delete_all", user_config);
scheduler::result_handle_ptr result_handle = handle.wait_any();
const scheduler::test_result_handle* test_result_handle =
dynamic_cast< const scheduler::test_result_handle* >(
result_handle.get());
ATF_REQUIRE_EQ(model::test_result(model::test_result_passed, "Exit 0"),
test_result_handle->test_result());
result_handle->cleanup();
result_handle.reset();
handle.cleanup();
}
ATF_TEST_CASE_WITHOUT_HEAD(debug_test);
ATF_TEST_CASE_BODY(debug_test)
{
const model::test_program_ptr program = model::test_program_builder(
"mock", fs::path("the-program"), fs::current_path(), "the-suite")
.add_test_case("print_params").build_ptr();
config::tree user_config = engine::empty_config();
user_config.set_string("test_suites.the-suite.one", "first variable");
user_config.set_string("test_suites.the-suite.two", "second variable");
scheduler::scheduler_handle handle = scheduler::setup();
const fs::path stdout_file("custom-stdout.txt");
const fs::path stderr_file("custom-stderr.txt");
scheduler::result_handle_ptr result_handle = handle.debug_test(
program, "print_params", user_config, stdout_file, stderr_file);
const scheduler::test_result_handle* test_result_handle =
dynamic_cast< const scheduler::test_result_handle* >(
result_handle.get());
ATF_REQUIRE_EQ(program, test_result_handle->test_program());
ATF_REQUIRE_EQ("print_params", test_result_handle->test_case_name());
ATF_REQUIRE_EQ(model::test_result(model::test_result_passed, "Exit 0"),
test_result_handle->test_result());
// The original output went to a file. It's only an artifact of
// debug_test() that we later get a copy in our own files.
ATF_REQUIRE(stdout_file != result_handle->stdout_file());
ATF_REQUIRE(stderr_file != result_handle->stderr_file());
result_handle->cleanup();
result_handle.reset();
handle.cleanup();
ATF_REQUIRE(atf::utils::compare_file(
stdout_file.str(),
"Test program: the-program\n"
"Test case: print_params\n"
"one=first variable\n"
"two=second variable\n"));
ATF_REQUIRE(atf::utils::compare_file(
stderr_file.str(), "stderr: print_params\n"));
}
ATF_TEST_CASE_WITHOUT_HEAD(ensure_valid_interface);
ATF_TEST_CASE_BODY(ensure_valid_interface)
{
scheduler::ensure_valid_interface("mock");
ATF_REQUIRE_THROW_RE(engine::error, "Unsupported test interface 'mock2'",
scheduler::ensure_valid_interface("mock2"));
scheduler::register_interface(
"mock2", std::shared_ptr< scheduler::interface >(new mock_interface()));
scheduler::ensure_valid_interface("mock2");
// Standard interfaces should not be present unless registered.
ATF_REQUIRE_THROW_RE(engine::error, "Unsupported test interface 'plain'",
scheduler::ensure_valid_interface("plain"));
}
ATF_TEST_CASE_WITHOUT_HEAD(registered_interface_names);
ATF_TEST_CASE_BODY(registered_interface_names)
{
std::set< std::string > exp_names;
exp_names.insert("mock");
ATF_REQUIRE_EQ(exp_names, scheduler::registered_interface_names());
scheduler::register_interface(
"mock2", std::shared_ptr< scheduler::interface >(new mock_interface()));
exp_names.insert("mock2");
ATF_REQUIRE_EQ(exp_names, scheduler::registered_interface_names());
}
ATF_TEST_CASE_WITHOUT_HEAD(current_context);
ATF_TEST_CASE_BODY(current_context)
{
const model::context context = scheduler::current_context();
ATF_REQUIRE_EQ(fs::current_path(), context.cwd());
ATF_REQUIRE(utils::getallenv() == context.env());
}
ATF_TEST_CASE_WITHOUT_HEAD(generate_config__empty);
ATF_TEST_CASE_BODY(generate_config__empty)
{
const config::tree user_config = engine::empty_config();
const config::properties_map exp_props;
ATF_REQUIRE_EQ(exp_props,
scheduler::generate_config(user_config, "missing"));
}
ATF_TEST_CASE_WITHOUT_HEAD(generate_config__no_matches);
ATF_TEST_CASE_BODY(generate_config__no_matches)
{
config::tree user_config = engine::empty_config();
user_config.set_string("architecture", "foo");
user_config.set_string("test_suites.one.var1", "value 1");
const config::properties_map exp_props;
ATF_REQUIRE_EQ(exp_props,
scheduler::generate_config(user_config, "two"));
}
ATF_TEST_CASE_WITHOUT_HEAD(generate_config__some_matches);
ATF_TEST_CASE_BODY(generate_config__some_matches)
{
std::vector< passwd::user > mock_users;
mock_users.push_back(passwd::user("nobody", 1234, 5678));
passwd::set_mock_users_for_testing(mock_users);
config::tree user_config = engine::empty_config();
user_config.set_string("architecture", "foo");
user_config.set_string("unprivileged_user", "nobody");
user_config.set_string("test_suites.one.var1", "value 1");
user_config.set_string("test_suites.two.var2", "value 2");
config::properties_map exp_props;
exp_props["unprivileged-user"] = "nobody";
exp_props["var1"] = "value 1";
ATF_REQUIRE_EQ(exp_props,
scheduler::generate_config(user_config, "one"));
}
ATF_INIT_TEST_CASES(tcs)
{
scheduler::register_interface(
"mock", std::shared_ptr< scheduler::interface >(new mock_interface()));
ATF_ADD_TEST_CASE(tcs, integration__list_some);
ATF_ADD_TEST_CASE(tcs, integration__list_check_paths);
ATF_ADD_TEST_CASE(tcs, integration__list_timeout);
ATF_ADD_TEST_CASE(tcs, integration__list_fail);
ATF_ADD_TEST_CASE(tcs, integration__list_empty);
ATF_ADD_TEST_CASE(tcs, integration__run_one);
ATF_ADD_TEST_CASE(tcs, integration__run_many);
ATF_ADD_TEST_CASE(tcs, integration__run_check_paths);
ATF_ADD_TEST_CASE(tcs, integration__parameters_and_output);
ATF_ADD_TEST_CASE(tcs, integration__fake_result);
ATF_ADD_TEST_CASE(tcs, integration__cleanup__head_skips);
ATF_ADD_TEST_CASE(tcs, integration__cleanup__body_skips);
ATF_ADD_TEST_CASE(tcs, integration__cleanup__body_ok__cleanup_bad);
ATF_ADD_TEST_CASE(tcs, integration__cleanup__body_bad__cleanup_ok);
ATF_ADD_TEST_CASE(tcs, integration__cleanup__body_bad__cleanup_bad);
ATF_ADD_TEST_CASE(tcs, integration__cleanup__timeout);
ATF_ADD_TEST_CASE(tcs, integration__check_requirements);
ATF_ADD_TEST_CASE(tcs, integration__stacktrace);
ATF_ADD_TEST_CASE(tcs, integration__list_files_on_failure__none);
ATF_ADD_TEST_CASE(tcs, integration__list_files_on_failure__some);
ATF_ADD_TEST_CASE(tcs, integration__prevent_clobbering_control_files);
ATF_ADD_TEST_CASE(tcs, debug_test);
ATF_ADD_TEST_CASE(tcs, ensure_valid_interface);
ATF_ADD_TEST_CASE(tcs, registered_interface_names);
ATF_ADD_TEST_CASE(tcs, current_context);
ATF_ADD_TEST_CASE(tcs, generate_config__empty);
ATF_ADD_TEST_CASE(tcs, generate_config__no_matches);
ATF_ADD_TEST_CASE(tcs, generate_config__some_matches);
}