//
// Automated Testing Framework (atf)
//
// Copyright (c) 2007 The NetBSD Foundation, Inc.
// All rights reserved.
//
// Redistribution and use in source and binary forms, with or without
// modification, are permitted provided that the following conditions
// are met:
// 1. Redistributions of source code must retain the above copyright
// notice, this list of conditions and the following disclaimer.
// 2. Redistributions in binary form must reproduce the above copyright
// notice, this list of conditions and the following disclaimer in the
// documentation and/or other materials provided with the distribution.
//
// THIS SOFTWARE IS PROVIDED BY THE NETBSD FOUNDATION, INC. AND
// CONTRIBUTORS ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES,
// INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF
// MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED.
// IN NO EVENT SHALL THE FOUNDATION OR CONTRIBUTORS BE LIABLE FOR ANY
// DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
// DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE
// GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
// INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER
// IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR
// OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN
// IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
//
extern "C" {
#include <sys/types.h>
#include <sys/param.h>
#include <sys/stat.h>
#include <sys/wait.h>
#include <unistd.h>
}
#include <algorithm>
#include <cassert>
#include <cerrno>
#include <cstdlib>
#include <cstring>
#include <fstream>
#include <iostream>
#include <map>
#include <string>
#include "application.hpp"
#include "atffile.hpp"
#include "config.hpp"
#include "config_file.hpp"
#include "env.hpp"
#include "exceptions.hpp"
#include "fs.hpp"
#include "parser.hpp"
#include "process.hpp"
#include "requirements.hpp"
#include "test-program.hpp"
#include "text.hpp"
namespace {
typedef std::map< std::string, std::string > vars_map;
} // anonymous namespace
class atf_run : public tools::application::app {
static const char* m_description;
vars_map m_cmdline_vars;
static vars_map::value_type parse_var(const std::string&);
void process_option(int, const char*);
std::string specific_args(void) const;
options_set specific_options(void) const;
void parse_vflag(const std::string&);
std::vector< std::string > conf_args(void) const;
size_t count_tps(std::vector< std::string >) const;
int run_test(const tools::fs::path&, const std::string &,
tools::test_program::atf_tps_writer&,
const vars_map&);
int run_test_directory(const tools::fs::path&,
tools::test_program::atf_tps_writer&);
int run_test_program(const tools::fs::path&,
const std::string tc,
tools::test_program::atf_tps_writer&,
const vars_map&);
tools::test_program::test_case_result get_test_case_result(
const std::string&, const tools::process::status&,
const tools::fs::path&) const;
public:
atf_run(void);
int main(void);
};
static void
sanitize_gdb_env(void)
{
try {
tools::env::unset("TERM");
} catch (...) {
// Just swallow exceptions here; they cannot propagate into C, which
// is where this function is called from, and even if these exceptions
// appear they are benign.
}
}
static void
dump_stacktrace(const tools::fs::path& tp, const tools::process::status& s,
const tools::fs::path& workdir,
tools::test_program::atf_tps_writer& w)
{
assert(s.signaled() && s.coredump());
w.stderr_tc("Test program crashed; attempting to get stack trace");
const tools::fs::path corename = workdir /
(tp.leaf_name().substr(0, MAXCOMLEN) + ".core");
if (!tools::fs::exists(corename)) {
w.stderr_tc("Expected file " + corename.str() + " not found");
return;
}
const tools::fs::path gdb(GDB);
const tools::fs::path gdbout = workdir / "gdb.out";
const tools::process::argv_array args(gdb.leaf_name().c_str(), "-batch",
"-q", "-ex", "bt", tp.c_str(),
corename.c_str(), NULL);
tools::process::status status = tools::process::exec(
gdb, args,
tools::process::stream_redirect_path(gdbout),
tools::process::stream_redirect_path(tools::fs::path("/dev/null")),
sanitize_gdb_env);
if (!status.exited() || status.exitstatus() != EXIT_SUCCESS) {
w.stderr_tc("Execution of " GDB " failed");
return;
}
std::ifstream input(gdbout.c_str());
if (input) {
std::string line;
while (std::getline(input, line).good())
w.stderr_tc(line);
input.close();
}
w.stderr_tc("Stack trace complete");
}
const char* atf_run::m_description =
"atf-run is a tool that runs tests programs and collects their "
"results.";
atf_run::atf_run(void) :
app(m_description, "atf-run(1)", "atf(7)")
{
}
void
atf_run::process_option(int ch, const char* arg)
{
switch (ch) {
case 'v':
parse_vflag(arg);
break;
default:
std::abort();
}
}
std::string
atf_run::specific_args(void)
const
{
return "[test1 .. testN]";
}
atf_run::options_set
atf_run::specific_options(void)
const
{
using tools::application::option;
options_set opts;
opts.insert(option('v', "var=value", "Sets the configuration variable "
"`var' to `value'; overrides "
"values in configuration files"));
return opts;
}
void
atf_run::parse_vflag(const std::string& str)
{
if (str.empty())
throw std::runtime_error("-v requires a non-empty argument");
std::vector< std::string > ws = tools::text::split(str, "=");
if (ws.size() == 1 && str[str.length() - 1] == '=') {
m_cmdline_vars[ws[0]] = "";
} else {
if (ws.size() != 2)
throw std::runtime_error("-v requires an argument of the form "
"var=value");
m_cmdline_vars[ws[0]] = ws[1];
}
}
int
atf_run::run_test(const tools::fs::path& tp,
const std::string &tc,
tools::test_program::atf_tps_writer& w,
const vars_map& config)
{
tools::fs::file_info fi(tp);
int errcode;
if (fi.get_type() == tools::fs::file_info::dir_type)
errcode = run_test_directory(tp, w);
else {
const vars_map effective_config =
tools::config_file::merge_configs(config, m_cmdline_vars);
errcode = run_test_program(tp, tc, w, effective_config);
}
return errcode;
}
int
atf_run::run_test_directory(const tools::fs::path& tp,
tools::test_program::atf_tps_writer& w)
{
tools::atffile af = tools::read_atffile(tp / "Atffile");
vars_map test_suite_vars;
{
vars_map::const_iterator iter = af.props().find("test-suite");
assert(iter != af.props().end());
test_suite_vars = tools::config_file::read_config_files((*iter).second);
}
bool ok = true;
for (std::vector< std::string >::const_iterator iter = af.tps().begin();
iter != af.tps().end(); iter++) {
const bool result = run_test(tp / *iter, "", w,
tools::config_file::merge_configs(af.conf(), test_suite_vars));
ok &= (result == EXIT_SUCCESS);
}
return ok ? EXIT_SUCCESS : EXIT_FAILURE;
}
tools::test_program::test_case_result
atf_run::get_test_case_result(const std::string& broken_reason,
const tools::process::status& s,
const tools::fs::path& resfile)
const
{
using tools::text::to_string;
using tools::test_program::read_test_case_result;
using tools::test_program::test_case_result;
if (!broken_reason.empty()) {
test_case_result tcr;
try {
tcr = read_test_case_result(resfile);
if (tcr.state() == "expected_timeout") {
return tcr;
} else {
return test_case_result("failed", -1, broken_reason);
}
} catch (const std::runtime_error&) {
return test_case_result("failed", -1, broken_reason);
}
}
if (s.exited()) {
test_case_result tcr;
try {
tcr = read_test_case_result(resfile);
} catch (const std::runtime_error& e) {
return test_case_result("failed", -1, "Test case exited "
"normally but failed to create the results file: " +
std::string(e.what()));
}
if (tcr.state() == "expected_death") {
return tcr;
} else if (tcr.state() == "expected_exit") {
if (tcr.value() == -1 || s.exitstatus() == tcr.value())
return tcr;
else
return test_case_result("failed", -1, "Test case was "
"expected to exit with a " + to_string(tcr.value()) +
" error code but returned " + to_string(s.exitstatus()));
} else if (tcr.state() == "expected_failure") {
if (s.exitstatus() == EXIT_SUCCESS)
return tcr;
else
return test_case_result("failed", -1, "Test case returned an "
"error in expected_failure mode but it should not have");
} else if (tcr.state() == "expected_signal") {
return test_case_result("failed", -1, "Test case exited cleanly "
"but was expected to receive a signal");
} else if (tcr.state() == "failed") {
if (s.exitstatus() == EXIT_SUCCESS)
return test_case_result("failed", -1, "Test case "
"exited successfully but reported failure");
else
return tcr;
} else if (tcr.state() == "passed") {
if (s.exitstatus() == EXIT_SUCCESS)
return tcr;
else
return test_case_result("failed", -1, "Test case exited as "
"passed but reported an error");
} else if (tcr.state() == "skipped") {
if (s.exitstatus() == EXIT_SUCCESS)
return tcr;
else
return test_case_result("failed", -1, "Test case exited as "
"skipped but reported an error");
}
} else if (s.signaled()) {
test_case_result tcr;
try {
tcr = read_test_case_result(resfile);
} catch (const std::runtime_error&) {
return test_case_result("failed", -1, "Test program received "
"signal " + tools::text::to_string(s.termsig()) +
(s.coredump() ? " (core dumped)" : ""));
}
if (tcr.state() == "expected_death") {
return tcr;
} else if (tcr.state() == "expected_signal") {
if (tcr.value() == -1 || s.termsig() == tcr.value())
return tcr;
else
return test_case_result("failed", -1, "Test case was "
"expected to exit due to a " + to_string(tcr.value()) +
" signal but got " + to_string(s.termsig()));
} else {
return test_case_result("failed", -1, "Test program received "
"signal " + tools::text::to_string(s.termsig()) +
(s.coredump() ? " (core dumped)" : "") + " and created a "
"bogus results file");
}
}
std::abort();
return test_case_result();
}
int
atf_run::run_test_program(const tools::fs::path& tp,
const std::string tc,
tools::test_program::atf_tps_writer& w,
const vars_map& config)
{
int errcode = EXIT_SUCCESS;
tools::test_program::metadata md;
try {
md = tools::test_program::get_metadata(tp, config);
} catch (const tools::parser::format_error& e) {
w.start_tp(tp.str(), 0);
w.end_tp("Invalid format for test case list: " + std::string(e.what()));
return EXIT_FAILURE;
} catch (const tools::parser::parse_errors& e) {
const std::string reason = tools::text::join(e, "; ");
w.start_tp(tp.str(), 0);
w.end_tp("Invalid format for test case list: " + reason);
return EXIT_FAILURE;
}
tools::fs::temp_dir resdir(
tools::fs::path(tools::config::get("atf_workdir")) / "atf-run.XXXXXX");
size_t nseltcs;
if (tc.empty()) {
nseltcs = md.test_cases.size();
} else {
nseltcs = 0;
for (std::map< std::string, vars_map >::const_iterator iter
= md.test_cases.begin(); iter != md.test_cases.end(); iter++) {
const std::string& tcname = (*iter).first;
if (tcname == tc)
nseltcs++;
}
if (nseltcs == 0)
throw std::runtime_error("No such test case");
}
w.start_tp(tp.str(), nseltcs);
if (md.test_cases.empty()) {
w.end_tp("Bogus test program: reported 0 test cases");
errcode = EXIT_FAILURE;
} else {
for (std::map< std::string, vars_map >::const_iterator iter
= md.test_cases.begin(); iter != md.test_cases.end(); iter++) {
const std::string& tcname = (*iter).first;
const vars_map& tcmd = (*iter).second;
if (! tc.empty() && tcname != tc)
continue;
w.start_tc(tcname);
try {
const std::string& reqfail = tools::check_requirements(
tcmd, config);
if (!reqfail.empty()) {
w.end_tc("skipped", reqfail);
continue;
}
} catch (const std::runtime_error& e) {
w.end_tc("failed", e.what());
errcode = EXIT_FAILURE;
continue;
}
const std::pair< int, int > user = tools::get_required_user(
tcmd, config);
tools::fs::path resfile = resdir.get_path() / "tcr";
assert(!tools::fs::exists(resfile));
try {
const bool has_cleanup = tools::text::to_bool(
(*tcmd.find("has.cleanup")).second);
tools::fs::temp_dir workdir(tools::fs::path(tools::config::get(
"atf_workdir")) / "atf-run.XXXXXX");
if (user.first != -1 && user.second != -1) {
if (::chown(workdir.get_path().c_str(), user.first,
user.second) == -1) {
throw tools::system_error("chown(" +
workdir.get_path().str() + ")", "chown(2) failed",
errno);
}
resfile = workdir.get_path() / "tcr";
}
std::pair< std::string, const tools::process::status > s =
tools::test_program::run_test_case(
tp, tcname, "body", tcmd, config,
resfile, workdir.get_path(), w);
if (s.second.signaled() && s.second.coredump())
dump_stacktrace(tp, s.second, workdir.get_path(), w);
if (has_cleanup)
(void)tools::test_program::run_test_case(
tp, tcname, "cleanup", tcmd,
config, resfile, workdir.get_path(), w);
// TODO: Force deletion of workdir.
tools::test_program::test_case_result tcr =
get_test_case_result(s.first, s.second, resfile);
w.end_tc(tcr.state(), tcr.reason());
if (tcr.state() == "failed")
errcode = EXIT_FAILURE;
} catch (...) {
if (tools::fs::exists(resfile))
tools::fs::remove(resfile);
throw;
}
if (tools::fs::exists(resfile))
tools::fs::remove(resfile);
}
w.end_tp("");
}
return errcode;
}
static void
colon_split(const std::string &s, std::string &tp, std::string &tc)
{
size_t colon_pos = s.rfind(':');
if (colon_pos != std::string::npos && colon_pos < s.size() - 1) {
tp = s.substr(0, colon_pos);
tc = s.substr(colon_pos + 1);
} else {
tp = s;
tc = "";
}
}
size_t
atf_run::count_tps(std::vector< std::string > tps)
const
{
size_t ntps = 0;
for (std::vector< std::string >::const_iterator iter = tps.begin();
iter != tps.end(); iter++) {
std::string tpname, tcname;
colon_split(*iter, tpname, tcname);
tools::fs::path tp(tpname);
tools::fs::file_info fi(tp);
if (fi.get_type() == tools::fs::file_info::dir_type) {
tools::atffile af = tools::read_atffile(tp / "Atffile");
std::vector< std::string > aux = af.tps();
for (std::vector< std::string >::iterator i2 = aux.begin();
i2 != aux.end(); i2++)
*i2 = (tp / *i2).str();
ntps += count_tps(aux);
} else
ntps++;
}
return ntps;
}
static
void
call_hook(const std::string& tool, const std::string& hook)
{
const tools::fs::path sh(tools::config::get("atf_shell"));
const tools::fs::path hooks =
tools::fs::path(tools::config::get("atf_pkgdatadir")) / (tool + ".hooks");
const tools::process::status s =
tools::process::exec(sh,
tools::process::argv_array(sh.c_str(), hooks.c_str(),
hook.c_str(), NULL),
tools::process::stream_inherit(),
tools::process::stream_inherit());
if (!s.exited() || s.exitstatus() != EXIT_SUCCESS)
throw std::runtime_error("Failed to run the '" + hook + "' hook "
"for '" + tool + "'");
}
int
atf_run::main(void)
{
tools::atffile af = tools::read_atffile(tools::fs::path("Atffile"));
std::vector< std::string > tps;
tps = af.tps();
if (m_argc >= 1) {
// TODO: Ensure that the given test names are listed in the
// Atffile. Take into account that the file can be using globs.
tps.clear();
for (int i = 0; i < m_argc; i++)
tps.push_back(m_argv[i]);
}
// Read configuration data for this test suite.
vars_map test_suite_vars;
{
vars_map::const_iterator iter = af.props().find("test-suite");
assert(iter != af.props().end());
test_suite_vars = tools::config_file::read_config_files((*iter).second);
}
tools::test_program::atf_tps_writer w(std::cout);
call_hook("atf-run", "info_start_hook");
w.ntps(count_tps(tps));
bool ok = true;
for (std::vector< std::string >::const_iterator iter = tps.begin();
iter != tps.end(); iter++) {
std::string tp, tc;
colon_split(*iter, tp, tc);
const bool result = run_test(tools::fs::path(tp), tc, w,
tools::config_file::merge_configs(af.conf(), test_suite_vars));
ok &= (result == EXIT_SUCCESS);
}
call_hook("atf-run", "info_end_hook");
return ok ? EXIT_SUCCESS : EXIT_FAILURE;
}
int
main(int argc, char* const* argv)
{
return atf_run().run(argc, argv);
}