path: root/tools/build/src/tools/testing.py

# Status: ported, except for --out-xml
# Base revision: 64488
#
# Copyright 2005 Dave Abrahams
# Copyright 2002, 2003, 2004, 2005, 2010 Vladimir Prus
# Distributed under the Boost Software License, Version 1.0.
# (See accompanying file LICENSE_1_0.txt or http://www.boost.org/LICENSE_1_0.txt)

# This module implements regression testing framework. It declares a number of
# main target rules which perform some action and, if the results are OK,
# creates an output file.
#
# The exact list of rules is:
# 'compile'       -- creates .test file if compilation of sources was
#                    successful.
# 'compile-fail'  -- creates .test file if compilation of sources failed.
# 'run'           -- creates .test file is running of executable produced from
#                    sources was successful. Also leaves behind .output file
#                    with the output from program run.
# 'run-fail'      -- same as above, but .test file is created if running fails.
#
# In all cases, presence of .test file is an indication that the test passed.
# For more convenient reporting, you might want to use C++ Boost regression
# testing utilities (see http://www.boost.org/more/regression.html).
#
# For historical reason, a 'unit-test' rule is available which has the same
# syntax as 'exe' and behaves just like 'run'.

# Things to do:
#  - Teach compiler_status handle Jamfile.v2.
# Notes:
#  - <no-warn> is not implemented, since it is Como-specific, and it is not
#    clear how to implement it
#  - std::locale-support is not implemented (it is used in one test).

import b2.build.feature as feature
import b2.build.type as type
import b2.build.targets as targets
import b2.build.generators as generators
import b2.build.toolset as toolset
import b2.tools.common as common
import b2.util.option as option
import b2.build_system as build_system



from b2.manager import get_manager
from b2.util import stem, bjam_signature
from b2.util.sequence import unique

import bjam

import re
import os.path
import sys

def init():
    pass

# Feature controling the command used to lanch test programs.
feature.feature("testing.launcher", [], ["free", "optional"])

feature.feature("test-info", [], ["free", "incidental"])
feature.feature("testing.arg", [], ["free", "incidental"])
feature.feature("testing.input-file", [], ["free", "dependency"])

feature.feature("preserve-test-targets", ["on", "off"], ["incidental", "propagated"])

# Register target types.
type.register("TEST", ["test"])
type.register("COMPILE", [], "TEST")
type.register("COMPILE_FAIL", [], "TEST")

type.register("RUN_OUTPUT", ["run"])
type.register("RUN", [], "TEST")
type.register("RUN_FAIL", [], "TEST")

type.register("LINK", [], "TEST")
type.register("LINK_FAIL", [], "TEST")
type.register("UNIT_TEST", ["passed"], "TEST")

__all_tests = []

# Declare the rules which create main targets. While the 'type' module already
# creates rules with the same names for us, we need extra convenience: default
# name of main target, so write our own versions.

# Helper rule. Create a test target, using basename of first source if no target
# name is explicitly passed. Remembers the created target in a global variable.
def make_test(target_type, sources, requirements, target_name=None):

    if not target_name:
        target_name = stem(os.path.basename(sources[0]))

    # Having periods (".") in the target name is problematic because the typed
    # generator will strip the suffix and use the bare name for the file
    # targets. Even though the location-prefix averts problems most times it
    # does not prevent ambiguity issues when referring to the test targets. For
    # example when using the XML log output. So we rename the target to remove
    # the periods, and provide an alias for users.
    real_name = target_name.replace(".", "~")

    project = get_manager().projects().current()
    # The <location-prefix> forces the build system for generate paths in the
    # form '$build_dir/array1.test/gcc/debug'. This is necessary to allow
    # post-processing tools to work.
    t = get_manager().targets().create_typed_target(
        type.type_from_rule_name(target_type), project, real_name, sources,
        requirements + ["<location-prefix>" + real_name + ".test"], [], [])

    # The alias to the real target, per period replacement above.
    if real_name != target_name:
        get_manager().projects().project_rules().all_names_["alias"](
            target_name, [t])

    # Remember the test (for --dump-tests). A good way would be to collect all
    # given a project. This has some technical problems: e.g. we can not call
    # this dump from a Jamfile since projects referred by 'build-project' are
    # not available until the whole Jamfile has been loaded.
    __all_tests.append(t)
    return t


# Note: passing more that one cpp file here is known to fail. Passing a cpp file
# and a library target works.
#
@bjam_signature((["sources", "*"], ["requirements", "*"], ["target_name", "?"]))
def compile(sources, requirements, target_name=None):
    return make_test("compile", sources, requirements, target_name)

@bjam_signature((["sources", "*"], ["requirements", "*"], ["target_name", "?"]))
def compile_fail(sources, requirements, target_name=None):
    return make_test("compile-fail", sources, requirements, target_name)

@bjam_signature((["sources", "*"], ["requirements", "*"], ["target_name", "?"]))
def link(sources, requirements, target_name=None):
    return make_test("link", sources, requirements, target_name)

@bjam_signature((["sources", "*"], ["requirements", "*"], ["target_name", "?"]))
def link_fail(sources, requirements, target_name=None):
    return make_test("link-fail", sources, requirements, target_name)

def handle_input_files(input_files):
    if len(input_files) > 1:
        # Check that sorting made when creating property-set instance will not
        # change the ordering.
        if sorted(input_files) != input_files:
            get_manager().errors()("Names of input files must be sorted alphabetically\n" +
                                   "due to internal limitations")
    return ["<testing.input-file>" + f for f in input_files]

@bjam_signature((["sources", "*"], ["args", "*"], ["input_files", "*"],
                 ["requirements", "*"], ["target_name", "?"],
                 ["default_build", "*"]))                 
def run(sources, args, input_files, requirements, target_name=None, default_build=[]):
    if args:
        requirements.append("<testing.arg>" + " ".join(args))
    requirements.extend(handle_input_files(input_files))
    return make_test("run", sources, requirements, target_name)

@bjam_signature((["sources", "*"], ["args", "*"], ["input_files", "*"],
                 ["requirements", "*"], ["target_name", "?"],
                 ["default_build", "*"]))                 
def run_fail(sources, args, input_files, requirements, target_name=None, default_build=[]):
    if args:
        requirements.append("<testing.arg>" + " ".join(args))
    requirements.extend(handle_input_files(input_files))
    return make_test("run-fail", sources, requirements, target_name)

# Register all the rules
for name in ["compile", "compile-fail", "link", "link-fail", "run", "run-fail"]:
    get_manager().projects().add_rule(name, getattr(sys.modules[__name__], name.replace("-", "_")))

# Use 'test-suite' as a synonym for 'alias', for backward compatibility.
from b2.build.alias import alias
get_manager().projects().add_rule("test-suite", alias)

# For all main targets in 'project-module', which are typed targets with type
# derived from 'TEST', produce some interesting information.
#
def dump_tests():
    for t in __all_tests:
        dump_test(t)

# Given a project location in normalized form (slashes are forward), compute the
# name of the Boost library.
#
__ln1 = re.compile("/(tools|libs)/(.*)/(test|example)")
__ln2 = re.compile("/(tools|libs)/(.*)$")
__ln3 = re.compile("(/status$)")
def get_library_name(path):
    
    path = path.replace("\\", "/")
    match1 = __ln1.match(path)
    match2 = __ln2.match(path)
    match3 = __ln3.match(path)

    if match1:
        return match1.group(2)
    elif match2:
        return match2.group(2)
    elif match3:
        return ""
    elif option.get("dump-tests", False, True):
        # The 'run' rule and others might be used outside boost. In that case,
        # just return the path, since the 'library name' makes no sense.
        return path

# Was an XML dump requested?
__out_xml = option.get("out-xml", False, True)

# Takes a target (instance of 'basic-target') and prints
#   - its type
#   - its name
#   - comments specified via the <test-info> property
#   - relative location of all source from the project root.
#
def dump_test(target):
    type = target.type()
    name = target.name()
    project = target.project()

    project_root = project.get('project-root')
    library = get_library_name(os.path.abspath(project.get('location')))
    if library:
        name = library + "/" + name

    sources = target.sources()
    source_files = []
    for s in sources:
        if isinstance(s, targets.FileReference):
            location = os.path.abspath(os.path.join(s.location(), s.name()))
            source_files.append(os.path.relpath(location, os.path.abspath(project_root)))

    target_name = project.get('location') + "//" + target.name() + ".test"

    test_info = target.requirements().get('test-info')
    test_info = " ".join('"' + ti + '"' for ti in test_info)

    # If the user requested XML output on the command-line, add the test info to
    # that XML file rather than dumping them to stdout.
    #if $(.out-xml)
    #{
#        local nl = "
#" ;
#        .contents on $(.out-xml) +=
#            "$(nl)  <test type=\"$(type)\" name=\"$(name)\">"
#            "$(nl)    <target><![CDATA[$(target-name)]]></target>"
#            "$(nl)    <info><![CDATA[$(test-info)]]></info>"
#            "$(nl)    <source><![CDATA[$(source-files)]]></source>"
#            "$(nl)  </test>"
#            ;
#    }
#    else

    source_files = " ".join('"' + s + '"' for s in source_files)
    if test_info:
        print 'boost-test(%s) "%s" [%s] : %s' % (type, name, test_info, source_files)
    else:
        print 'boost-test(%s) "%s" : %s' % (type, name, source_files)

# Register generators. Depending on target type, either 'expect-success' or
# 'expect-failure' rule will be used.
generators.register_standard("testing.expect-success", ["OBJ"], ["COMPILE"])
generators.register_standard("testing.expect-failure", ["OBJ"], ["COMPILE_FAIL"])
generators.register_standard("testing.expect-success", ["RUN_OUTPUT"], ["RUN"])
generators.register_standard("testing.expect-failure", ["RUN_OUTPUT"], ["RUN_FAIL"])
generators.register_standard("testing.expect-success", ["EXE"], ["LINK"])
generators.register_standard("testing.expect-failure", ["EXE"], ["LINK_FAIL"])

# Generator which runs an EXE and captures output.
generators.register_standard("testing.capture-output", ["EXE"], ["RUN_OUTPUT"])

# Generator which creates a target if sources run successfully. Differs from RUN
# in that run output is not captured. The reason why it exists is that the 'run'
# rule is much better for automated testing, but is not user-friendly (see
# http://article.gmane.org/gmane.comp.lib.boost.build/6353).
generators.register_standard("testing.unit-test", ["EXE"], ["UNIT_TEST"])

# FIXME: if those calls are after bjam.call, then bjam will crash
# when toolset.flags calls bjam.caller.
toolset.flags("testing.capture-output", "ARGS", [], ["<testing.arg>"])
toolset.flags("testing.capture-output", "INPUT_FILES", [], ["<testing.input-file>"])
toolset.flags("testing.capture-output", "LAUNCHER", [], ["<testing.launcher>"])

toolset.flags("testing.unit-test", "LAUNCHER", [], ["<testing.launcher>"])
toolset.flags("testing.unit-test", "ARGS", [], ["<testing.arg>"])

# This is a composing generator to support cases where a generator for the
# specified target constructs other targets as well. One such example is msvc's
# exe generator that constructs both EXE and PDB targets.
type.register("TIME", ["time"])
generators.register_composing("testing.time", [], ["TIME"])


# The following code sets up actions for this module. It's pretty convoluted,
# but the basic points is that we most of actions are defined by Jam code
# contained in testing-aux.jam, which we load into Jam module named 'testing'

def run_path_setup(target, sources, ps):

    # For testing, we need to make sure that all dynamic libraries needed by the
    # test are found. So, we collect all paths from dependency libraries (via
    # xdll-path property) and add whatever explicit dll-path user has specified.
    # The resulting paths are added to the environment on each test invocation.
    dll_paths = ps.get('dll-path')
    dll_paths.extend(ps.get('xdll-path'))
    dll_paths.extend(bjam.call("get-target-variable", sources, "RUN_PATH"))
    dll_paths = unique(dll_paths)
    if dll_paths:
        bjam.call("set-target-variable", target, "PATH_SETUP",
                  common.prepend_path_variable_command(
                     common.shared_library_path_variable(), dll_paths))

def capture_output_setup(target, sources, ps):
    run_path_setup(target, sources, ps)

    if ps.get('preserve-test-targets') == ['off']:
        bjam.call("set-target-variable", target, "REMOVE_TEST_TARGETS", "1")

get_manager().engine().register_bjam_action("testing.capture-output",
                                            capture_output_setup)


path = os.path.dirname(__file__)
import b2.util.os_j
get_manager().projects().project_rules()._import_rule("testing", "os.name",
                                                      b2.util.os_j.name)
import b2.tools.common
get_manager().projects().project_rules()._import_rule("testing", "common.rm-command",
                                                      b2.tools.common.rm_command)
get_manager().projects().project_rules()._import_rule("testing", "common.file-creation-command",
                                                      b2.tools.common.file_creation_command)

bjam.call("load", "testing", os.path.join(path, "testing-aux.jam"))


for name in ["expect-success", "expect-failure", "time"]:
    get_manager().engine().register_bjam_action("testing." + name)

get_manager().engine().register_bjam_action("testing.unit-test",
                                            run_path_setup)

if option.get("dump-tests", False, True):
    build_system.add_pre_build_hook(dump_tests)