1# Copyright (c) 2017 Mark D. Hill and David A. Wood
2# All rights reserved.
3#
4# Redistribution and use in source and binary forms, with or without
5# modification, are permitted provided that the following conditions are
6# met: redistributions of source code must retain the above copyright
7# notice, this list of conditions and the following disclaimer;
8# redistributions in binary form must reproduce the above copyright
9# notice, this list of conditions and the following disclaimer in the
10# documentation and/or other materials provided with the distribution;
11# neither the name of the copyright holders nor the names of its
12# contributors may be used to endorse or promote products derived from
13# this software without specific prior written permission.
14#
15# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
16# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
17# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
18# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
19# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
20# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
21# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
22# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
23# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
24# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
25# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
26#
27# Authors: Sean Wilson
28
29import os
30import copy
31import subprocess
32
33from testlib.test import TestFunction
34from testlib.suite import TestSuite
35from testlib.helper import log_call
36from testlib.config import constants, config
37from fixture import TempdirFixture, Gem5Fixture, VariableFixture
38import verifier
39
40def gem5_verify_config(name,
41                       config,
42                       config_args,
43                       verifiers,
44                       gem5_args=tuple(),
45                       fixtures=[],
46                       valid_isas=constants.supported_isas,
47                       valid_variants=constants.supported_variants,
48                       length=constants.supported_lengths[0],
49                       protocol=None):
50    '''
51    Helper class to generate common gem5 tests using verifiers.
52
53    The generated TestSuite will run gem5 with the provided config and
54    config_args. After that it will run any provided verifiers to verify
55    details about the gem5 run.
56
57    .. seealso::  For the verifiers see :mod:`testlib.gem5.verifier`
58
59    :param name: Name of the test.
60    :param config: The config to give gem5.
61    :param config_args: A list of arguments to pass to the given config.
62
63    :param verifiers: An iterable with Verifier instances which will be placed
64        into a suite that will be ran after a gem5 run.
65
66    :param gem5_args: An iterable with arguments to give to gem5. (Arguments
67        that would normally go before the config path.)
68
69    :param valid_isas: An iterable with the isas that this test can be ran
70        for. If None given, will run for all supported_isas.
71
72    :param valid_variants: An iterable with the variant levels that
73        this test can be ran for. (E.g. opt, debug)
74    '''
75    fixtures = list(fixtures)
76    testsuites = []
77    for opt in valid_variants:
78        for isa in valid_isas:
79
80            # Create a tempdir fixture to be shared throughout the test.
81            tempdir = TempdirFixture()
82            gem5_returncode = VariableFixture(
83                    name=constants.gem5_returncode_fixture_name)
84
85            # Common name of this generated testcase.
86            _name = '{given_name}-{isa}-{opt}'.format(
87                    given_name=name,
88                    isa=isa,
89                    opt=opt)
90            if protocol:
91                _name += '-'+protocol
92
93            # Create the running of gem5 subtest.
94            # NOTE: We specifically create this test before our verifiers so
95            # this is listed first.
96            tests = []
97            gem5_execution = TestFunction(
98                    _create_test_run_gem5(config, config_args, gem5_args),
99                    name=_name)
100            tests.append(gem5_execution)
101
102            # Create copies of the verifier subtests for this isa and
103            # variant.
104            for verifier in verifiers:
105                tests.append(verifier.instantiate_test(_name))
106
107            # Add the isa and variant to tags list.
108            tags = [isa, opt, length]
109
110            # Create the gem5 target for the specific architecture and
111            # variant.
112            _fixtures = copy.copy(fixtures)
113            _fixtures.append(Gem5Fixture(isa, opt, protocol))
114            _fixtures.append(tempdir)
115            _fixtures.append(gem5_returncode)
116
117            # Finally construct the self contained TestSuite out of our
118            # tests.
119            testsuites.append(TestSuite(
120                name=_name,
121                fixtures=_fixtures,
122                tags=tags,
123                tests=tests))
124    return testsuites
125
126def _create_test_run_gem5(config, config_args, gem5_args):
127    def test_run_gem5(params):
128        '''
129        Simple \'test\' which runs gem5 and saves the result into a tempdir.
130
131        NOTE: Requires fixtures: tempdir, gem5
132        '''
133        fixtures = params.fixtures
134
135        if gem5_args is None:
136            _gem5_args = tuple()
137        elif isinstance(gem5_args, str):
138            # If just a single str, place it in an iterable
139            _gem5_args = (gem5_args,)
140        else:
141            _gem5_args = gem5_args
142
143        # FIXME/TODO: I don't like the idea of having to modify this test run
144        # or always collect results even if not using a verifier. There should
145        # be some configuration in here that only gathers certain results for
146        # certain verifiers.
147        #
148        # I.E. Only the returncode verifier will use the gem5_returncode
149        # fixture, but we always require it even if that verifier isn't being
150        # ran.
151        returncode = fixtures[constants.gem5_returncode_fixture_name]
152        tempdir = fixtures[constants.tempdir_fixture_name].path
153        gem5 = fixtures[constants.gem5_binary_fixture_name].path
154        command = [
155            gem5,
156            '-d',  # Set redirect dir to tempdir.
157            tempdir,
158            '-re',# TODO: Change to const. Redirect stdout and stderr
159        ]
160        command.extend(_gem5_args)
161        command.append(config)
162        # Config_args should set up the program args.
163        command.extend(config_args)
164        returncode.value = log_call(params.log, command)
165
166    return test_run_gem5
167