1# -*- mode:python -*-
2#
3# Copyright (c) 2016 ARM Limited
4# All rights reserved
5#
6# The license below extends only to copyright in the software and shall
7# not be construed as granting a license to any other intellectual
8# property including but not limited to intellectual property relating
9# to a hardware implementation of the functionality of the software
10# licensed hereunder.  You may use the software subject to the license
11# terms below provided that you ensure that this notice is replicated
12# unmodified and in its entirety in all distributions of the software,
13# modified or unmodified, in source code or in binary form.
14#
15# Copyright (c) 2004-2006 The Regents of The University of Michigan
16# All rights reserved.
17#
18# Redistribution and use in source and binary forms, with or without
19# modification, are permitted provided that the following conditions are
20# met: redistributions of source code must retain the above copyright
21# notice, this list of conditions and the following disclaimer;
22# redistributions in binary form must reproduce the above copyright
23# notice, this list of conditions and the following disclaimer in the
24# documentation and/or other materials provided with the distribution;
25# neither the name of the copyright holders nor the names of its
26# contributors may be used to endorse or promote products derived from
27# this software without specific prior written permission.
28#
29# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
30# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
31# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
32# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
33# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
34# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
35# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
36# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
37# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
38# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
39# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
40#
41# Authors: Steve Reinhardt
42#          Kevin Lim
43#          Andreas Sandberg
44
45from __future__ import print_function
46
47from SCons.Script.SConscript import SConsEnvironment
48import os
49import pickle
50import sys
51
52sys.path.insert(0, Dir(".").srcnode().abspath)
53import testing.tests as tests
54import testing.results as results
55from gem5_scons.util import get_termcap
56
57Import('env')
58
59# get the termcap from the environment
60termcap = get_termcap()
61
62# Dict that accumulates lists of tests by category (quick, medium, long)
63env.Tests = {}
64gpu_isa = env['TARGET_GPU_ISA'] if env['BUILD_GPU'] else None
65for cat in tests.all_categories:
66    env.Tests[cat] = tuple(
67        tests.get_tests(env["TARGET_ISA"],
68                        categories=(cat, ),
69                        ruby_protocol=env["PROTOCOL"],
70                        gpu_isa=gpu_isa))
71
72def color_message(color, msg):
73    return color + msg + termcap.Normal
74
75def run_test(target, source, env):
76    """Run a test and produce results as a pickle file.
77
78    Targets are as follows:
79    target[0] : Pickle file
80
81    Sources are:
82    source[0] : gem5 binary
83    source[1] : tests/run.py script
84    source[2:] : reference files
85
86    """
87    tgt_dir = os.path.dirname(str(target[0]))
88    config = tests.ClassicConfig(*tgt_dir.split('/')[-6:])
89    test = tests.ClassicTest(source[0].abspath, tgt_dir, config,
90                             timeout=5*60*60,
91                             skip_diff_out=True)
92
93    for ref in test.ref_files():
94        out_file = os.path.join(tgt_dir, ref)
95        if os.path.exists(out_file):
96            env.Execute(Delete(out_file))
97
98    with open(target[0].abspath, "wb") as fout:
99        formatter = results.Pickle(fout=fout)
100        formatter.dump_suites([ test.run() ])
101
102    return 0
103
104def run_test_string(target, source, env):
105    return env.subst("Running test in ${TARGETS[0].dir}.",
106                     target=target, source=source)
107
108testAction = env.Action(run_test, run_test_string)
109
110def print_test(target, source, env):
111    """Run a test and produce results as a pickle file.
112
113    Targets are as follows:
114    target[*] : Dummy targets
115
116    Sources are:
117    source[0] : Pickle file
118
119    """
120    with open(source[0].abspath, "rb") as fin:
121        result = pickle.load(fin)
122
123    assert len(result) == 1
124    result = result[0]
125
126    formatter = None
127    if result.skipped():
128        status = color_message(termcap.Cyan, "skipped.")
129    elif result.changed():
130        status = color_message(termcap.Yellow, "CHANGED!")
131        formatter = results.Text()
132    elif result:
133        status = color_message(termcap.Green, "passed.")
134    else:
135        status = color_message(termcap.Red, "FAILED!")
136        formatter = results.Text()
137
138    if formatter:
139        formatter.dump_suites([result])
140
141    print("***** %s: %s" % (source[0].dir, status))
142    return 0
143
144printAction = env.Action(print_test, strfunction=None)
145
146def update_test(target, source, env):
147    """Update test reference data
148
149    Targets are as follows:
150    target[0] : Dummy file
151
152    Sources are:
153    source[0] : Pickle file
154    """
155
156    src_dir = os.path.dirname(str(source[0]))
157    config = tests.ClassicConfig(*src_dir.split('/')[-6:])
158    test = tests.ClassicTest(source[0].abspath, src_dir, config)
159    ref_dir = test.ref_dir
160
161    with open(source[0].abspath, "rb") as fin:
162        result = pickle.load(fin)
163
164    assert len(result) == 1
165    result = result[0]
166
167    if result.skipped():
168        print("*** %s: %s: Test skipped, not updating." %
169              (source[0].dir, color_message(termcap.Yellow, "WARNING")))
170        return 0
171    elif result:
172        print("*** %s: %s: Test successful, not updating." %
173              (source[0].dir, color_message(termcap.Green, "skipped")))
174        return 0
175    elif result.failed_run():
176        print("*** %s: %s: Test failed, not updating." %
177              (source[0].dir, color_message(termcap.Red, "ERROR")))
178        return 1
179
180    print("** Updating %s" % test)
181    test.update_ref()
182
183    return 0
184
185def update_test_string(target, source, env):
186    return env.subst("Updating ${SOURCES[0].dir}",
187                     target=target, source=source)
188
189updateAction = env.Action(update_test, update_test_string)
190
191def test_builder(test_tuple):
192    """Define a test."""
193
194    out_dir = "/".join(test_tuple)
195    binary = env.M5Binary.abspath
196    test = tests.ClassicTest(binary, out_dir, test_tuple)
197
198    def tgt(name):
199        return os.path.join(out_dir, name)
200
201    def ref(name):
202        return os.path.join(test.ref_dir, name)
203
204    pickle_file = tgt("status.pickle")
205    targets = [
206        pickle_file,
207    ]
208
209    sources = [
210        env.M5Binary,
211        "run.py",
212    ] + [ ref(f) for f in test.ref_files() ]
213
214    env.Command(targets, sources, testAction)
215
216    # phony target to echo status
217    if GetOption('update_ref'):
218        p = env.Command(tgt("_update"), [pickle_file], updateAction)
219    else:
220        p = env.Command(tgt("_print"), [pickle_file], printAction)
221
222    env.AlwaysBuild(p)
223
224def list_tests(target, source, env):
225    """Create a list of tests
226
227    Targets are as follows:
228    target[0] : List file (e.g., tests/opt/all.list,  tests/opt/quick.list)
229
230    Sources are: -
231
232    """
233
234    tgt_name = os.path.basename(str(target[0]))
235    base, ext = os.path.splitext(tgt_name)
236    categories = tests.all_categories if base == "all" else (base, )
237
238    with open(target[0].abspath, "w") as fout:
239        for cat in categories:
240            for test in env.Tests[cat]:
241                print("/".join(test), file=fout)
242
243    return 0
244
245testListAction = env.Action(list_tests, strfunction=None)
246
247env.Command("all.list", tuple(), testListAction)
248for cat, test_list in env.Tests.items():
249    env.Command("%s.list" % cat, tuple(), testListAction)
250    for test in test_list:
251        test_builder(test)
252