1#!/usr/bin/env python2
2#
3# Copyright (c) 2016 ARM Limited
4# All rights reserved
5#
6# The license below extends only to copyright in the software and shall
7# not be construed as granting a license to any other intellectual
8# property including but not limited to intellectual property relating
9# to a hardware implementation of the functionality of the software
10# licensed hereunder. You may use the software subject to the license
11# terms below provided that you ensure that this notice is replicated
12# unmodified and in its entirety in all distributions of the software,
13# modified or unmodified, in source code or in binary form.
14#
15# Redistribution and use in source and binary forms, with or without
16# modification, are permitted provided that the following conditions are
17# met: redistributions of source code must retain the above copyright
18# notice, this list of conditions and the following disclaimer;
19# redistributions in binary form must reproduce the above copyright
20# notice, this list of conditions and the following disclaimer in the
21# documentation and/or other materials provided with the distribution;
22# neither the name of the copyright holders nor the names of its
23# contributors may be used to endorse or promote products derived from
24# this software without specific prior written permission.
25#
26# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
27# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
28# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
29# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
30# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
31# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
32# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
33# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
34# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
35# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
36# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
37#
38# Authors: Andreas Sandberg
39
40from __future__ import print_function
41
42import argparse
43import sys
44import os
45import pickle
46
47from testing.tests import *
48import testing.results
49
50class ParagraphHelpFormatter(argparse.HelpFormatter):
51 def _fill_text(self, text, width, indent):
52 return "\n\n".join([
53 super(ParagraphHelpFormatter, self)._fill_text(p, width, indent) \
54 for p in text.split("\n\n") ])
55
56formatters = {
57 "junit" : testing.results.JUnit,
58 "text" : testing.results.Text,
59 "summary" : testing.results.TextSummary,
60 "pickle" : testing.results.Pickle,
61}
62
63
64def _add_format_args(parser):
65 parser.add_argument("--format", choices=formatters, default="text",
66 help="Output format")
67
68 parser.add_argument("--no-junit-xlate-names", action="store_true",
69 help="Don't translate test names to " \
70 "package-like names")
71
72 parser.add_argument("--output", "-o",
73 type=argparse.FileType('w'), default=sys.stdout,
74 help="Test result output file")
75
76
77def _create_formatter(args):
78 formatter = formatters[args.format]
79 kwargs = {
80 "fout" : args.output,
81 "verbose" : args.verbose
82 }
83
84 if issubclass(formatter, testing.results.JUnit):
85 kwargs.update({
86 "translate_names" : not args.no_junit_xlate_names,
87 })
88
89 return formatter(**kwargs)
90
91
92def _list_tests_args(subparsers):
93 parser = subparsers.add_parser(
94 "list",
95 formatter_class=ParagraphHelpFormatter,
96 help="List available tests",
97 description="List available tests",
98 epilog="""
99 Generate a list of available tests using a list filter.
100
101 The filter is a string consisting of the target ISA optionally
102 followed by the test category and mode separated by
103 slashes. The test names emitted by this command can be fed
104 into the run command.
105
106 For example, to list all quick arm tests, run the following:
107 tests.py list arm/quick
108
109 Non-mandatory parts of the filter string (anything other than
110 the ISA) can be left out or replaced with the wildcard
111 character. For example, all full-system tests can be listed
112 with this command: tests.py list arm/*/fs""")
113
114 parser.add_argument("--ruby-protocol", type=str, default=None,
115 help="Ruby protocol")
116
117 parser.add_argument("--gpu-isa", type=str, default=None,
118 help="GPU ISA")
119
120 parser.add_argument("list_filter", metavar="ISA[/category/mode]",
121 action="append", type=str,
122 help="List available test cases")
123
124def _list_tests(args):
125 for isa, categories, modes in \
126 ( parse_test_filter(f) for f in args.list_filter ):
127
128 for test in get_tests(isa, categories=categories, modes=modes,
129 ruby_protocol=args.ruby_protocol,
130 gpu_isa=args.gpu_isa):
131 print("/".join(test))
132 sys.exit(0)
133
134def _run_tests_args(subparsers):
135 parser = subparsers.add_parser(
136 "run",
137 formatter_class=ParagraphHelpFormatter,
138 help='Run one or more tests',
139 description="Run one or more tests.",
140 epilog="""
141 Run one or more tests described by a gem5 test tuple.
142
143 The test tuple consists of a test category (quick or long), a
144 test mode (fs or se), a workload name, an isa, an operating
145 system, and a config name separate by slashes. For example:
146 quick/se/00.hello/arm/linux/simple-timing
147
148 Available tests can be listed using the 'list' sub-command
149 (e.g., "tests.py list arm/quick" or one of the scons test list
150 targets (e.g., "scons build/ARM/tests/opt/quick.list").
151
152 The test results can be stored in multiple different output
153 formats. See the help for the show command for more details
154 about output formatting.""")
155
156 parser.add_argument("gem5", type=str,
157 help="gem5 binary")
158
159 parser.add_argument("test", type=str, nargs="*",
160 help="List of tests to execute")
161
162 parser.add_argument("--directory", "-d",
163 type=str, default="m5tests",
164 help="Test work directory")
165
166 parser.add_argument("--timeout", "-t",
167 type=int, default="0", metavar="MINUTES",
168 help="Timeout, 0 to disable")
169
170 parser.add_argument("--skip-diff-out", action="store_true",
171 help="Skip output diffing stage")
172
173 parser.add_argument("--skip-diff-stat", action="store_true",
174 help="Skip stat diffing stage")
175
176 _add_format_args(parser)
177
178def _run_tests(args):
179 if not os.path.isfile(args.gem5) or not os.access(args.gem5, os.X_OK):
180 print("gem5 binary '%s' not an executable file" % args.gem5,
181 file=sys.stderr)
182 sys.exit(2)
183
184 formatter = _create_formatter(args)
185
186 out_base = os.path.abspath(args.directory)
187 if not os.path.exists(out_base):
188 os.mkdir(out_base)
189 tests = []
190 for test_name in args.test:
191 config = ClassicConfig(*test_name.split("/"))
192 out_dir = os.path.join(out_base, "/".join(config))
193 tests.append(
194 ClassicTest(args.gem5, out_dir, config,
195 timeout=args.timeout,
196 skip_diff_stat=args.skip_diff_stat,
197 skip_diff_out=args.skip_diff_out))
198
199 all_results = []
200 print("Running %i tests" % len(tests))
201 for testno, test in enumerate(tests):
202 print("%i: Running '%s'..." % (testno, test))
203
204 all_results.append(test.run())
205
206 formatter.dump_suites(all_results)
207
208def _show_args(subparsers):
209 parser = subparsers.add_parser(
210 "show",
211 formatter_class=ParagraphHelpFormatter,
212 help='Display pickled test results',
213 description='Display pickled test results',
214 epilog="""
215 Reformat the pickled output from one or more test runs. This
216 command is typically used with the output from a single test
217 run, but it can also be used to merge the outputs from
218 multiple runs.
219
220 The 'text' format is a verbose output format that provides
221 information about individual test units and the output from
222 failed tests. It's mainly useful for debugging test failures.
223
224 The 'summary' format provides outputs the results of one test
225 per line with the test's overall status (OK, SKIPPED, or
226 FAILED).
227
228 The 'junit' format is primarily intended for use with CI
229 systems. It provides an XML representation of test
230 status. Similar to the text format, it includes detailed
231 information about test failures. Since many JUnit parser make
232 assume that test names look like Java packet strings, the
233 JUnit formatter automatically to something the looks like a
234 Java class path ('.'->'-', '/'->'.').
235
236 The 'pickle' format stores the raw results in a format that
237 can be reformatted using this command. It's typically used
238 with the show command to merge multiple test results into one
239 pickle file.""")
240
241 _add_format_args(parser)
242
243 parser.add_argument("result", type=argparse.FileType("rb"), nargs="*",
244 help="Pickled test results")
245
246def _show(args):
247 def _load(f):
248 # Load the pickled status file, sometimes e.g., when a
249 # regression is still running the status file might be
250 # incomplete.
251 try:
252 return pickle.load(f)
253 except EOFError:
254 print('Could not read file %s' % f.name, file=sys.stderr)
255 return []
256
257 formatter = _create_formatter(args)
258 suites = sum([ _load(f) for f in args.result ], [])
259 formatter.dump_suites(suites)
260
261def _test_args(subparsers):
262 parser = subparsers.add_parser(
263 "test",
264 formatter_class=ParagraphHelpFormatter,
265 help='Probe test results and set exit code',
266 epilog="""
267
268 Load one or more pickled test file and return an exit code
269 corresponding to the test outcome. The following exit codes
270 can be returned:
271
272 0: All tests were successful or skipped.
273
274 1: General fault in the script such as incorrect parameters or
275 failing to parse a pickle file.
276
277 2: At least one test failed to run. This is what the summary
278 formatter usually shows as a 'FAILED'.
279
280 3: All tests ran correctly, but at least one failed to
281 verify its output. When displaying test output using the
282 summary formatter, such a test would show up as 'CHANGED'.
283 """)
284
285 parser.add_argument("result", type=argparse.FileType("rb"), nargs="*",
286 help="Pickled test results")
287
288def _test(args):
289 try:
290 suites = sum([ pickle.load(f) for f in args.result ], [])
291 except EOFError:
292 print('Could not read all files', file=sys.stderr)
293 sys.exit(2)
294
295 if all(s for s in suites):
296 sys.exit(0)
297 elif any([ s.failed_run() for s in suites ]):
298 sys.exit(2)
299 elif any([ s.changed() for s in suites ]):
300 sys.exit(3)
301 else:
302 assert False, "Unexpected return status from test"
303
304_commands = {
305 "list" : (_list_tests, _list_tests_args),
306 "run" : (_run_tests, _run_tests_args),
307 "show" : (_show, _show_args),
308 "test" : (_test, _test_args),
309}
310
311def main():
312 parser = argparse.ArgumentParser(
313 formatter_class=ParagraphHelpFormatter,
314 description="""gem5 testing multi tool.""",
315 epilog="""
316 This tool provides an interface to gem5's test framework that
317 doesn't depend on gem5's build system. It supports test
318 listing, running, and output formatting.
319
320 The list sub-command (e.g., "test.py list arm/quick") produces
321 a list of tests tuples that can be used by the run command
322 (e.g., "tests.py run gem5.opt
323 quick/se/00.hello/arm/linux/simple-timing").
324
325 The run command supports several output formats. One of them,
326 pickle, contains the raw output from the tests and can be
327 re-formatted using the show command (e.g., "tests.py show
328 --format summary *.pickle"). Such pickle files are also
329 generated by the build system when scons is used to run
330 regressions.
331
332 See the usage strings for the individual sub-commands for
333 details.""")
334
335 parser.add_argument("--verbose", action="store_true",
336 help="Produce more verbose output")
337
338 subparsers = parser.add_subparsers(dest="command")
339
340 for key, (impl, cmd_parser) in _commands.items():
341 cmd_parser(subparsers)
342
343 args = parser.parse_args()
344 impl, cmd_parser = _commands[args.command]
345 impl(args)
346
347if __name__ == "__main__":
348 main()