tests.py revision 12575 
1#!/usr/bin/env python2
2#
3# Copyright (c) 2016 ARM Limited
4# All rights reserved
5#
6# The license below extends only to copyright in the software and shall
7# not be construed as granting a license to any other intellectual
8# property including but not limited to intellectual property relating
9# to a hardware implementation of the functionality of the software
10# licensed hereunder.  You may use the software subject to the license
11# terms below provided that you ensure that this notice is replicated
12# unmodified and in its entirety in all distributions of the software,
13# modified or unmodified, in source code or in binary form.
14#
15# Redistribution and use in source and binary forms, with or without
16# modification, are permitted provided that the following conditions are
17# met: redistributions of source code must retain the above copyright
18# notice, this list of conditions and the following disclaimer;
19# redistributions in binary form must reproduce the above copyright
20# notice, this list of conditions and the following disclaimer in the
21# documentation and/or other materials provided with the distribution;
22# neither the name of the copyright holders nor the names of its
23# contributors may be used to endorse or promote products derived from
24# this software without specific prior written permission.
25#
26# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
27# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
28# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
29# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
30# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
31# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
32# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
33# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
34# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
35# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
36# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
37#
38# Authors: Andreas Sandberg
39
40from __future__ import print_function
41
42import argparse
43import sys
44import os
45import pickle
46
47from testing.tests import *
48import testing.results
49
50class ParagraphHelpFormatter(argparse.HelpFormatter):
51    def _fill_text(self, text, width, indent):
52        return "\n\n".join([
53            super(ParagraphHelpFormatter, self)._fill_text(p, width, indent) \
54            for p in text.split("\n\n") ])
55
56formatters = {
57    "junit" : testing.results.JUnit,
58    "text" : testing.results.Text,
59    "summary" : testing.results.TextSummary,
60    "pickle" : testing.results.Pickle,
61}
62
63
64def _add_format_args(parser):
65    parser.add_argument("--format", choices=formatters, default="text",
66                        help="Output format")
67
68    parser.add_argument("--no-junit-xlate-names", action="store_true",
69                        help="Don't translate test names to " \
70                        "package-like names")
71
72    parser.add_argument("--output", "-o",
73                        type=argparse.FileType('w'), default=sys.stdout,
74                        help="Test result output file")
75
76
77def _create_formatter(args):
78    formatter = formatters[args.format]
79    kwargs = {
80        "fout" : args.output,
81        "verbose" : args.verbose
82    }
83
84    if issubclass(formatter, testing.results.JUnit):
85        kwargs.update({
86            "translate_names" : not args.no_junit_xlate_names,
87        })
88
89    return formatter(**kwargs)
90
91
92def _list_tests_args(subparsers):
93    parser = subparsers.add_parser(
94        "list",
95        formatter_class=ParagraphHelpFormatter,
96        help="List available tests",
97        description="List available tests",
98        epilog="""
99        Generate a list of available tests using a list filter.
100
101        The filter is a string consisting of the target ISA optionally
102        followed by the test category and mode separated by
103        slashes. The test names emitted by this command can be fed
104        into the run command.
105
106        For example, to list all quick arm tests, run the following:
107        tests.py list arm/quick
108
109        Non-mandatory parts of the filter string (anything other than
110        the ISA) can be left out or replaced with the wildcard
111        character. For example, all full-system tests can be listed
112        with this command: tests.py list arm/*/fs""")
113
114    parser.add_argument("--ruby-protocol", type=str, default=None,
115                        help="Ruby protocol")
116
117    parser.add_argument("--gpu-isa", type=str, default=None,
118                        help="GPU ISA")
119
120    parser.add_argument("list_filter", metavar="ISA[/category/mode]",
121                        action="append", type=str,
122                        help="List available test cases")
123
124def _list_tests(args):
125    for isa, categories, modes in \
126        ( parse_test_filter(f) for f in args.list_filter ):
127
128        for test in get_tests(isa, categories=categories, modes=modes,
129                              ruby_protocol=args.ruby_protocol,
130                              gpu_isa=args.gpu_isa):
131            print("/".join(test))
132    sys.exit(0)
133
134def _run_tests_args(subparsers):
135    parser = subparsers.add_parser(
136        "run",
137        formatter_class=ParagraphHelpFormatter,
138        help='Run one or more tests',
139        description="Run one or more tests.",
140        epilog="""
141        Run one or more tests described by a gem5 test tuple.
142
143        The test tuple consists of a test category (quick or long), a
144        test mode (fs or se), a workload name, an isa, an operating
145        system, and a config name separate by slashes. For example:
146        quick/se/00.hello/arm/linux/simple-timing
147
148        Available tests can be listed using the 'list' sub-command
149        (e.g., "tests.py list arm/quick" or one of the scons test list
150        targets (e.g., "scons build/ARM/tests/opt/quick.list").
151
152        The test results can be stored in multiple different output
153        formats. See the help for the show command for more details
154        about output formatting.""")
155
156    parser.add_argument("gem5", type=str,
157                        help="gem5 binary")
158
159    parser.add_argument("test", type=str, nargs="*",
160                        help="List of tests to execute")
161
162    parser.add_argument("--directory", "-d",
163                        type=str, default="m5tests",
164                        help="Test work directory")
165
166    parser.add_argument("--timeout", "-t",
167                        type=int, default="0", metavar="MINUTES",
168                        help="Timeout, 0 to disable")
169
170    parser.add_argument("--skip-diff-out", action="store_true",
171                        help="Skip output diffing stage")
172
173    parser.add_argument("--skip-diff-stat", action="store_true",
174                        help="Skip stat diffing stage")
175
176    _add_format_args(parser)
177
178def _run_tests(args):
179    if not os.path.isfile(args.gem5) or not os.access(args.gem5, os.X_OK):
180        print("gem5 binary '%s' not an executable file" % args.gem5,
181            file=sys.stderr)
182        sys.exit(2)
183
184    formatter = _create_formatter(args)
185
186    out_base = os.path.abspath(args.directory)
187    if not os.path.exists(out_base):
188        os.mkdir(out_base)
189    tests = []
190    for test_name in args.test:
191        config = ClassicConfig(*test_name.split("/"))
192        out_dir = os.path.join(out_base, "/".join(config))
193        tests.append(
194            ClassicTest(args.gem5, out_dir, config,
195                        timeout=args.timeout,
196                        skip_diff_stat=args.skip_diff_stat,
197                        skip_diff_out=args.skip_diff_out))
198
199    all_results = []
200    print("Running %i tests" % len(tests))
201    for testno, test in enumerate(tests):
202        print("%i: Running '%s'..." % (testno, test))
203
204        all_results.append(test.run())
205
206    formatter.dump_suites(all_results)
207
208def _show_args(subparsers):
209    parser = subparsers.add_parser(
210        "show",
211        formatter_class=ParagraphHelpFormatter,
212        help='Display pickled test results',
213        description='Display pickled test results',
214        epilog="""
215        Reformat the pickled output from one or more test runs. This
216        command is typically used with the output from a single test
217        run, but it can also be used to merge the outputs from
218        multiple runs.
219
220        The 'text' format is a verbose output format that provides
221        information about individual test units and the output from
222        failed tests. It's mainly useful for debugging test failures.
223
224        The 'summary' format provides outputs the results of one test
225        per line with the test's overall status (OK, SKIPPED, or
226        FAILED).
227
228        The 'junit' format is primarily intended for use with CI
229        systems. It provides an XML representation of test
230        status. Similar to the text format, it includes detailed
231        information about test failures. Since many JUnit parser make
232        assume that test names look like Java packet strings, the
233        JUnit formatter automatically to something the looks like a
234        Java class path ('.'->'-', '/'->'.').
235
236        The 'pickle' format stores the raw results in a format that
237        can be reformatted using this command. It's typically used
238        with the show command to merge multiple test results into one
239        pickle file.""")
240
241    _add_format_args(parser)
242
243    parser.add_argument("result", type=argparse.FileType("rb"), nargs="*",
244                        help="Pickled test results")
245
246def _show(args):
247    def _load(f):
248        # Load the pickled status file, sometimes e.g., when a
249        # regression is still running the status file might be
250        # incomplete.
251        try:
252            return pickle.load(f)
253        except EOFError:
254            print('Could not read file %s' % f.name, file=sys.stderr)
255            return []
256
257    formatter = _create_formatter(args)
258    suites = sum([ _load(f) for f in args.result ], [])
259    formatter.dump_suites(suites)
260
261def _test_args(subparsers):
262    parser = subparsers.add_parser(
263        "test",
264        formatter_class=ParagraphHelpFormatter,
265        help='Probe test results and set exit code',
266        epilog="""
267
268        Load one or more pickled test file and return an exit code
269        corresponding to the test outcome. The following exit codes
270        can be returned:
271
272        0: All tests were successful or skipped.
273
274        1: General fault in the script such as incorrect parameters or
275        failing to parse a pickle file.
276
277        2: At least one test failed to run. This is what the summary
278        formatter usually shows as a 'FAILED'.
279
280        3: All tests ran correctly, but at least one failed to
281        verify its output. When displaying test output using the
282        summary formatter, such a test would show up as 'CHANGED'.
283        """)
284
285    parser.add_argument("result", type=argparse.FileType("rb"), nargs="*",
286                        help="Pickled test results")
287
288def _test(args):
289    try:
290        suites = sum([ pickle.load(f) for f in args.result ], [])
291    except EOFError:
292        print('Could not read all files', file=sys.stderr)
293        sys.exit(2)
294
295    if all(s for s in suites):
296        sys.exit(0)
297    elif any([ s.failed_run() for s in suites ]):
298        sys.exit(2)
299    elif any([ s.changed() for s in suites ]):
300        sys.exit(3)
301    else:
302        assert False, "Unexpected return status from test"
303
304_commands = {
305    "list" : (_list_tests, _list_tests_args),
306    "run" : (_run_tests, _run_tests_args),
307    "show" : (_show, _show_args),
308    "test" : (_test, _test_args),
309}
310
311def main():
312    parser = argparse.ArgumentParser(
313        formatter_class=ParagraphHelpFormatter,
314        description="""gem5 testing multi tool.""",
315        epilog="""
316        This tool provides an interface to gem5's test framework that
317        doesn't depend on gem5's build system. It supports test
318        listing, running, and output formatting.
319
320        The list sub-command (e.g., "test.py list arm/quick") produces
321        a list of tests tuples that can be used by the run command
322        (e.g., "tests.py run gem5.opt
323        quick/se/00.hello/arm/linux/simple-timing").
324
325        The run command supports several output formats. One of them,
326        pickle, contains the raw output from the tests and can be
327        re-formatted using the show command (e.g., "tests.py show
328        --format summary *.pickle"). Such pickle files are also
329        generated by the build system when scons is used to run
330        regressions.
331
332        See the usage strings for the individual sub-commands for
333        details.""")
334
335    parser.add_argument("--verbose", action="store_true",
336                        help="Produce more verbose output")
337
338    subparsers = parser.add_subparsers(dest="command")
339
340    for key, (impl, cmd_parser) in _commands.items():
341        cmd_parser(subparsers)
342
343    args = parser.parse_args()
344    impl, cmd_parser = _commands[args.command]
345    impl(args)
346
347if __name__ == "__main__":
348    main()
349