tests.py revision 11836:3195e72010da 
1#!/usr/bin/env python2
2#
3# Copyright (c) 2016 ARM Limited
4# All rights reserved
5#
6# The license below extends only to copyright in the software and shall
7# not be construed as granting a license to any other intellectual
8# property including but not limited to intellectual property relating
9# to a hardware implementation of the functionality of the software
10# licensed hereunder.  You may use the software subject to the license
11# terms below provided that you ensure that this notice is replicated
12# unmodified and in its entirety in all distributions of the software,
13# modified or unmodified, in source code or in binary form.
14#
15# Redistribution and use in source and binary forms, with or without
16# modification, are permitted provided that the following conditions are
17# met: redistributions of source code must retain the above copyright
18# notice, this list of conditions and the following disclaimer;
19# redistributions in binary form must reproduce the above copyright
20# notice, this list of conditions and the following disclaimer in the
21# documentation and/or other materials provided with the distribution;
22# neither the name of the copyright holders nor the names of its
23# contributors may be used to endorse or promote products derived from
24# this software without specific prior written permission.
25#
26# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
27# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
28# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
29# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
30# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
31# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
32# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
33# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
34# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
35# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
36# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
37#
38# Authors: Andreas Sandberg
39
40import argparse
41import sys
42import os
43import pickle
44
45from testing.tests import *
46import testing.results
47
48class ParagraphHelpFormatter(argparse.HelpFormatter):
49    def _fill_text(self, text, width, indent):
50        return "\n\n".join([
51            super(ParagraphHelpFormatter, self)._fill_text(p, width, indent) \
52            for p in text.split("\n\n") ])
53
54formatters = {
55    "junit" : testing.results.JUnit,
56    "text" : testing.results.Text,
57    "summary" : testing.results.TextSummary,
58    "pickle" : testing.results.Pickle,
59}
60
61
62def _add_format_args(parser):
63    parser.add_argument("--format", choices=formatters, default="text",
64                        help="Output format")
65
66    parser.add_argument("--no-junit-xlate-names", action="store_true",
67                        help="Don't translate test names to " \
68                        "package-like names")
69
70    parser.add_argument("--output", "-o",
71                        type=argparse.FileType('w'), default=sys.stdout,
72                        help="Test result output file")
73
74
75def _create_formatter(args):
76    formatter = formatters[args.format]
77    kwargs = {
78        "fout" : args.output,
79        "verbose" : args.verbose
80    }
81
82    if issubclass(formatter, testing.results.JUnit):
83        kwargs.update({
84            "translate_names" : not args.no_junit_xlate_names,
85        })
86
87    return formatter(**kwargs)
88
89
90def _list_tests_args(subparsers):
91    parser = subparsers.add_parser(
92        "list",
93        formatter_class=ParagraphHelpFormatter,
94        help="List available tests",
95        description="List available tests",
96        epilog="""
97        Generate a list of available tests using a list filter.
98
99        The filter is a string consisting of the target ISA optionally
100        followed by the test category and mode separated by
101        slashes. The test names emitted by this command can be fed
102        into the run command.
103
104        For example, to list all quick arm tests, run the following:
105        tests.py list arm/quick
106
107        Non-mandatory parts of the filter string (anything other than
108        the ISA) can be left out or replaced with the wildcard
109        character. For example, all full-system tests can be listed
110        with this command: tests.py list arm/*/fs""")
111
112    parser.add_argument("--ruby-protocol", type=str, default=None,
113                        help="Ruby protocol")
114
115    parser.add_argument("--gpu-isa", type=str, default=None,
116                        help="GPU ISA")
117
118    parser.add_argument("list_filter", metavar="ISA[/category/mode]",
119                        action="append", type=str,
120                        help="List available test cases")
121
122def _list_tests(args):
123    for isa, categories, modes in \
124        ( parse_test_filter(f) for f in args.list_filter ):
125
126        for test in get_tests(isa, categories=categories, modes=modes,
127                              ruby_protocol=args.ruby_protocol,
128                              gpu_isa=args.gpu_isa):
129            print "/".join(test)
130    sys.exit(0)
131
132def _run_tests_args(subparsers):
133    parser = subparsers.add_parser(
134        "run",
135        formatter_class=ParagraphHelpFormatter,
136        help='Run one or more tests',
137        description="Run one or more tests.",
138        epilog="""
139        Run one or more tests described by a gem5 test tuple.
140
141        The test tuple consists of a test category (quick or long), a
142        test mode (fs or se), a workload name, an isa, an operating
143        system, and a config name separate by slashes. For example:
144        quick/se/00.hello/arm/linux/simple-timing
145
146        Available tests can be listed using the 'list' sub-command
147        (e.g., "tests.py list arm/quick" or one of the scons test list
148        targets (e.g., "scons build/ARM/tests/opt/quick.list").
149
150        The test results can be stored in multiple different output
151        formats. See the help for the show command for more details
152        about output formatting.""")
153
154    parser.add_argument("gem5", type=str,
155                        help="gem5 binary")
156
157    parser.add_argument("test", type=str, nargs="*",
158                        help="List of tests to execute")
159
160    parser.add_argument("--directory", "-d",
161                        type=str, default="m5tests",
162                        help="Test work directory")
163
164    parser.add_argument("--timeout", "-t",
165                        type=int, default="0", metavar="MINUTES",
166                        help="Timeout, 0 to disable")
167
168    parser.add_argument("--skip-diff-out", action="store_true",
169                        help="Skip output diffing stage")
170
171    parser.add_argument("--skip-diff-stat", action="store_true",
172                        help="Skip stat diffing stage")
173
174    _add_format_args(parser)
175
176def _run_tests(args):
177    if not os.path.isfile(args.gem5) or not os.access(args.gem5, os.X_OK):
178        print >> sys.stderr, \
179            "gem5 binary '%s' not an executable file" % args.gem5
180        sys.exit(2)
181
182    formatter = _create_formatter(args)
183
184    out_base = os.path.abspath(args.directory)
185    if not os.path.exists(out_base):
186        os.mkdir(out_base)
187    tests = []
188    for test_name in args.test:
189        config = ClassicConfig(*test_name.split("/"))
190        out_dir = os.path.join(out_base, "/".join(config))
191        tests.append(
192            ClassicTest(args.gem5, out_dir, config,
193                        timeout=args.timeout,
194                        skip_diff_stat=args.skip_diff_stat,
195                        skip_diff_out=args.skip_diff_out))
196
197    all_results = []
198    print "Running %i tests" % len(tests)
199    for testno, test in enumerate(tests):
200        print "%i: Running '%s'..." % (testno, test)
201
202        all_results.append(test.run())
203
204    formatter.dump_suites(all_results)
205
206def _show_args(subparsers):
207    parser = subparsers.add_parser(
208        "show",
209        formatter_class=ParagraphHelpFormatter,
210        help='Display pickled test results',
211        description='Display pickled test results',
212        epilog="""
213        Reformat the pickled output from one or more test runs. This
214        command is typically used with the output from a single test
215        run, but it can also be used to merge the outputs from
216        multiple runs.
217
218        The 'text' format is a verbose output format that provides
219        information about individual test units and the output from
220        failed tests. It's mainly useful for debugging test failures.
221
222        The 'summary' format provides outputs the results of one test
223        per line with the test's overall status (OK, SKIPPED, or
224        FAILED).
225
226        The 'junit' format is primarily intended for use with CI
227        systems. It provides an XML representation of test
228        status. Similar to the text format, it includes detailed
229        information about test failures. Since many JUnit parser make
230        assume that test names look like Java packet strings, the
231        JUnit formatter automatically to something the looks like a
232        Java class path ('.'->'-', '/'->'.').
233
234        The 'pickle' format stores the raw results in a format that
235        can be reformatted using this command. It's typically used
236        with the show command to merge multiple test results into one
237        pickle file.""")
238
239    _add_format_args(parser)
240
241    parser.add_argument("result", type=argparse.FileType("rb"), nargs="*",
242                        help="Pickled test results")
243
244def _show(args):
245    formatter = _create_formatter(args)
246    suites = sum([ pickle.load(f) for f in args.result ], [])
247    formatter.dump_suites(suites)
248
249def _test_args(subparsers):
250    parser = subparsers.add_parser(
251        "test",
252        formatter_class=ParagraphHelpFormatter,
253        help='Probe test results and set exit code',
254        epilog="""
255
256        Load one or more pickled test file and return an exit code
257        corresponding to the test outcome. The following exit codes
258        can be returned:
259
260        0: All tests were successful or skipped.
261
262        1: General fault in the script such as incorrect parameters or
263        failing to parse a pickle file.
264
265        2: At least one test failed to run. This is what the summary
266        formatter usually shows as a 'FAILED'.
267
268        3: All tests ran correctly, but at least one failed to
269        verify its output. When displaying test output using the
270        summary formatter, such a test would show up as 'CHANGED'.
271        """)
272
273    _add_format_args(parser)
274
275    parser.add_argument("result", type=argparse.FileType("rb"), nargs="*",
276                        help="Pickled test results")
277
278def _test(args):
279    suites = sum([ pickle.load(f) for f in args.result ], [])
280
281    if all(s for s in suites):
282        sys.exit(0)
283    elif any([ s.failed_run() for s in suites ]):
284        sys.exit(2)
285    elif any([ s.changed() for s in suites ]):
286        sys.exit(3)
287    else:
288        assert False, "Unexpected return status from test"
289
290_commands = {
291    "list" : (_list_tests, _list_tests_args),
292    "run" : (_run_tests, _run_tests_args),
293    "show" : (_show, _show_args),
294    "test" : (_test, _test_args),
295}
296
297def main():
298    parser = argparse.ArgumentParser(
299        formatter_class=ParagraphHelpFormatter,
300        description="""gem5 testing multi tool.""",
301        epilog="""
302        This tool provides an interface to gem5's test framework that
303        doesn't depend on gem5's build system. It supports test
304        listing, running, and output formatting.
305
306        The list sub-command (e.g., "test.py list arm/quick") produces
307        a list of tests tuples that can be used by the run command
308        (e.g., "tests.py run gem5.opt
309        quick/se/00.hello/arm/linux/simple-timing").
310
311        The run command supports several output formats. One of them,
312        pickle, contains the raw output from the tests and can be
313        re-formatted using the show command (e.g., "tests.py show
314        --format summary *.pickle"). Such pickle files are also
315        generated by the build system when scons is used to run
316        regressions.
317
318        See the usage strings for the individual sub-commands for
319        details.""")
320
321    parser.add_argument("--verbose", action="store_true",
322                        help="Produce more verbose output")
323
324    subparsers = parser.add_subparsers(dest="command")
325
326    for key, (impl, cmd_parser) in _commands.items():
327        cmd_parser(subparsers)
328
329    args = parser.parse_args()
330    impl, cmd_parser = _commands[args.command]
331    impl(args)
332
333if __name__ == "__main__":
334    main()
335