tests.py revision 11917:6b5cded90c35 
1#!/usr/bin/env python2
2#
3# Copyright (c) 2016 ARM Limited
4# All rights reserved
5#
6# The license below extends only to copyright in the software and shall
7# not be construed as granting a license to any other intellectual
8# property including but not limited to intellectual property relating
9# to a hardware implementation of the functionality of the software
10# licensed hereunder.  You may use the software subject to the license
11# terms below provided that you ensure that this notice is replicated
12# unmodified and in its entirety in all distributions of the software,
13# modified or unmodified, in source code or in binary form.
14#
15# Redistribution and use in source and binary forms, with or without
16# modification, are permitted provided that the following conditions are
17# met: redistributions of source code must retain the above copyright
18# notice, this list of conditions and the following disclaimer;
19# redistributions in binary form must reproduce the above copyright
20# notice, this list of conditions and the following disclaimer in the
21# documentation and/or other materials provided with the distribution;
22# neither the name of the copyright holders nor the names of its
23# contributors may be used to endorse or promote products derived from
24# this software without specific prior written permission.
25#
26# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
27# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
28# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
29# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
30# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
31# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
32# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
33# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
34# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
35# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
36# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
37#
38# Authors: Andreas Sandberg
39
40import argparse
41import sys
42import os
43import pickle
44
45from testing.tests import *
46import testing.results
47
48class ParagraphHelpFormatter(argparse.HelpFormatter):
49    def _fill_text(self, text, width, indent):
50        return "\n\n".join([
51            super(ParagraphHelpFormatter, self)._fill_text(p, width, indent) \
52            for p in text.split("\n\n") ])
53
54formatters = {
55    "junit" : testing.results.JUnit,
56    "text" : testing.results.Text,
57    "summary" : testing.results.TextSummary,
58    "pickle" : testing.results.Pickle,
59}
60
61
62def _add_format_args(parser):
63    parser.add_argument("--format", choices=formatters, default="text",
64                        help="Output format")
65
66    parser.add_argument("--no-junit-xlate-names", action="store_true",
67                        help="Don't translate test names to " \
68                        "package-like names")
69
70    parser.add_argument("--output", "-o",
71                        type=argparse.FileType('w'), default=sys.stdout,
72                        help="Test result output file")
73
74
75def _create_formatter(args):
76    formatter = formatters[args.format]
77    kwargs = {
78        "fout" : args.output,
79        "verbose" : args.verbose
80    }
81
82    if issubclass(formatter, testing.results.JUnit):
83        kwargs.update({
84            "translate_names" : not args.no_junit_xlate_names,
85        })
86
87    return formatter(**kwargs)
88
89
90def _list_tests_args(subparsers):
91    parser = subparsers.add_parser(
92        "list",
93        formatter_class=ParagraphHelpFormatter,
94        help="List available tests",
95        description="List available tests",
96        epilog="""
97        Generate a list of available tests using a list filter.
98
99        The filter is a string consisting of the target ISA optionally
100        followed by the test category and mode separated by
101        slashes. The test names emitted by this command can be fed
102        into the run command.
103
104        For example, to list all quick arm tests, run the following:
105        tests.py list arm/quick
106
107        Non-mandatory parts of the filter string (anything other than
108        the ISA) can be left out or replaced with the wildcard
109        character. For example, all full-system tests can be listed
110        with this command: tests.py list arm/*/fs""")
111
112    parser.add_argument("--ruby-protocol", type=str, default=None,
113                        help="Ruby protocol")
114
115    parser.add_argument("--gpu-isa", type=str, default=None,
116                        help="GPU ISA")
117
118    parser.add_argument("list_filter", metavar="ISA[/category/mode]",
119                        action="append", type=str,
120                        help="List available test cases")
121
122def _list_tests(args):
123    for isa, categories, modes in \
124        ( parse_test_filter(f) for f in args.list_filter ):
125
126        for test in get_tests(isa, categories=categories, modes=modes,
127                              ruby_protocol=args.ruby_protocol,
128                              gpu_isa=args.gpu_isa):
129            print "/".join(test)
130    sys.exit(0)
131
132def _run_tests_args(subparsers):
133    parser = subparsers.add_parser(
134        "run",
135        formatter_class=ParagraphHelpFormatter,
136        help='Run one or more tests',
137        description="Run one or more tests.",
138        epilog="""
139        Run one or more tests described by a gem5 test tuple.
140
141        The test tuple consists of a test category (quick or long), a
142        test mode (fs or se), a workload name, an isa, an operating
143        system, and a config name separate by slashes. For example:
144        quick/se/00.hello/arm/linux/simple-timing
145
146        Available tests can be listed using the 'list' sub-command
147        (e.g., "tests.py list arm/quick" or one of the scons test list
148        targets (e.g., "scons build/ARM/tests/opt/quick.list").
149
150        The test results can be stored in multiple different output
151        formats. See the help for the show command for more details
152        about output formatting.""")
153
154    parser.add_argument("gem5", type=str,
155                        help="gem5 binary")
156
157    parser.add_argument("test", type=str, nargs="*",
158                        help="List of tests to execute")
159
160    parser.add_argument("--directory", "-d",
161                        type=str, default="m5tests",
162                        help="Test work directory")
163
164    parser.add_argument("--timeout", "-t",
165                        type=int, default="0", metavar="MINUTES",
166                        help="Timeout, 0 to disable")
167
168    parser.add_argument("--skip-diff-out", action="store_true",
169                        help="Skip output diffing stage")
170
171    parser.add_argument("--skip-diff-stat", action="store_true",
172                        help="Skip stat diffing stage")
173
174    _add_format_args(parser)
175
176def _run_tests(args):
177    if not os.path.isfile(args.gem5) or not os.access(args.gem5, os.X_OK):
178        print >> sys.stderr, \
179            "gem5 binary '%s' not an executable file" % args.gem5
180        sys.exit(2)
181
182    formatter = _create_formatter(args)
183
184    out_base = os.path.abspath(args.directory)
185    if not os.path.exists(out_base):
186        os.mkdir(out_base)
187    tests = []
188    for test_name in args.test:
189        config = ClassicConfig(*test_name.split("/"))
190        out_dir = os.path.join(out_base, "/".join(config))
191        tests.append(
192            ClassicTest(args.gem5, out_dir, config,
193                        timeout=args.timeout,
194                        skip_diff_stat=args.skip_diff_stat,
195                        skip_diff_out=args.skip_diff_out))
196
197    all_results = []
198    print "Running %i tests" % len(tests)
199    for testno, test in enumerate(tests):
200        print "%i: Running '%s'..." % (testno, test)
201
202        all_results.append(test.run())
203
204    formatter.dump_suites(all_results)
205
206def _show_args(subparsers):
207    parser = subparsers.add_parser(
208        "show",
209        formatter_class=ParagraphHelpFormatter,
210        help='Display pickled test results',
211        description='Display pickled test results',
212        epilog="""
213        Reformat the pickled output from one or more test runs. This
214        command is typically used with the output from a single test
215        run, but it can also be used to merge the outputs from
216        multiple runs.
217
218        The 'text' format is a verbose output format that provides
219        information about individual test units and the output from
220        failed tests. It's mainly useful for debugging test failures.
221
222        The 'summary' format provides outputs the results of one test
223        per line with the test's overall status (OK, SKIPPED, or
224        FAILED).
225
226        The 'junit' format is primarily intended for use with CI
227        systems. It provides an XML representation of test
228        status. Similar to the text format, it includes detailed
229        information about test failures. Since many JUnit parser make
230        assume that test names look like Java packet strings, the
231        JUnit formatter automatically to something the looks like a
232        Java class path ('.'->'-', '/'->'.').
233
234        The 'pickle' format stores the raw results in a format that
235        can be reformatted using this command. It's typically used
236        with the show command to merge multiple test results into one
237        pickle file.""")
238
239    _add_format_args(parser)
240
241    parser.add_argument("result", type=argparse.FileType("rb"), nargs="*",
242                        help="Pickled test results")
243
244def _show(args):
245    def _load(f):
246        # Load the pickled status file, sometimes e.g., when a
247        # regression is still running the status file might be
248        # incomplete.
249        try:
250            return pickle.load(f)
251        except EOFError:
252            print >> sys.stderr, 'Could not read file %s' % f.name
253            return []
254
255    formatter = _create_formatter(args)
256    suites = sum([ _load(f) for f in args.result ], [])
257    formatter.dump_suites(suites)
258
259def _test_args(subparsers):
260    parser = subparsers.add_parser(
261        "test",
262        formatter_class=ParagraphHelpFormatter,
263        help='Probe test results and set exit code',
264        epilog="""
265
266        Load one or more pickled test file and return an exit code
267        corresponding to the test outcome. The following exit codes
268        can be returned:
269
270        0: All tests were successful or skipped.
271
272        1: General fault in the script such as incorrect parameters or
273        failing to parse a pickle file.
274
275        2: At least one test failed to run. This is what the summary
276        formatter usually shows as a 'FAILED'.
277
278        3: All tests ran correctly, but at least one failed to
279        verify its output. When displaying test output using the
280        summary formatter, such a test would show up as 'CHANGED'.
281        """)
282
283    _add_format_args(parser)
284
285    parser.add_argument("result", type=argparse.FileType("rb"), nargs="*",
286                        help="Pickled test results")
287
288def _test(args):
289    try:
290        suites = sum([ pickle.load(f) for f in args.result ], [])
291    except EOFError:
292        print >> sys.stderr, 'Could not read all files'
293        sys.exit(2)
294
295    if all(s for s in suites):
296        sys.exit(0)
297    elif any([ s.failed_run() for s in suites ]):
298        sys.exit(2)
299    elif any([ s.changed() for s in suites ]):
300        sys.exit(3)
301    else:
302        assert False, "Unexpected return status from test"
303
304_commands = {
305    "list" : (_list_tests, _list_tests_args),
306    "run" : (_run_tests, _run_tests_args),
307    "show" : (_show, _show_args),
308    "test" : (_test, _test_args),
309}
310
311def main():
312    parser = argparse.ArgumentParser(
313        formatter_class=ParagraphHelpFormatter,
314        description="""gem5 testing multi tool.""",
315        epilog="""
316        This tool provides an interface to gem5's test framework that
317        doesn't depend on gem5's build system. It supports test
318        listing, running, and output formatting.
319
320        The list sub-command (e.g., "test.py list arm/quick") produces
321        a list of tests tuples that can be used by the run command
322        (e.g., "tests.py run gem5.opt
323        quick/se/00.hello/arm/linux/simple-timing").
324
325        The run command supports several output formats. One of them,
326        pickle, contains the raw output from the tests and can be
327        re-formatted using the show command (e.g., "tests.py show
328        --format summary *.pickle"). Such pickle files are also
329        generated by the build system when scons is used to run
330        regressions.
331
332        See the usage strings for the individual sub-commands for
333        details.""")
334
335    parser.add_argument("--verbose", action="store_true",
336                        help="Produce more verbose output")
337
338    subparsers = parser.add_subparsers(dest="command")
339
340    for key, (impl, cmd_parser) in _commands.items():
341        cmd_parser(subparsers)
342
343    args = parser.parse_args()
344    impl, cmd_parser = _commands[args.command]
345    impl(args)
346
347if __name__ == "__main__":
348    main()
349