tests.py revision 11828:36b064696175 
1#!/usr/bin/env python2
2#
3# Copyright (c) 2016 ARM Limited
4# All rights reserved
5#
6# The license below extends only to copyright in the software and shall
7# not be construed as granting a license to any other intellectual
8# property including but not limited to intellectual property relating
9# to a hardware implementation of the functionality of the software
10# licensed hereunder.  You may use the software subject to the license
11# terms below provided that you ensure that this notice is replicated
12# unmodified and in its entirety in all distributions of the software,
13# modified or unmodified, in source code or in binary form.
14#
15# Redistribution and use in source and binary forms, with or without
16# modification, are permitted provided that the following conditions are
17# met: redistributions of source code must retain the above copyright
18# notice, this list of conditions and the following disclaimer;
19# redistributions in binary form must reproduce the above copyright
20# notice, this list of conditions and the following disclaimer in the
21# documentation and/or other materials provided with the distribution;
22# neither the name of the copyright holders nor the names of its
23# contributors may be used to endorse or promote products derived from
24# this software without specific prior written permission.
25#
26# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
27# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
28# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
29# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
30# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
31# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
32# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
33# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
34# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
35# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
36# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
37#
38# Authors: Andreas Sandberg
39
40import argparse
41import sys
42import os
43import pickle
44
45from testing.tests import *
46import testing.results
47
48class ParagraphHelpFormatter(argparse.HelpFormatter):
49    def _fill_text(self, text, width, indent):
50        return "\n\n".join([
51            super(ParagraphHelpFormatter, self)._fill_text(p, width, indent) \
52            for p in text.split("\n\n") ])
53
54formatters = {
55    "junit" : testing.results.JUnit,
56    "text" : testing.results.Text,
57    "summary" : testing.results.TextSummary,
58    "pickle" : testing.results.Pickle,
59}
60
61
62def _add_format_args(parser):
63    parser.add_argument("--format", choices=formatters, default="text",
64                        help="Output format")
65
66    parser.add_argument("--no-junit-xlate-names", action="store_true",
67                        help="Don't translate test names to " \
68                        "package-like names")
69
70    parser.add_argument("--output", "-o",
71                        type=argparse.FileType('w'), default=sys.stdout,
72                        help="Test result output file")
73
74
75def _create_formatter(args):
76    formatter = formatters[args.format]
77    kwargs = {
78        "fout" : args.output,
79        "verbose" : args.verbose
80    }
81
82    if issubclass(formatter, testing.results.JUnit):
83        kwargs.update({
84            "translate_names" : not args.no_junit_xlate_names,
85        })
86
87    return formatter(**kwargs)
88
89
90def _list_tests_args(subparsers):
91    parser = subparsers.add_parser(
92        "list",
93        formatter_class=ParagraphHelpFormatter,
94        help="List available tests",
95        description="List available tests",
96        epilog="""
97        Generate a list of available tests using a list filter.
98
99        The filter is a string consisting of the target ISA optionally
100        followed by the test category and mode separated by
101        slashes. The test names emitted by this command can be fed
102        into the run command.
103
104        For example, to list all quick arm tests, run the following:
105        tests.py list arm/quick
106
107        Non-mandatory parts of the filter string (anything other than
108        the ISA) can be left out or replaced with the wildcard
109        character. For example, all full-system tests can be listed
110        with this command: tests.py list arm/*/fs""")
111
112    parser.add_argument("--ruby-protocol", type=str, default=None,
113                        help="Ruby protocol")
114
115    parser.add_argument("--gpu-isa", type=str, default=None,
116                        help="GPU ISA")
117
118    parser.add_argument("list_filter", metavar="ISA[/category/mode]",
119                        action="append", type=str,
120                        help="List available test cases")
121
122def _list_tests(args):
123    for isa, categories, modes in \
124        ( parse_test_filter(f) for f in args.list_filter ):
125
126        for test in get_tests(isa, categories=categories, modes=modes,
127                              ruby_protocol=args.ruby_protocol,
128                              gpu_isa=args.gpu_isa):
129            print "/".join(test)
130    sys.exit(0)
131
132def _run_tests_args(subparsers):
133    parser = subparsers.add_parser(
134        "run",
135        formatter_class=ParagraphHelpFormatter,
136        help='Run one or more tests',
137        description="Run one or more tests.",
138        epilog="""
139        Run one or more tests described by a gem5 test tuple.
140
141        The test tuple consists of a test category (quick or long), a
142        test mode (fs or se), a workload name, an isa, an operating
143        system, and a config name separate by slashes. For example:
144        quick/se/00.hello/arm/linux/simple-timing
145
146        Available tests can be listed using the 'list' sub-command
147        (e.g., "tests.py list arm/quick" or one of the scons test list
148        targets (e.g., "scons build/ARM/tests/opt/quick.list").
149
150        The test results can be stored in multiple different output
151        formats. See the help for the show command for more details
152        about output formatting.""")
153
154    parser.add_argument("gem5", type=str,
155                        help="gem5 binary")
156
157    parser.add_argument("test", type=str, nargs="*",
158                        help="List of tests to execute")
159
160    parser.add_argument("--directory", "-d",
161                        type=str, default="m5tests",
162                        help="Test work directory")
163
164    parser.add_argument("--timeout", "-t",
165                        type=int, default="0", metavar="MINUTES",
166                        help="Timeout, 0 to disable")
167
168    parser.add_argument("--skip-diff-out", action="store_true",
169                        help="Skip output diffing stage")
170
171    parser.add_argument("--skip-diff-stat", action="store_true",
172                        help="Skip stat diffing stage")
173
174    _add_format_args(parser)
175
176def _run_tests(args):
177    formatter = _create_formatter(args)
178
179    out_base = os.path.abspath(args.directory)
180    if not os.path.exists(out_base):
181        os.mkdir(out_base)
182    tests = []
183    for test_name in args.test:
184        config = ClassicConfig(*test_name.split("/"))
185        out_dir = os.path.join(out_base, "/".join(config))
186        tests.append(
187            ClassicTest(args.gem5, out_dir, config,
188                        timeout=args.timeout,
189                        skip_diff_stat=args.skip_diff_stat,
190                        skip_diff_out=args.skip_diff_out))
191
192    all_results = []
193    print "Running %i tests" % len(tests)
194    for testno, test in enumerate(tests):
195        print "%i: Running '%s'..." % (testno, test)
196
197        all_results.append(test.run())
198
199    formatter.dump_suites(all_results)
200
201def _show_args(subparsers):
202    parser = subparsers.add_parser(
203        "show",
204        formatter_class=ParagraphHelpFormatter,
205        help='Display pickled test results',
206        description='Display pickled test results',
207        epilog="""
208        Reformat the pickled output from one or more test runs. This
209        command is typically used with the output from a single test
210        run, but it can also be used to merge the outputs from
211        multiple runs.
212
213        The 'text' format is a verbose output format that provides
214        information about individual test units and the output from
215        failed tests. It's mainly useful for debugging test failures.
216
217        The 'summary' format provides outputs the results of one test
218        per line with the test's overall status (OK, SKIPPED, or
219        FAILED).
220
221        The 'junit' format is primarily intended for use with CI
222        systems. It provides an XML representation of test
223        status. Similar to the text format, it includes detailed
224        information about test failures. Since many JUnit parser make
225        assume that test names look like Java packet strings, the
226        JUnit formatter automatically to something the looks like a
227        Java class path ('.'->'-', '/'->'.').
228
229        The 'pickle' format stores the raw results in a format that
230        can be reformatted using this command. It's typically used
231        with the show command to merge multiple test results into one
232        pickle file.""")
233
234    _add_format_args(parser)
235
236    parser.add_argument("result", type=argparse.FileType("rb"), nargs="*",
237                        help="Pickled test results")
238
239def _show(args):
240    formatter = _create_formatter(args)
241    suites = sum([ pickle.load(f) for f in args.result ], [])
242    formatter.dump_suites(suites)
243
244def _test_args(subparsers):
245    parser = subparsers.add_parser(
246        "test",
247        formatter_class=ParagraphHelpFormatter,
248        help='Probe test results and set exit code',
249        epilog="""
250
251        Load one or more pickled test file and return an exit code
252        corresponding to the test outcome. The following exit codes
253        can be returned:
254
255        0: All tests were successful or skipped.
256
257        1: General fault in the script such as incorrect parameters or
258        failing to parse a pickle file.
259
260        2: At least one test failed to run. This is what the summary
261        formatter usually shows as a 'FAILED'.
262
263        3: All tests ran correctly, but at least one failed to
264        verify its output. When displaying test output using the
265        summary formatter, such a test would show up as 'CHANGED'.
266        """)
267
268    _add_format_args(parser)
269
270    parser.add_argument("result", type=argparse.FileType("rb"), nargs="*",
271                        help="Pickled test results")
272
273def _test(args):
274    suites = sum([ pickle.load(f) for f in args.result ], [])
275
276    if all(s for s in suites):
277        sys.exit(0)
278    elif any([ s.failed_run() for s in suites ]):
279        sys.exit(2)
280    elif any([ s.changed() for s in suites ]):
281        sys.exit(3)
282    else:
283        assert False, "Unexpected return status from test"
284
285_commands = {
286    "list" : (_list_tests, _list_tests_args),
287    "run" : (_run_tests, _run_tests_args),
288    "show" : (_show, _show_args),
289    "test" : (_test, _test_args),
290}
291
292def main():
293    parser = argparse.ArgumentParser(
294        formatter_class=ParagraphHelpFormatter,
295        description="""gem5 testing multi tool.""",
296        epilog="""
297        This tool provides an interface to gem5's test framework that
298        doesn't depend on gem5's build system. It supports test
299        listing, running, and output formatting.
300
301        The list sub-command (e.g., "test.py list arm/quick") produces
302        a list of tests tuples that can be used by the run command
303        (e.g., "tests.py run gem5.opt
304        quick/se/00.hello/arm/linux/simple-timing").
305
306        The run command supports several output formats. One of them,
307        pickle, contains the raw output from the tests and can be
308        re-formatted using the show command (e.g., "tests.py show
309        --format summary *.pickle"). Such pickle files are also
310        generated by the build system when scons is used to run
311        regressions.
312
313        See the usage strings for the individual sub-commands for
314        details.""")
315
316    parser.add_argument("--verbose", action="store_true",
317                        help="Produce more verbose output")
318
319    subparsers = parser.add_subparsers(dest="command")
320
321    for key, (impl, cmd_parser) in _commands.items():
322        cmd_parser(subparsers)
323
324    args = parser.parse_args()
325    impl, cmd_parser = _commands[args.command]
326    impl(args)
327
328if __name__ == "__main__":
329    main()
330