1#!/usr/bin/env python
2#
3# Copyright (c) 2016 ARM Limited
4# All rights reserved
5#
6# The license below extends only to copyright in the software and shall
7# not be construed as granting a license to any other intellectual
8# property including but not limited to intellectual property relating
9# to a hardware implementation of the functionality of the software
10# licensed hereunder. You may use the software subject to the license
11# terms below provided that you ensure that this notice is replicated
12# unmodified and in its entirety in all distributions of the software,
13# modified or unmodified, in source code or in binary form.
14#
15# Redistribution and use in source and binary forms, with or without
16# modification, are permitted provided that the following conditions are
17# met: redistributions of source code must retain the above copyright
18# notice, this list of conditions and the following disclaimer;
19# redistributions in binary form must reproduce the above copyright
20# notice, this list of conditions and the following disclaimer in the
21# documentation and/or other materials provided with the distribution;
22# neither the name of the copyright holders nor the names of its
23# contributors may be used to endorse or promote products derived from
24# this software without specific prior written permission.
25#
26# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
27# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
28# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
29# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
30# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
31# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
32# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
33# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
34# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
35# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
36# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
37#
38# Authors: Andreas Sandberg
39
40import argparse
41import sys
42import os
43import pickle
44
45from testing.tests import *
46import testing.results
47
48class ParagraphHelpFormatter(argparse.HelpFormatter):
49 def _fill_text(self, text, width, indent):
50 return "\n\n".join([
51 super(ParagraphHelpFormatter, self)._fill_text(p, width, indent) \
52 for p in text.split("\n\n") ])
53
54formatters = {
55 "junit" : testing.results.JUnit,
56 "text" : testing.results.Text,
57 "summary" : testing.results.TextSummary,
58 "pickle" : testing.results.Pickle,
59}
60
61
62def _add_format_args(parser):
63 parser.add_argument("--format", choices=formatters, default="text",
64 help="Output format")
65
66 parser.add_argument("--no-junit-xlate-names", action="store_true",
67 help="Don't translate test names to " \
68 "package-like names")
69
70 parser.add_argument("--output", "-o",
71 type=argparse.FileType('w'), default=sys.stdout,
72 help="Test result output file")
73
74
75def _create_formatter(args):
76 formatter = formatters[args.format]
77 kwargs = {
78 "fout" : args.output,
79 "verbose" : args.verbose
80 }
81
82 if issubclass(formatter, testing.results.JUnit):
83 kwargs.update({
84 "translate_names" : not args.no_junit_xlate_names,
85 })
86
87 return formatter(**kwargs)
88
89
90def _list_tests_args(subparsers):
91 parser = subparsers.add_parser(
92 "list",
93 formatter_class=ParagraphHelpFormatter,
94 help="List available tests",
95 description="List available tests",
96 epilog="""
97 Generate a list of available tests using a list filter.
98
99 The filter is a string consisting of the target ISA optionally
100 followed by the test category and mode separated by
101 slashes. The test names emitted by this command can be fed
102 into the run command.
103
104 For example, to list all quick arm tests, run the following:
105 tests.py list arm/quick
106
107 Non-mandatory parts of the filter string (anything other than
108 the ISA) can be left out or replaced with the wildcard
109 character. For example, all full-system tests can be listed
110 with this command: tests.py list arm/*/fs""")
111
112 parser.add_argument("--ruby-protocol", type=str, default=None,
113 help="Ruby protocol")
114
115 parser.add_argument("--gpu-isa", type=str, default=None,
116 help="GPU ISA")
117
118 parser.add_argument("list_filter", metavar="ISA[/category/mode]",
119 action="append", type=str,
120 help="List available test cases")
121
122def _list_tests(args):
123 for isa, categories, modes in \
124 ( parse_test_filter(f) for f in args.list_filter ):
125
126 for test in get_tests(isa, categories=categories, modes=modes,
127 ruby_protocol=args.ruby_protocol,
128 gpu_isa=args.gpu_isa):
129 print "/".join(test)
130 sys.exit(0)
131
132def _run_tests_args(subparsers):
133 parser = subparsers.add_parser(
134 "run",
135 formatter_class=ParagraphHelpFormatter,
136 help='Run one or more tests',
137 description="Run one or more tests.",
138 epilog="""
139 Run one or more tests described by a gem5 test tuple.
140
141 The test tuple consists of a test category (quick or long), a
142 test mode (fs or se), a workload name, an isa, an operating
143 system, and a config name separate by slashes. For example:
144 quick/se/00.hello/arm/linux/simple-timing
145
146 Available tests can be listed using the 'list' sub-command
147 (e.g., "tests.py list arm/quick" or one of the scons test list
148 targets (e.g., "scons build/ARM/tests/opt/quick.list").
149
150 The test results can be stored in multiple different output
151 formats. See the help for the show command for more details
152 about output formatting.""")
153
154 parser.add_argument("gem5", type=str,
155 help="gem5 binary")
156
157 parser.add_argument("test", type=str, nargs="*",
158 help="List of tests to execute")
159
160 parser.add_argument("--directory", "-d",
161 type=str, default="m5tests",
162 help="Test work directory")
163
164 parser.add_argument("--timeout", "-t",
165 type=int, default="0", metavar="MINUTES",
166 help="Timeout, 0 to disable")
167
168 parser.add_argument("--skip-diff-out", action="store_true",
169 help="Skip output diffing stage")
170
171 parser.add_argument("--skip-diff-stat", action="store_true",
172 help="Skip stat diffing stage")
173
174 _add_format_args(parser)
175
176def _run_tests(args):
177 formatter = _create_formatter(args)
178
179 out_base = os.path.abspath(args.directory)
180 if not os.path.exists(out_base):
181 os.mkdir(out_base)
182 tests = []
183 for test_name in args.test:
184 config = ClassicConfig(*test_name.split("/"))
185 out_dir = os.path.join(out_base, "/".join(config))
186 tests.append(
187 ClassicTest(args.gem5, out_dir, config,
188 timeout=args.timeout,
189 skip_diff_stat=args.skip_diff_stat,
190 skip_diff_out=args.skip_diff_out))
191
192 all_results = []
193 print "Running %i tests" % len(tests)
194 for testno, test in enumerate(tests):
195 print "%i: Running '%s'..." % (testno, test)
196
197 all_results.append(test.run())
198
199 formatter.dump_suites(all_results)
200
201def _show_args(subparsers):
202 parser = subparsers.add_parser(
203 "show",
204 formatter_class=ParagraphHelpFormatter,
205 help='Display pickled test results',
206 description='Display pickled test results',
207 epilog="""
208 Reformat the pickled output from one or more test runs. This
209 command is typically used with the output from a single test
210 run, but it can also be used to merge the outputs from
211 multiple runs.
212
213 The 'text' format is a verbose output format that provides
214 information about individual test units and the output from
215 failed tests. It's mainly useful for debugging test failures.
216
217 The 'summary' format provides outputs the results of one test
218 per line with the test's overall status (OK, SKIPPED, or
219 FAILED).
220
221 The 'junit' format is primarily intended for use with CI
222 systems. It provides an XML representation of test
223 status. Similar to the text format, it includes detailed
224 information about test failures. Since many JUnit parser make
225 assume that test names look like Java packet strings, the
226 JUnit formatter automatically to something the looks like a
227 Java class path ('.'->'-', '/'->'.').
228
229 The 'pickle' format stores the raw results in a format that
230 can be reformatted using this command. It's typically used
231 with the show command to merge multiple test results into one
232 pickle file.""")
233
234 _add_format_args(parser)
235
236 parser.add_argument("result", type=argparse.FileType("rb"), nargs="*",
237 help="Pickled test results")
238
239def _show(args):
240 formatter = _create_formatter(args)
241 suites = sum([ pickle.load(f) for f in args.result ], [])
242 formatter.dump_suites(suites)
243
| 1#!/usr/bin/env python
2#
3# Copyright (c) 2016 ARM Limited
4# All rights reserved
5#
6# The license below extends only to copyright in the software and shall
7# not be construed as granting a license to any other intellectual
8# property including but not limited to intellectual property relating
9# to a hardware implementation of the functionality of the software
10# licensed hereunder. You may use the software subject to the license
11# terms below provided that you ensure that this notice is replicated
12# unmodified and in its entirety in all distributions of the software,
13# modified or unmodified, in source code or in binary form.
14#
15# Redistribution and use in source and binary forms, with or without
16# modification, are permitted provided that the following conditions are
17# met: redistributions of source code must retain the above copyright
18# notice, this list of conditions and the following disclaimer;
19# redistributions in binary form must reproduce the above copyright
20# notice, this list of conditions and the following disclaimer in the
21# documentation and/or other materials provided with the distribution;
22# neither the name of the copyright holders nor the names of its
23# contributors may be used to endorse or promote products derived from
24# this software without specific prior written permission.
25#
26# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
27# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
28# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
29# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
30# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
31# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
32# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
33# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
34# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
35# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
36# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
37#
38# Authors: Andreas Sandberg
39
40import argparse
41import sys
42import os
43import pickle
44
45from testing.tests import *
46import testing.results
47
48class ParagraphHelpFormatter(argparse.HelpFormatter):
49 def _fill_text(self, text, width, indent):
50 return "\n\n".join([
51 super(ParagraphHelpFormatter, self)._fill_text(p, width, indent) \
52 for p in text.split("\n\n") ])
53
54formatters = {
55 "junit" : testing.results.JUnit,
56 "text" : testing.results.Text,
57 "summary" : testing.results.TextSummary,
58 "pickle" : testing.results.Pickle,
59}
60
61
62def _add_format_args(parser):
63 parser.add_argument("--format", choices=formatters, default="text",
64 help="Output format")
65
66 parser.add_argument("--no-junit-xlate-names", action="store_true",
67 help="Don't translate test names to " \
68 "package-like names")
69
70 parser.add_argument("--output", "-o",
71 type=argparse.FileType('w'), default=sys.stdout,
72 help="Test result output file")
73
74
75def _create_formatter(args):
76 formatter = formatters[args.format]
77 kwargs = {
78 "fout" : args.output,
79 "verbose" : args.verbose
80 }
81
82 if issubclass(formatter, testing.results.JUnit):
83 kwargs.update({
84 "translate_names" : not args.no_junit_xlate_names,
85 })
86
87 return formatter(**kwargs)
88
89
90def _list_tests_args(subparsers):
91 parser = subparsers.add_parser(
92 "list",
93 formatter_class=ParagraphHelpFormatter,
94 help="List available tests",
95 description="List available tests",
96 epilog="""
97 Generate a list of available tests using a list filter.
98
99 The filter is a string consisting of the target ISA optionally
100 followed by the test category and mode separated by
101 slashes. The test names emitted by this command can be fed
102 into the run command.
103
104 For example, to list all quick arm tests, run the following:
105 tests.py list arm/quick
106
107 Non-mandatory parts of the filter string (anything other than
108 the ISA) can be left out or replaced with the wildcard
109 character. For example, all full-system tests can be listed
110 with this command: tests.py list arm/*/fs""")
111
112 parser.add_argument("--ruby-protocol", type=str, default=None,
113 help="Ruby protocol")
114
115 parser.add_argument("--gpu-isa", type=str, default=None,
116 help="GPU ISA")
117
118 parser.add_argument("list_filter", metavar="ISA[/category/mode]",
119 action="append", type=str,
120 help="List available test cases")
121
122def _list_tests(args):
123 for isa, categories, modes in \
124 ( parse_test_filter(f) for f in args.list_filter ):
125
126 for test in get_tests(isa, categories=categories, modes=modes,
127 ruby_protocol=args.ruby_protocol,
128 gpu_isa=args.gpu_isa):
129 print "/".join(test)
130 sys.exit(0)
131
132def _run_tests_args(subparsers):
133 parser = subparsers.add_parser(
134 "run",
135 formatter_class=ParagraphHelpFormatter,
136 help='Run one or more tests',
137 description="Run one or more tests.",
138 epilog="""
139 Run one or more tests described by a gem5 test tuple.
140
141 The test tuple consists of a test category (quick or long), a
142 test mode (fs or se), a workload name, an isa, an operating
143 system, and a config name separate by slashes. For example:
144 quick/se/00.hello/arm/linux/simple-timing
145
146 Available tests can be listed using the 'list' sub-command
147 (e.g., "tests.py list arm/quick" or one of the scons test list
148 targets (e.g., "scons build/ARM/tests/opt/quick.list").
149
150 The test results can be stored in multiple different output
151 formats. See the help for the show command for more details
152 about output formatting.""")
153
154 parser.add_argument("gem5", type=str,
155 help="gem5 binary")
156
157 parser.add_argument("test", type=str, nargs="*",
158 help="List of tests to execute")
159
160 parser.add_argument("--directory", "-d",
161 type=str, default="m5tests",
162 help="Test work directory")
163
164 parser.add_argument("--timeout", "-t",
165 type=int, default="0", metavar="MINUTES",
166 help="Timeout, 0 to disable")
167
168 parser.add_argument("--skip-diff-out", action="store_true",
169 help="Skip output diffing stage")
170
171 parser.add_argument("--skip-diff-stat", action="store_true",
172 help="Skip stat diffing stage")
173
174 _add_format_args(parser)
175
176def _run_tests(args):
177 formatter = _create_formatter(args)
178
179 out_base = os.path.abspath(args.directory)
180 if not os.path.exists(out_base):
181 os.mkdir(out_base)
182 tests = []
183 for test_name in args.test:
184 config = ClassicConfig(*test_name.split("/"))
185 out_dir = os.path.join(out_base, "/".join(config))
186 tests.append(
187 ClassicTest(args.gem5, out_dir, config,
188 timeout=args.timeout,
189 skip_diff_stat=args.skip_diff_stat,
190 skip_diff_out=args.skip_diff_out))
191
192 all_results = []
193 print "Running %i tests" % len(tests)
194 for testno, test in enumerate(tests):
195 print "%i: Running '%s'..." % (testno, test)
196
197 all_results.append(test.run())
198
199 formatter.dump_suites(all_results)
200
201def _show_args(subparsers):
202 parser = subparsers.add_parser(
203 "show",
204 formatter_class=ParagraphHelpFormatter,
205 help='Display pickled test results',
206 description='Display pickled test results',
207 epilog="""
208 Reformat the pickled output from one or more test runs. This
209 command is typically used with the output from a single test
210 run, but it can also be used to merge the outputs from
211 multiple runs.
212
213 The 'text' format is a verbose output format that provides
214 information about individual test units and the output from
215 failed tests. It's mainly useful for debugging test failures.
216
217 The 'summary' format provides outputs the results of one test
218 per line with the test's overall status (OK, SKIPPED, or
219 FAILED).
220
221 The 'junit' format is primarily intended for use with CI
222 systems. It provides an XML representation of test
223 status. Similar to the text format, it includes detailed
224 information about test failures. Since many JUnit parser make
225 assume that test names look like Java packet strings, the
226 JUnit formatter automatically to something the looks like a
227 Java class path ('.'->'-', '/'->'.').
228
229 The 'pickle' format stores the raw results in a format that
230 can be reformatted using this command. It's typically used
231 with the show command to merge multiple test results into one
232 pickle file.""")
233
234 _add_format_args(parser)
235
236 parser.add_argument("result", type=argparse.FileType("rb"), nargs="*",
237 help="Pickled test results")
238
239def _show(args):
240 formatter = _create_formatter(args)
241 suites = sum([ pickle.load(f) for f in args.result ], [])
242 formatter.dump_suites(suites)
243
|