1#!/usr/bin/env python2 2# 3# Copyright (c) 2016 ARM Limited 4# All rights reserved 5# 6# The license below extends only to copyright in the software and shall 7# not be construed as granting a license to any other intellectual 8# property including but not limited to intellectual property relating 9# to a hardware implementation of the functionality of the software 10# licensed hereunder. You may use the software subject to the license 11# terms below provided that you ensure that this notice is replicated 12# unmodified and in its entirety in all distributions of the software, 13# modified or unmodified, in source code or in binary form. 14# 15# Redistribution and use in source and binary forms, with or without 16# modification, are permitted provided that the following conditions are 17# met: redistributions of source code must retain the above copyright 18# notice, this list of conditions and the following disclaimer; 19# redistributions in binary form must reproduce the above copyright 20# notice, this list of conditions and the following disclaimer in the 21# documentation and/or other materials provided with the distribution; 22# neither the name of the copyright holders nor the names of its 23# contributors may be used to endorse or promote products derived from 24# this software without specific prior written permission. 25# 26# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS 27# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT 28# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR 29# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT 30# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, 31# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT 32# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, 33# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY 34# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT 35# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE 36# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 37# 38# Authors: Andreas Sandberg 39
| 1#!/usr/bin/env python2 2# 3# Copyright (c) 2016 ARM Limited 4# All rights reserved 5# 6# The license below extends only to copyright in the software and shall 7# not be construed as granting a license to any other intellectual 8# property including but not limited to intellectual property relating 9# to a hardware implementation of the functionality of the software 10# licensed hereunder. You may use the software subject to the license 11# terms below provided that you ensure that this notice is replicated 12# unmodified and in its entirety in all distributions of the software, 13# modified or unmodified, in source code or in binary form. 14# 15# Redistribution and use in source and binary forms, with or without 16# modification, are permitted provided that the following conditions are 17# met: redistributions of source code must retain the above copyright 18# notice, this list of conditions and the following disclaimer; 19# redistributions in binary form must reproduce the above copyright 20# notice, this list of conditions and the following disclaimer in the 21# documentation and/or other materials provided with the distribution; 22# neither the name of the copyright holders nor the names of its 23# contributors may be used to endorse or promote products derived from 24# this software without specific prior written permission. 25# 26# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS 27# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT 28# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR 29# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT 30# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, 31# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT 32# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, 33# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY 34# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT 35# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE 36# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 37# 38# Authors: Andreas Sandberg 39
|
| 40from __future__ import print_function 41
|
40import argparse 41import sys 42import os 43import pickle 44 45from testing.tests import * 46import testing.results 47 48class ParagraphHelpFormatter(argparse.HelpFormatter): 49 def _fill_text(self, text, width, indent): 50 return "\n\n".join([ 51 super(ParagraphHelpFormatter, self)._fill_text(p, width, indent) \ 52 for p in text.split("\n\n") ]) 53 54formatters = { 55 "junit" : testing.results.JUnit, 56 "text" : testing.results.Text, 57 "summary" : testing.results.TextSummary, 58 "pickle" : testing.results.Pickle, 59} 60 61 62def _add_format_args(parser): 63 parser.add_argument("--format", choices=formatters, default="text", 64 help="Output format") 65 66 parser.add_argument("--no-junit-xlate-names", action="store_true", 67 help="Don't translate test names to " \ 68 "package-like names") 69 70 parser.add_argument("--output", "-o", 71 type=argparse.FileType('w'), default=sys.stdout, 72 help="Test result output file") 73 74 75def _create_formatter(args): 76 formatter = formatters[args.format] 77 kwargs = { 78 "fout" : args.output, 79 "verbose" : args.verbose 80 } 81 82 if issubclass(formatter, testing.results.JUnit): 83 kwargs.update({ 84 "translate_names" : not args.no_junit_xlate_names, 85 }) 86 87 return formatter(**kwargs) 88 89 90def _list_tests_args(subparsers): 91 parser = subparsers.add_parser( 92 "list", 93 formatter_class=ParagraphHelpFormatter, 94 help="List available tests", 95 description="List available tests", 96 epilog=""" 97 Generate a list of available tests using a list filter. 98 99 The filter is a string consisting of the target ISA optionally 100 followed by the test category and mode separated by 101 slashes. The test names emitted by this command can be fed 102 into the run command. 103 104 For example, to list all quick arm tests, run the following: 105 tests.py list arm/quick 106 107 Non-mandatory parts of the filter string (anything other than 108 the ISA) can be left out or replaced with the wildcard 109 character. For example, all full-system tests can be listed 110 with this command: tests.py list arm/*/fs""") 111 112 parser.add_argument("--ruby-protocol", type=str, default=None, 113 help="Ruby protocol") 114 115 parser.add_argument("--gpu-isa", type=str, default=None, 116 help="GPU ISA") 117 118 parser.add_argument("list_filter", metavar="ISA[/category/mode]", 119 action="append", type=str, 120 help="List available test cases") 121 122def _list_tests(args): 123 for isa, categories, modes in \ 124 ( parse_test_filter(f) for f in args.list_filter ): 125 126 for test in get_tests(isa, categories=categories, modes=modes, 127 ruby_protocol=args.ruby_protocol, 128 gpu_isa=args.gpu_isa):
| 42import argparse 43import sys 44import os 45import pickle 46 47from testing.tests import * 48import testing.results 49 50class ParagraphHelpFormatter(argparse.HelpFormatter): 51 def _fill_text(self, text, width, indent): 52 return "\n\n".join([ 53 super(ParagraphHelpFormatter, self)._fill_text(p, width, indent) \ 54 for p in text.split("\n\n") ]) 55 56formatters = { 57 "junit" : testing.results.JUnit, 58 "text" : testing.results.Text, 59 "summary" : testing.results.TextSummary, 60 "pickle" : testing.results.Pickle, 61} 62 63 64def _add_format_args(parser): 65 parser.add_argument("--format", choices=formatters, default="text", 66 help="Output format") 67 68 parser.add_argument("--no-junit-xlate-names", action="store_true", 69 help="Don't translate test names to " \ 70 "package-like names") 71 72 parser.add_argument("--output", "-o", 73 type=argparse.FileType('w'), default=sys.stdout, 74 help="Test result output file") 75 76 77def _create_formatter(args): 78 formatter = formatters[args.format] 79 kwargs = { 80 "fout" : args.output, 81 "verbose" : args.verbose 82 } 83 84 if issubclass(formatter, testing.results.JUnit): 85 kwargs.update({ 86 "translate_names" : not args.no_junit_xlate_names, 87 }) 88 89 return formatter(**kwargs) 90 91 92def _list_tests_args(subparsers): 93 parser = subparsers.add_parser( 94 "list", 95 formatter_class=ParagraphHelpFormatter, 96 help="List available tests", 97 description="List available tests", 98 epilog=""" 99 Generate a list of available tests using a list filter. 100 101 The filter is a string consisting of the target ISA optionally 102 followed by the test category and mode separated by 103 slashes. The test names emitted by this command can be fed 104 into the run command. 105 106 For example, to list all quick arm tests, run the following: 107 tests.py list arm/quick 108 109 Non-mandatory parts of the filter string (anything other than 110 the ISA) can be left out or replaced with the wildcard 111 character. For example, all full-system tests can be listed 112 with this command: tests.py list arm/*/fs""") 113 114 parser.add_argument("--ruby-protocol", type=str, default=None, 115 help="Ruby protocol") 116 117 parser.add_argument("--gpu-isa", type=str, default=None, 118 help="GPU ISA") 119 120 parser.add_argument("list_filter", metavar="ISA[/category/mode]", 121 action="append", type=str, 122 help="List available test cases") 123 124def _list_tests(args): 125 for isa, categories, modes in \ 126 ( parse_test_filter(f) for f in args.list_filter ): 127 128 for test in get_tests(isa, categories=categories, modes=modes, 129 ruby_protocol=args.ruby_protocol, 130 gpu_isa=args.gpu_isa):
|
129 print "/".join(test)
| 131 print("/".join(test))
|
130 sys.exit(0) 131 132def _run_tests_args(subparsers): 133 parser = subparsers.add_parser( 134 "run", 135 formatter_class=ParagraphHelpFormatter, 136 help='Run one or more tests', 137 description="Run one or more tests.", 138 epilog=""" 139 Run one or more tests described by a gem5 test tuple. 140 141 The test tuple consists of a test category (quick or long), a 142 test mode (fs or se), a workload name, an isa, an operating 143 system, and a config name separate by slashes. For example: 144 quick/se/00.hello/arm/linux/simple-timing 145 146 Available tests can be listed using the 'list' sub-command 147 (e.g., "tests.py list arm/quick" or one of the scons test list 148 targets (e.g., "scons build/ARM/tests/opt/quick.list"). 149 150 The test results can be stored in multiple different output 151 formats. See the help for the show command for more details 152 about output formatting.""") 153 154 parser.add_argument("gem5", type=str, 155 help="gem5 binary") 156 157 parser.add_argument("test", type=str, nargs="*", 158 help="List of tests to execute") 159 160 parser.add_argument("--directory", "-d", 161 type=str, default="m5tests", 162 help="Test work directory") 163 164 parser.add_argument("--timeout", "-t", 165 type=int, default="0", metavar="MINUTES", 166 help="Timeout, 0 to disable") 167 168 parser.add_argument("--skip-diff-out", action="store_true", 169 help="Skip output diffing stage") 170 171 parser.add_argument("--skip-diff-stat", action="store_true", 172 help="Skip stat diffing stage") 173 174 _add_format_args(parser) 175 176def _run_tests(args): 177 if not os.path.isfile(args.gem5) or not os.access(args.gem5, os.X_OK):
| 132 sys.exit(0) 133 134def _run_tests_args(subparsers): 135 parser = subparsers.add_parser( 136 "run", 137 formatter_class=ParagraphHelpFormatter, 138 help='Run one or more tests', 139 description="Run one or more tests.", 140 epilog=""" 141 Run one or more tests described by a gem5 test tuple. 142 143 The test tuple consists of a test category (quick or long), a 144 test mode (fs or se), a workload name, an isa, an operating 145 system, and a config name separate by slashes. For example: 146 quick/se/00.hello/arm/linux/simple-timing 147 148 Available tests can be listed using the 'list' sub-command 149 (e.g., "tests.py list arm/quick" or one of the scons test list 150 targets (e.g., "scons build/ARM/tests/opt/quick.list"). 151 152 The test results can be stored in multiple different output 153 formats. See the help for the show command for more details 154 about output formatting.""") 155 156 parser.add_argument("gem5", type=str, 157 help="gem5 binary") 158 159 parser.add_argument("test", type=str, nargs="*", 160 help="List of tests to execute") 161 162 parser.add_argument("--directory", "-d", 163 type=str, default="m5tests", 164 help="Test work directory") 165 166 parser.add_argument("--timeout", "-t", 167 type=int, default="0", metavar="MINUTES", 168 help="Timeout, 0 to disable") 169 170 parser.add_argument("--skip-diff-out", action="store_true", 171 help="Skip output diffing stage") 172 173 parser.add_argument("--skip-diff-stat", action="store_true", 174 help="Skip stat diffing stage") 175 176 _add_format_args(parser) 177 178def _run_tests(args): 179 if not os.path.isfile(args.gem5) or not os.access(args.gem5, os.X_OK):
|
178 print >> sys.stderr, \ 179 "gem5 binary '%s' not an executable file" % args.gem5
| 180 print("gem5 binary '%s' not an executable file" % args.gem5, 181 file=sys.stderr)
|
180 sys.exit(2) 181 182 formatter = _create_formatter(args) 183 184 out_base = os.path.abspath(args.directory) 185 if not os.path.exists(out_base): 186 os.mkdir(out_base) 187 tests = [] 188 for test_name in args.test: 189 config = ClassicConfig(*test_name.split("/")) 190 out_dir = os.path.join(out_base, "/".join(config)) 191 tests.append( 192 ClassicTest(args.gem5, out_dir, config, 193 timeout=args.timeout, 194 skip_diff_stat=args.skip_diff_stat, 195 skip_diff_out=args.skip_diff_out)) 196 197 all_results = []
| 182 sys.exit(2) 183 184 formatter = _create_formatter(args) 185 186 out_base = os.path.abspath(args.directory) 187 if not os.path.exists(out_base): 188 os.mkdir(out_base) 189 tests = [] 190 for test_name in args.test: 191 config = ClassicConfig(*test_name.split("/")) 192 out_dir = os.path.join(out_base, "/".join(config)) 193 tests.append( 194 ClassicTest(args.gem5, out_dir, config, 195 timeout=args.timeout, 196 skip_diff_stat=args.skip_diff_stat, 197 skip_diff_out=args.skip_diff_out)) 198 199 all_results = []
|
198 print "Running %i tests" % len(tests)
| 200 print("Running %i tests" % len(tests))
|
199 for testno, test in enumerate(tests):
| 201 for testno, test in enumerate(tests):
|
200 print "%i: Running '%s'..." % (testno, test)
| 202 print("%i: Running '%s'..." % (testno, test))
|
201 202 all_results.append(test.run()) 203 204 formatter.dump_suites(all_results) 205 206def _show_args(subparsers): 207 parser = subparsers.add_parser( 208 "show", 209 formatter_class=ParagraphHelpFormatter, 210 help='Display pickled test results', 211 description='Display pickled test results', 212 epilog=""" 213 Reformat the pickled output from one or more test runs. This 214 command is typically used with the output from a single test 215 run, but it can also be used to merge the outputs from 216 multiple runs. 217 218 The 'text' format is a verbose output format that provides 219 information about individual test units and the output from 220 failed tests. It's mainly useful for debugging test failures. 221 222 The 'summary' format provides outputs the results of one test 223 per line with the test's overall status (OK, SKIPPED, or 224 FAILED). 225 226 The 'junit' format is primarily intended for use with CI 227 systems. It provides an XML representation of test 228 status. Similar to the text format, it includes detailed 229 information about test failures. Since many JUnit parser make 230 assume that test names look like Java packet strings, the 231 JUnit formatter automatically to something the looks like a 232 Java class path ('.'->'-', '/'->'.'). 233 234 The 'pickle' format stores the raw results in a format that 235 can be reformatted using this command. It's typically used 236 with the show command to merge multiple test results into one 237 pickle file.""") 238 239 _add_format_args(parser) 240 241 parser.add_argument("result", type=argparse.FileType("rb"), nargs="*", 242 help="Pickled test results") 243 244def _show(args): 245 def _load(f): 246 # Load the pickled status file, sometimes e.g., when a 247 # regression is still running the status file might be 248 # incomplete. 249 try: 250 return pickle.load(f) 251 except EOFError:
| 203 204 all_results.append(test.run()) 205 206 formatter.dump_suites(all_results) 207 208def _show_args(subparsers): 209 parser = subparsers.add_parser( 210 "show", 211 formatter_class=ParagraphHelpFormatter, 212 help='Display pickled test results', 213 description='Display pickled test results', 214 epilog=""" 215 Reformat the pickled output from one or more test runs. This 216 command is typically used with the output from a single test 217 run, but it can also be used to merge the outputs from 218 multiple runs. 219 220 The 'text' format is a verbose output format that provides 221 information about individual test units and the output from 222 failed tests. It's mainly useful for debugging test failures. 223 224 The 'summary' format provides outputs the results of one test 225 per line with the test's overall status (OK, SKIPPED, or 226 FAILED). 227 228 The 'junit' format is primarily intended for use with CI 229 systems. It provides an XML representation of test 230 status. Similar to the text format, it includes detailed 231 information about test failures. Since many JUnit parser make 232 assume that test names look like Java packet strings, the 233 JUnit formatter automatically to something the looks like a 234 Java class path ('.'->'-', '/'->'.'). 235 236 The 'pickle' format stores the raw results in a format that 237 can be reformatted using this command. It's typically used 238 with the show command to merge multiple test results into one 239 pickle file.""") 240 241 _add_format_args(parser) 242 243 parser.add_argument("result", type=argparse.FileType("rb"), nargs="*", 244 help="Pickled test results") 245 246def _show(args): 247 def _load(f): 248 # Load the pickled status file, sometimes e.g., when a 249 # regression is still running the status file might be 250 # incomplete. 251 try: 252 return pickle.load(f) 253 except EOFError:
|
252 print >> sys.stderr, 'Could not read file %s' % f.name
| 254 print('Could not read file %s' % f.name, file=sys.stderr)
|
253 return [] 254 255 formatter = _create_formatter(args) 256 suites = sum([ _load(f) for f in args.result ], []) 257 formatter.dump_suites(suites) 258 259def _test_args(subparsers): 260 parser = subparsers.add_parser( 261 "test", 262 formatter_class=ParagraphHelpFormatter, 263 help='Probe test results and set exit code', 264 epilog=""" 265 266 Load one or more pickled test file and return an exit code 267 corresponding to the test outcome. The following exit codes 268 can be returned: 269 270 0: All tests were successful or skipped. 271 272 1: General fault in the script such as incorrect parameters or 273 failing to parse a pickle file. 274 275 2: At least one test failed to run. This is what the summary 276 formatter usually shows as a 'FAILED'. 277 278 3: All tests ran correctly, but at least one failed to 279 verify its output. When displaying test output using the 280 summary formatter, such a test would show up as 'CHANGED'. 281 """) 282 283 parser.add_argument("result", type=argparse.FileType("rb"), nargs="*", 284 help="Pickled test results") 285 286def _test(args): 287 try: 288 suites = sum([ pickle.load(f) for f in args.result ], []) 289 except EOFError:
| 255 return [] 256 257 formatter = _create_formatter(args) 258 suites = sum([ _load(f) for f in args.result ], []) 259 formatter.dump_suites(suites) 260 261def _test_args(subparsers): 262 parser = subparsers.add_parser( 263 "test", 264 formatter_class=ParagraphHelpFormatter, 265 help='Probe test results and set exit code', 266 epilog=""" 267 268 Load one or more pickled test file and return an exit code 269 corresponding to the test outcome. The following exit codes 270 can be returned: 271 272 0: All tests were successful or skipped. 273 274 1: General fault in the script such as incorrect parameters or 275 failing to parse a pickle file. 276 277 2: At least one test failed to run. This is what the summary 278 formatter usually shows as a 'FAILED'. 279 280 3: All tests ran correctly, but at least one failed to 281 verify its output. When displaying test output using the 282 summary formatter, such a test would show up as 'CHANGED'. 283 """) 284 285 parser.add_argument("result", type=argparse.FileType("rb"), nargs="*", 286 help="Pickled test results") 287 288def _test(args): 289 try: 290 suites = sum([ pickle.load(f) for f in args.result ], []) 291 except EOFError:
|
290 print >> sys.stderr, 'Could not read all files'
| 292 print('Could not read all files', file=sys.stderr)
|
291 sys.exit(2) 292 293 if all(s for s in suites): 294 sys.exit(0) 295 elif any([ s.failed_run() for s in suites ]): 296 sys.exit(2) 297 elif any([ s.changed() for s in suites ]): 298 sys.exit(3) 299 else: 300 assert False, "Unexpected return status from test" 301 302_commands = { 303 "list" : (_list_tests, _list_tests_args), 304 "run" : (_run_tests, _run_tests_args), 305 "show" : (_show, _show_args), 306 "test" : (_test, _test_args), 307} 308 309def main(): 310 parser = argparse.ArgumentParser( 311 formatter_class=ParagraphHelpFormatter, 312 description="""gem5 testing multi tool.""", 313 epilog=""" 314 This tool provides an interface to gem5's test framework that 315 doesn't depend on gem5's build system. It supports test 316 listing, running, and output formatting. 317 318 The list sub-command (e.g., "test.py list arm/quick") produces 319 a list of tests tuples that can be used by the run command 320 (e.g., "tests.py run gem5.opt 321 quick/se/00.hello/arm/linux/simple-timing"). 322 323 The run command supports several output formats. One of them, 324 pickle, contains the raw output from the tests and can be 325 re-formatted using the show command (e.g., "tests.py show 326 --format summary *.pickle"). Such pickle files are also 327 generated by the build system when scons is used to run 328 regressions. 329 330 See the usage strings for the individual sub-commands for 331 details.""") 332 333 parser.add_argument("--verbose", action="store_true", 334 help="Produce more verbose output") 335 336 subparsers = parser.add_subparsers(dest="command") 337 338 for key, (impl, cmd_parser) in _commands.items(): 339 cmd_parser(subparsers) 340 341 args = parser.parse_args() 342 impl, cmd_parser = _commands[args.command] 343 impl(args) 344 345if __name__ == "__main__": 346 main()
| 293 sys.exit(2) 294 295 if all(s for s in suites): 296 sys.exit(0) 297 elif any([ s.failed_run() for s in suites ]): 298 sys.exit(2) 299 elif any([ s.changed() for s in suites ]): 300 sys.exit(3) 301 else: 302 assert False, "Unexpected return status from test" 303 304_commands = { 305 "list" : (_list_tests, _list_tests_args), 306 "run" : (_run_tests, _run_tests_args), 307 "show" : (_show, _show_args), 308 "test" : (_test, _test_args), 309} 310 311def main(): 312 parser = argparse.ArgumentParser( 313 formatter_class=ParagraphHelpFormatter, 314 description="""gem5 testing multi tool.""", 315 epilog=""" 316 This tool provides an interface to gem5's test framework that 317 doesn't depend on gem5's build system. It supports test 318 listing, running, and output formatting. 319 320 The list sub-command (e.g., "test.py list arm/quick") produces 321 a list of tests tuples that can be used by the run command 322 (e.g., "tests.py run gem5.opt 323 quick/se/00.hello/arm/linux/simple-timing"). 324 325 The run command supports several output formats. One of them, 326 pickle, contains the raw output from the tests and can be 327 re-formatted using the show command (e.g., "tests.py show 328 --format summary *.pickle"). Such pickle files are also 329 generated by the build system when scons is used to run 330 regressions. 331 332 See the usage strings for the individual sub-commands for 333 details.""") 334 335 parser.add_argument("--verbose", action="store_true", 336 help="Produce more verbose output") 337 338 subparsers = parser.add_subparsers(dest="command") 339 340 for key, (impl, cmd_parser) in _commands.items(): 341 cmd_parser(subparsers) 342 343 args = parser.parse_args() 344 impl, cmd_parser = _commands[args.command] 345 impl(args) 346 347if __name__ == "__main__": 348 main()
|