1#!/usr/bin/env python2 2# 3# Copyright (c) 2016 ARM Limited 4# All rights reserved 5# 6# The license below extends only to copyright in the software and shall 7# not be construed as granting a license to any other intellectual 8# property including but not limited to intellectual property relating 9# to a hardware implementation of the functionality of the software 10# licensed hereunder. You may use the software subject to the license 11# terms below provided that you ensure that this notice is replicated 12# unmodified and in its entirety in all distributions of the software, 13# modified or unmodified, in source code or in binary form. 14# 15# Redistribution and use in source and binary forms, with or without 16# modification, are permitted provided that the following conditions are 17# met: redistributions of source code must retain the above copyright 18# notice, this list of conditions and the following disclaimer; 19# redistributions in binary form must reproduce the above copyright 20# notice, this list of conditions and the following disclaimer in the 21# documentation and/or other materials provided with the distribution; 22# neither the name of the copyright holders nor the names of its 23# contributors may be used to endorse or promote products derived from 24# this software without specific prior written permission. 25# 26# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS 27# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT 28# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR 29# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT 30# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, 31# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT 32# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, 33# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY 34# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT 35# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE 36# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 37# 38# Authors: Andreas Sandberg 39 40import argparse 41import sys 42import os 43import pickle 44 45from testing.tests import * 46import testing.results 47 48class ParagraphHelpFormatter(argparse.HelpFormatter): 49 def _fill_text(self, text, width, indent): 50 return "\n\n".join([ 51 super(ParagraphHelpFormatter, self)._fill_text(p, width, indent) \ 52 for p in text.split("\n\n") ]) 53 54formatters = { 55 "junit" : testing.results.JUnit, 56 "text" : testing.results.Text, 57 "summary" : testing.results.TextSummary, 58 "pickle" : testing.results.Pickle, 59} 60 61 62def _add_format_args(parser): 63 parser.add_argument("--format", choices=formatters, default="text", 64 help="Output format") 65 66 parser.add_argument("--no-junit-xlate-names", action="store_true", 67 help="Don't translate test names to " \ 68 "package-like names") 69 70 parser.add_argument("--output", "-o", 71 type=argparse.FileType('w'), default=sys.stdout, 72 help="Test result output file") 73 74 75def _create_formatter(args): 76 formatter = formatters[args.format] 77 kwargs = { 78 "fout" : args.output, 79 "verbose" : args.verbose 80 } 81 82 if issubclass(formatter, testing.results.JUnit): 83 kwargs.update({ 84 "translate_names" : not args.no_junit_xlate_names, 85 }) 86 87 return formatter(**kwargs) 88 89 90def _list_tests_args(subparsers): 91 parser = subparsers.add_parser( 92 "list", 93 formatter_class=ParagraphHelpFormatter, 94 help="List available tests", 95 description="List available tests", 96 epilog=""" 97 Generate a list of available tests using a list filter. 98 99 The filter is a string consisting of the target ISA optionally 100 followed by the test category and mode separated by 101 slashes. The test names emitted by this command can be fed 102 into the run command. 103 104 For example, to list all quick arm tests, run the following: 105 tests.py list arm/quick 106 107 Non-mandatory parts of the filter string (anything other than 108 the ISA) can be left out or replaced with the wildcard 109 character. For example, all full-system tests can be listed 110 with this command: tests.py list arm/*/fs""") 111 112 parser.add_argument("--ruby-protocol", type=str, default=None, 113 help="Ruby protocol") 114 115 parser.add_argument("--gpu-isa", type=str, default=None, 116 help="GPU ISA") 117 118 parser.add_argument("list_filter", metavar="ISA[/category/mode]", 119 action="append", type=str, 120 help="List available test cases") 121 122def _list_tests(args): 123 for isa, categories, modes in \ 124 ( parse_test_filter(f) for f in args.list_filter ): 125 126 for test in get_tests(isa, categories=categories, modes=modes, 127 ruby_protocol=args.ruby_protocol, 128 gpu_isa=args.gpu_isa): 129 print "/".join(test) 130 sys.exit(0) 131 132def _run_tests_args(subparsers): 133 parser = subparsers.add_parser( 134 "run", 135 formatter_class=ParagraphHelpFormatter, 136 help='Run one or more tests', 137 description="Run one or more tests.", 138 epilog=""" 139 Run one or more tests described by a gem5 test tuple. 140 141 The test tuple consists of a test category (quick or long), a 142 test mode (fs or se), a workload name, an isa, an operating 143 system, and a config name separate by slashes. For example: 144 quick/se/00.hello/arm/linux/simple-timing 145 146 Available tests can be listed using the 'list' sub-command 147 (e.g., "tests.py list arm/quick" or one of the scons test list 148 targets (e.g., "scons build/ARM/tests/opt/quick.list"). 149 150 The test results can be stored in multiple different output 151 formats. See the help for the show command for more details 152 about output formatting.""") 153 154 parser.add_argument("gem5", type=str, 155 help="gem5 binary") 156 157 parser.add_argument("test", type=str, nargs="*", 158 help="List of tests to execute") 159 160 parser.add_argument("--directory", "-d", 161 type=str, default="m5tests", 162 help="Test work directory") 163 164 parser.add_argument("--timeout", "-t", 165 type=int, default="0", metavar="MINUTES", 166 help="Timeout, 0 to disable") 167 168 parser.add_argument("--skip-diff-out", action="store_true", 169 help="Skip output diffing stage") 170 171 parser.add_argument("--skip-diff-stat", action="store_true", 172 help="Skip stat diffing stage") 173 174 _add_format_args(parser) 175 176def _run_tests(args):
| 1#!/usr/bin/env python2 2# 3# Copyright (c) 2016 ARM Limited 4# All rights reserved 5# 6# The license below extends only to copyright in the software and shall 7# not be construed as granting a license to any other intellectual 8# property including but not limited to intellectual property relating 9# to a hardware implementation of the functionality of the software 10# licensed hereunder. You may use the software subject to the license 11# terms below provided that you ensure that this notice is replicated 12# unmodified and in its entirety in all distributions of the software, 13# modified or unmodified, in source code or in binary form. 14# 15# Redistribution and use in source and binary forms, with or without 16# modification, are permitted provided that the following conditions are 17# met: redistributions of source code must retain the above copyright 18# notice, this list of conditions and the following disclaimer; 19# redistributions in binary form must reproduce the above copyright 20# notice, this list of conditions and the following disclaimer in the 21# documentation and/or other materials provided with the distribution; 22# neither the name of the copyright holders nor the names of its 23# contributors may be used to endorse or promote products derived from 24# this software without specific prior written permission. 25# 26# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS 27# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT 28# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR 29# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT 30# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, 31# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT 32# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, 33# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY 34# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT 35# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE 36# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 37# 38# Authors: Andreas Sandberg 39 40import argparse 41import sys 42import os 43import pickle 44 45from testing.tests import * 46import testing.results 47 48class ParagraphHelpFormatter(argparse.HelpFormatter): 49 def _fill_text(self, text, width, indent): 50 return "\n\n".join([ 51 super(ParagraphHelpFormatter, self)._fill_text(p, width, indent) \ 52 for p in text.split("\n\n") ]) 53 54formatters = { 55 "junit" : testing.results.JUnit, 56 "text" : testing.results.Text, 57 "summary" : testing.results.TextSummary, 58 "pickle" : testing.results.Pickle, 59} 60 61 62def _add_format_args(parser): 63 parser.add_argument("--format", choices=formatters, default="text", 64 help="Output format") 65 66 parser.add_argument("--no-junit-xlate-names", action="store_true", 67 help="Don't translate test names to " \ 68 "package-like names") 69 70 parser.add_argument("--output", "-o", 71 type=argparse.FileType('w'), default=sys.stdout, 72 help="Test result output file") 73 74 75def _create_formatter(args): 76 formatter = formatters[args.format] 77 kwargs = { 78 "fout" : args.output, 79 "verbose" : args.verbose 80 } 81 82 if issubclass(formatter, testing.results.JUnit): 83 kwargs.update({ 84 "translate_names" : not args.no_junit_xlate_names, 85 }) 86 87 return formatter(**kwargs) 88 89 90def _list_tests_args(subparsers): 91 parser = subparsers.add_parser( 92 "list", 93 formatter_class=ParagraphHelpFormatter, 94 help="List available tests", 95 description="List available tests", 96 epilog=""" 97 Generate a list of available tests using a list filter. 98 99 The filter is a string consisting of the target ISA optionally 100 followed by the test category and mode separated by 101 slashes. The test names emitted by this command can be fed 102 into the run command. 103 104 For example, to list all quick arm tests, run the following: 105 tests.py list arm/quick 106 107 Non-mandatory parts of the filter string (anything other than 108 the ISA) can be left out or replaced with the wildcard 109 character. For example, all full-system tests can be listed 110 with this command: tests.py list arm/*/fs""") 111 112 parser.add_argument("--ruby-protocol", type=str, default=None, 113 help="Ruby protocol") 114 115 parser.add_argument("--gpu-isa", type=str, default=None, 116 help="GPU ISA") 117 118 parser.add_argument("list_filter", metavar="ISA[/category/mode]", 119 action="append", type=str, 120 help="List available test cases") 121 122def _list_tests(args): 123 for isa, categories, modes in \ 124 ( parse_test_filter(f) for f in args.list_filter ): 125 126 for test in get_tests(isa, categories=categories, modes=modes, 127 ruby_protocol=args.ruby_protocol, 128 gpu_isa=args.gpu_isa): 129 print "/".join(test) 130 sys.exit(0) 131 132def _run_tests_args(subparsers): 133 parser = subparsers.add_parser( 134 "run", 135 formatter_class=ParagraphHelpFormatter, 136 help='Run one or more tests', 137 description="Run one or more tests.", 138 epilog=""" 139 Run one or more tests described by a gem5 test tuple. 140 141 The test tuple consists of a test category (quick or long), a 142 test mode (fs or se), a workload name, an isa, an operating 143 system, and a config name separate by slashes. For example: 144 quick/se/00.hello/arm/linux/simple-timing 145 146 Available tests can be listed using the 'list' sub-command 147 (e.g., "tests.py list arm/quick" or one of the scons test list 148 targets (e.g., "scons build/ARM/tests/opt/quick.list"). 149 150 The test results can be stored in multiple different output 151 formats. See the help for the show command for more details 152 about output formatting.""") 153 154 parser.add_argument("gem5", type=str, 155 help="gem5 binary") 156 157 parser.add_argument("test", type=str, nargs="*", 158 help="List of tests to execute") 159 160 parser.add_argument("--directory", "-d", 161 type=str, default="m5tests", 162 help="Test work directory") 163 164 parser.add_argument("--timeout", "-t", 165 type=int, default="0", metavar="MINUTES", 166 help="Timeout, 0 to disable") 167 168 parser.add_argument("--skip-diff-out", action="store_true", 169 help="Skip output diffing stage") 170 171 parser.add_argument("--skip-diff-stat", action="store_true", 172 help="Skip stat diffing stage") 173 174 _add_format_args(parser) 175 176def _run_tests(args):
|
177 formatter = _create_formatter(args) 178 179 out_base = os.path.abspath(args.directory) 180 if not os.path.exists(out_base): 181 os.mkdir(out_base) 182 tests = [] 183 for test_name in args.test: 184 config = ClassicConfig(*test_name.split("/")) 185 out_dir = os.path.join(out_base, "/".join(config)) 186 tests.append( 187 ClassicTest(args.gem5, out_dir, config, 188 timeout=args.timeout, 189 skip_diff_stat=args.skip_diff_stat, 190 skip_diff_out=args.skip_diff_out)) 191 192 all_results = [] 193 print "Running %i tests" % len(tests) 194 for testno, test in enumerate(tests): 195 print "%i: Running '%s'..." % (testno, test) 196 197 all_results.append(test.run()) 198 199 formatter.dump_suites(all_results) 200 201def _show_args(subparsers): 202 parser = subparsers.add_parser( 203 "show", 204 formatter_class=ParagraphHelpFormatter, 205 help='Display pickled test results', 206 description='Display pickled test results', 207 epilog=""" 208 Reformat the pickled output from one or more test runs. This 209 command is typically used with the output from a single test 210 run, but it can also be used to merge the outputs from 211 multiple runs. 212 213 The 'text' format is a verbose output format that provides 214 information about individual test units and the output from 215 failed tests. It's mainly useful for debugging test failures. 216 217 The 'summary' format provides outputs the results of one test 218 per line with the test's overall status (OK, SKIPPED, or 219 FAILED). 220 221 The 'junit' format is primarily intended for use with CI 222 systems. It provides an XML representation of test 223 status. Similar to the text format, it includes detailed 224 information about test failures. Since many JUnit parser make 225 assume that test names look like Java packet strings, the 226 JUnit formatter automatically to something the looks like a 227 Java class path ('.'->'-', '/'->'.'). 228 229 The 'pickle' format stores the raw results in a format that 230 can be reformatted using this command. It's typically used 231 with the show command to merge multiple test results into one 232 pickle file.""") 233 234 _add_format_args(parser) 235 236 parser.add_argument("result", type=argparse.FileType("rb"), nargs="*", 237 help="Pickled test results") 238 239def _show(args): 240 formatter = _create_formatter(args) 241 suites = sum([ pickle.load(f) for f in args.result ], []) 242 formatter.dump_suites(suites) 243 244def _test_args(subparsers): 245 parser = subparsers.add_parser( 246 "test", 247 formatter_class=ParagraphHelpFormatter, 248 help='Probe test results and set exit code', 249 epilog=""" 250 251 Load one or more pickled test file and return an exit code 252 corresponding to the test outcome. The following exit codes 253 can be returned: 254 255 0: All tests were successful or skipped. 256 257 1: General fault in the script such as incorrect parameters or 258 failing to parse a pickle file. 259 260 2: At least one test failed to run. This is what the summary 261 formatter usually shows as a 'FAILED'. 262 263 3: All tests ran correctly, but at least one failed to 264 verify its output. When displaying test output using the 265 summary formatter, such a test would show up as 'CHANGED'. 266 """) 267 268 _add_format_args(parser) 269 270 parser.add_argument("result", type=argparse.FileType("rb"), nargs="*", 271 help="Pickled test results") 272 273def _test(args): 274 suites = sum([ pickle.load(f) for f in args.result ], []) 275 276 if all(s for s in suites): 277 sys.exit(0) 278 elif any([ s.failed_run() for s in suites ]): 279 sys.exit(2) 280 elif any([ s.changed() for s in suites ]): 281 sys.exit(3) 282 else: 283 assert False, "Unexpected return status from test" 284 285_commands = { 286 "list" : (_list_tests, _list_tests_args), 287 "run" : (_run_tests, _run_tests_args), 288 "show" : (_show, _show_args), 289 "test" : (_test, _test_args), 290} 291 292def main(): 293 parser = argparse.ArgumentParser( 294 formatter_class=ParagraphHelpFormatter, 295 description="""gem5 testing multi tool.""", 296 epilog=""" 297 This tool provides an interface to gem5's test framework that 298 doesn't depend on gem5's build system. It supports test 299 listing, running, and output formatting. 300 301 The list sub-command (e.g., "test.py list arm/quick") produces 302 a list of tests tuples that can be used by the run command 303 (e.g., "tests.py run gem5.opt 304 quick/se/00.hello/arm/linux/simple-timing"). 305 306 The run command supports several output formats. One of them, 307 pickle, contains the raw output from the tests and can be 308 re-formatted using the show command (e.g., "tests.py show 309 --format summary *.pickle"). Such pickle files are also 310 generated by the build system when scons is used to run 311 regressions. 312 313 See the usage strings for the individual sub-commands for 314 details.""") 315 316 parser.add_argument("--verbose", action="store_true", 317 help="Produce more verbose output") 318 319 subparsers = parser.add_subparsers(dest="command") 320 321 for key, (impl, cmd_parser) in _commands.items(): 322 cmd_parser(subparsers) 323 324 args = parser.parse_args() 325 impl, cmd_parser = _commands[args.command] 326 impl(args) 327 328if __name__ == "__main__": 329 main()
| 182 formatter = _create_formatter(args) 183 184 out_base = os.path.abspath(args.directory) 185 if not os.path.exists(out_base): 186 os.mkdir(out_base) 187 tests = [] 188 for test_name in args.test: 189 config = ClassicConfig(*test_name.split("/")) 190 out_dir = os.path.join(out_base, "/".join(config)) 191 tests.append( 192 ClassicTest(args.gem5, out_dir, config, 193 timeout=args.timeout, 194 skip_diff_stat=args.skip_diff_stat, 195 skip_diff_out=args.skip_diff_out)) 196 197 all_results = [] 198 print "Running %i tests" % len(tests) 199 for testno, test in enumerate(tests): 200 print "%i: Running '%s'..." % (testno, test) 201 202 all_results.append(test.run()) 203 204 formatter.dump_suites(all_results) 205 206def _show_args(subparsers): 207 parser = subparsers.add_parser( 208 "show", 209 formatter_class=ParagraphHelpFormatter, 210 help='Display pickled test results', 211 description='Display pickled test results', 212 epilog=""" 213 Reformat the pickled output from one or more test runs. This 214 command is typically used with the output from a single test 215 run, but it can also be used to merge the outputs from 216 multiple runs. 217 218 The 'text' format is a verbose output format that provides 219 information about individual test units and the output from 220 failed tests. It's mainly useful for debugging test failures. 221 222 The 'summary' format provides outputs the results of one test 223 per line with the test's overall status (OK, SKIPPED, or 224 FAILED). 225 226 The 'junit' format is primarily intended for use with CI 227 systems. It provides an XML representation of test 228 status. Similar to the text format, it includes detailed 229 information about test failures. Since many JUnit parser make 230 assume that test names look like Java packet strings, the 231 JUnit formatter automatically to something the looks like a 232 Java class path ('.'->'-', '/'->'.'). 233 234 The 'pickle' format stores the raw results in a format that 235 can be reformatted using this command. It's typically used 236 with the show command to merge multiple test results into one 237 pickle file.""") 238 239 _add_format_args(parser) 240 241 parser.add_argument("result", type=argparse.FileType("rb"), nargs="*", 242 help="Pickled test results") 243 244def _show(args): 245 formatter = _create_formatter(args) 246 suites = sum([ pickle.load(f) for f in args.result ], []) 247 formatter.dump_suites(suites) 248 249def _test_args(subparsers): 250 parser = subparsers.add_parser( 251 "test", 252 formatter_class=ParagraphHelpFormatter, 253 help='Probe test results and set exit code', 254 epilog=""" 255 256 Load one or more pickled test file and return an exit code 257 corresponding to the test outcome. The following exit codes 258 can be returned: 259 260 0: All tests were successful or skipped. 261 262 1: General fault in the script such as incorrect parameters or 263 failing to parse a pickle file. 264 265 2: At least one test failed to run. This is what the summary 266 formatter usually shows as a 'FAILED'. 267 268 3: All tests ran correctly, but at least one failed to 269 verify its output. When displaying test output using the 270 summary formatter, such a test would show up as 'CHANGED'. 271 """) 272 273 _add_format_args(parser) 274 275 parser.add_argument("result", type=argparse.FileType("rb"), nargs="*", 276 help="Pickled test results") 277 278def _test(args): 279 suites = sum([ pickle.load(f) for f in args.result ], []) 280 281 if all(s for s in suites): 282 sys.exit(0) 283 elif any([ s.failed_run() for s in suites ]): 284 sys.exit(2) 285 elif any([ s.changed() for s in suites ]): 286 sys.exit(3) 287 else: 288 assert False, "Unexpected return status from test" 289 290_commands = { 291 "list" : (_list_tests, _list_tests_args), 292 "run" : (_run_tests, _run_tests_args), 293 "show" : (_show, _show_args), 294 "test" : (_test, _test_args), 295} 296 297def main(): 298 parser = argparse.ArgumentParser( 299 formatter_class=ParagraphHelpFormatter, 300 description="""gem5 testing multi tool.""", 301 epilog=""" 302 This tool provides an interface to gem5's test framework that 303 doesn't depend on gem5's build system. It supports test 304 listing, running, and output formatting. 305 306 The list sub-command (e.g., "test.py list arm/quick") produces 307 a list of tests tuples that can be used by the run command 308 (e.g., "tests.py run gem5.opt 309 quick/se/00.hello/arm/linux/simple-timing"). 310 311 The run command supports several output formats. One of them, 312 pickle, contains the raw output from the tests and can be 313 re-formatted using the show command (e.g., "tests.py show 314 --format summary *.pickle"). Such pickle files are also 315 generated by the build system when scons is used to run 316 regressions. 317 318 See the usage strings for the individual sub-commands for 319 details.""") 320 321 parser.add_argument("--verbose", action="store_true", 322 help="Produce more verbose output") 323 324 subparsers = parser.add_subparsers(dest="command") 325 326 for key, (impl, cmd_parser) in _commands.items(): 327 cmd_parser(subparsers) 328 329 args = parser.parse_args() 330 impl, cmd_parser = _commands[args.command] 331 impl(args) 332 333if __name__ == "__main__": 334 main()
|