tests.py revision 11543
111723Sar4jc@virginia.edu#!/usr/bin/env python 211723Sar4jc@virginia.edu# 311723Sar4jc@virginia.edu# Copyright (c) 2016 ARM Limited 412808Srobert.scheffel1@tu-dresden.de# All rights reserved 511723Sar4jc@virginia.edu# 611723Sar4jc@virginia.edu# The license below extends only to copyright in the software and shall 711723Sar4jc@virginia.edu# not be construed as granting a license to any other intellectual 811723Sar4jc@virginia.edu# property including but not limited to intellectual property relating 911723Sar4jc@virginia.edu# to a hardware implementation of the functionality of the software 1011723Sar4jc@virginia.edu# licensed hereunder. You may use the software subject to the license 1111723Sar4jc@virginia.edu# terms below provided that you ensure that this notice is replicated 1211723Sar4jc@virginia.edu# unmodified and in its entirety in all distributions of the software, 1311723Sar4jc@virginia.edu# modified or unmodified, in source code or in binary form. 1411723Sar4jc@virginia.edu# 1511723Sar4jc@virginia.edu# Redistribution and use in source and binary forms, with or without 1611723Sar4jc@virginia.edu# modification, are permitted provided that the following conditions are 1711723Sar4jc@virginia.edu# met: redistributions of source code must retain the above copyright 1811723Sar4jc@virginia.edu# notice, this list of conditions and the following disclaimer; 1911723Sar4jc@virginia.edu# redistributions in binary form must reproduce the above copyright 2011723Sar4jc@virginia.edu# notice, this list of conditions and the following disclaimer in the 2111723Sar4jc@virginia.edu# documentation and/or other materials provided with the distribution; 2211723Sar4jc@virginia.edu# neither the name of the copyright holders nor the names of its 2311723Sar4jc@virginia.edu# contributors may be used to endorse or promote products derived from 2411723Sar4jc@virginia.edu# this software without specific prior written permission. 2511723Sar4jc@virginia.edu# 2611723Sar4jc@virginia.edu# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS 2711723Sar4jc@virginia.edu# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT 2811723Sar4jc@virginia.edu# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR 2911723Sar4jc@virginia.edu# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT 3011723Sar4jc@virginia.edu# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, 3112808Srobert.scheffel1@tu-dresden.de# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT 3211723Sar4jc@virginia.edu# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, 3311723Sar4jc@virginia.edu# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY 3411723Sar4jc@virginia.edu# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT 3511723Sar4jc@virginia.edu# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE 3611723Sar4jc@virginia.edu# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 3712850Salec.roelke@gmail.com# 3811723Sar4jc@virginia.edu# Authors: Andreas Sandberg 3911723Sar4jc@virginia.edu 4012850Salec.roelke@gmail.comimport argparse 4112848Sar4jc@virginia.eduimport sys 4211723Sar4jc@virginia.eduimport os 4311723Sar4jc@virginia.eduimport pickle 4411723Sar4jc@virginia.edu 4511723Sar4jc@virginia.edufrom testing.tests import * 4611723Sar4jc@virginia.eduimport testing.results 4711723Sar4jc@virginia.edu 4812848Sar4jc@virginia.educlass ParagraphHelpFormatter(argparse.HelpFormatter): 4912848Sar4jc@virginia.edu def _fill_text(self, text, width, indent): 5012848Sar4jc@virginia.edu return "\n\n".join([ 5112848Sar4jc@virginia.edu super(ParagraphHelpFormatter, self)._fill_text(p, width, indent) \ 5212848Sar4jc@virginia.edu for p in text.split("\n\n") ]) 5312848Sar4jc@virginia.edu 5412848Sar4jc@virginia.eduformatters = { 5511725Sar4jc@virginia.edu "junit" : testing.results.JUnit, 5612848Sar4jc@virginia.edu "text" : testing.results.Text, 5711723Sar4jc@virginia.edu "summary" : testing.results.TextSummary, 5811723Sar4jc@virginia.edu "pickle" : testing.results.Pickle, 5911723Sar4jc@virginia.edu} 6011723Sar4jc@virginia.edu 6111723Sar4jc@virginia.edu 6211723Sar4jc@virginia.edudef _add_format_args(parser): 6311723Sar4jc@virginia.edu parser.add_argument("--format", choices=formatters, default="text", 6411723Sar4jc@virginia.edu help="Output format") 6511723Sar4jc@virginia.edu 6611723Sar4jc@virginia.edu parser.add_argument("--no-junit-xlate-names", action="store_true", 6711723Sar4jc@virginia.edu help="Don't translate test names to " \ 6811723Sar4jc@virginia.edu "package-like names") 6912848Sar4jc@virginia.edu 7012848Sar4jc@virginia.edu parser.add_argument("--output", "-o", 7112848Sar4jc@virginia.edu type=argparse.FileType('w'), default=sys.stdout, 7212848Sar4jc@virginia.edu help="Test result output file") 7312848Sar4jc@virginia.edu 7411723Sar4jc@virginia.edu 7511723Sar4jc@virginia.edudef _create_formatter(args): 7611723Sar4jc@virginia.edu formatter = formatters[args.format] 7711723Sar4jc@virginia.edu kwargs = { 7811723Sar4jc@virginia.edu "fout" : args.output, 7911723Sar4jc@virginia.edu "verbose" : args.verbose 8012850Salec.roelke@gmail.com } 8112850Salec.roelke@gmail.com 8211723Sar4jc@virginia.edu if issubclass(formatter, testing.results.JUnit): 8312848Sar4jc@virginia.edu kwargs.update({ 8412848Sar4jc@virginia.edu "translate_names" : not args.no_junit_xlate_names, 8511723Sar4jc@virginia.edu }) 8611723Sar4jc@virginia.edu 8712849Sar4jc@virginia.edu return formatter(**kwargs) 8812848Sar4jc@virginia.edu 8912848Sar4jc@virginia.edu 9012849Sar4jc@virginia.edudef _list_tests_args(subparsers): 9111723Sar4jc@virginia.edu parser = subparsers.add_parser( 9212848Sar4jc@virginia.edu "list", 9312848Sar4jc@virginia.edu formatter_class=ParagraphHelpFormatter, 9411723Sar4jc@virginia.edu help="List available tests", 9511723Sar4jc@virginia.edu description="List available tests", 9612808Srobert.scheffel1@tu-dresden.de epilog=""" 9712808Srobert.scheffel1@tu-dresden.de Generate a list of available tests using a list filter. 9812808Srobert.scheffel1@tu-dresden.de 9912808Srobert.scheffel1@tu-dresden.de The filter is a string consisting of the target ISA optionally 10012808Srobert.scheffel1@tu-dresden.de followed by the test category and mode separated by 10112808Srobert.scheffel1@tu-dresden.de slashes. The test names emitted by this command can be fed 10212808Srobert.scheffel1@tu-dresden.de into the run command. 10312808Srobert.scheffel1@tu-dresden.de 10412808Srobert.scheffel1@tu-dresden.de For example, to list all quick arm tests, run the following: 10512808Srobert.scheffel1@tu-dresden.de tests.py list arm/quick 10612808Srobert.scheffel1@tu-dresden.de 10712808Srobert.scheffel1@tu-dresden.de Non-mandatory parts of the filter string (anything other than 10812808Srobert.scheffel1@tu-dresden.de the ISA) can be left out or replaced with the wildcard 10912808Srobert.scheffel1@tu-dresden.de character. For example, all full-system tests can be listed 11012808Srobert.scheffel1@tu-dresden.de with this command: tests.py list arm/*/fs""") 11112808Srobert.scheffel1@tu-dresden.de 11212808Srobert.scheffel1@tu-dresden.de parser.add_argument("--ruby-protocol", type=str, default=None, 11312808Srobert.scheffel1@tu-dresden.de help="Ruby protocol") 11412808Srobert.scheffel1@tu-dresden.de 11512808Srobert.scheffel1@tu-dresden.de parser.add_argument("--gpu-isa", type=str, default=None, 11612808Srobert.scheffel1@tu-dresden.de help="GPU ISA") 11711723Sar4jc@virginia.edu 11812849Sar4jc@virginia.edu parser.add_argument("list_filter", metavar="ISA[/category/mode]", 11912849Sar4jc@virginia.edu action="append", type=str, 12012849Sar4jc@virginia.edu help="List available test cases") 12112849Sar4jc@virginia.edu 12212849Sar4jc@virginia.edudef _list_tests(args): 12312849Sar4jc@virginia.edu for isa, categories, modes in \ 12412849Sar4jc@virginia.edu ( parse_test_filter(f) for f in args.list_filter ): 12512849Sar4jc@virginia.edu 12612849Sar4jc@virginia.edu for test in get_tests(isa, categories=categories, modes=modes, 12712849Sar4jc@virginia.edu ruby_protocol=args.ruby_protocol, 12812849Sar4jc@virginia.edu gpu_isa=args.gpu_isa): 12912849Sar4jc@virginia.edu print "/".join(test) 13012849Sar4jc@virginia.edu sys.exit(0) 13112849Sar4jc@virginia.edu 13211723Sar4jc@virginia.edudef _run_tests_args(subparsers): 13311723Sar4jc@virginia.edu parser = subparsers.add_parser( 13412849Sar4jc@virginia.edu "run", 13512849Sar4jc@virginia.edu formatter_class=ParagraphHelpFormatter, 13611723Sar4jc@virginia.edu help='Run one or more tests', 13711723Sar4jc@virginia.edu description="Run one or more tests.", 13812848Sar4jc@virginia.edu epilog=""" 13911723Sar4jc@virginia.edu Run one or more tests described by a gem5 test tuple. 14011723Sar4jc@virginia.edu 14112849Sar4jc@virginia.edu The test tuple consists of a test category (quick or long), a 14212136Sar4jc@virginia.edu test mode (fs or se), a workload name, an isa, an operating 14312136Sar4jc@virginia.edu system, and a config name separate by slashes. For example: 14412136Sar4jc@virginia.edu quick/se/00.hello/arm/linux/simple-timing 14512848Sar4jc@virginia.edu 14612136Sar4jc@virginia.edu Available tests can be listed using the 'list' sub-command 14712849Sar4jc@virginia.edu (e.g., "tests.py list arm/quick" or one of the scons test list 14812849Sar4jc@virginia.edu targets (e.g., "scons build/ARM/tests/opt/quick.list"). 14912136Sar4jc@virginia.edu 15012136Sar4jc@virginia.edu The test results can be stored in multiple different output 15112848Sar4jc@virginia.edu formats. See the help for the show command for more details 15212136Sar4jc@virginia.edu about output formatting.""") 15312136Sar4jc@virginia.edu 15412849Sar4jc@virginia.edu parser.add_argument("gem5", type=str, 15511723Sar4jc@virginia.edu help="gem5 binary") 15611723Sar4jc@virginia.edu 15711723Sar4jc@virginia.edu parser.add_argument("test", type=str, nargs="*", 15812848Sar4jc@virginia.edu help="List of tests to execute") 15911723Sar4jc@virginia.edu 16012849Sar4jc@virginia.edu parser.add_argument("--directory", "-d", 16112849Sar4jc@virginia.edu type=str, default="m5tests", 16212848Sar4jc@virginia.edu help="Test work directory") 16311723Sar4jc@virginia.edu 16411723Sar4jc@virginia.edu parser.add_argument("--timeout", "-t", 16512848Sar4jc@virginia.edu type=int, default="0", metavar="MINUTES", 16611723Sar4jc@virginia.edu help="Timeout, 0 to disable") 16711723Sar4jc@virginia.edu 16812849Sar4jc@virginia.edu parser.add_argument("--skip-diff-out", action="store_true", 16911725Sar4jc@virginia.edu help="Skip output diffing stage") 17011725Sar4jc@virginia.edu 17111725Sar4jc@virginia.edu parser.add_argument("--skip-diff-stat", action="store_true", 17212848Sar4jc@virginia.edu help="Skip stat diffing stage") 17311725Sar4jc@virginia.edu 17412849Sar4jc@virginia.edu _add_format_args(parser) 17512849Sar4jc@virginia.edu 17612848Sar4jc@virginia.edudef _run_tests(args): 17711725Sar4jc@virginia.edu formatter = _create_formatter(args) 17811725Sar4jc@virginia.edu 17912848Sar4jc@virginia.edu out_base = os.path.abspath(args.directory) 18011725Sar4jc@virginia.edu if not os.path.exists(out_base): 18111725Sar4jc@virginia.edu os.mkdir(out_base) 18212849Sar4jc@virginia.edu tests = [] 18312849Sar4jc@virginia.edu for test_name in args.test: 18412849Sar4jc@virginia.edu config = ClassicConfig(*test_name.split("/")) 18512849Sar4jc@virginia.edu out_dir = os.path.join(out_base, "/".join(config)) 18612849Sar4jc@virginia.edu tests.append( 18712849Sar4jc@virginia.edu ClassicTest(args.gem5, out_dir, config, 18812849Sar4jc@virginia.edu timeout=args.timeout, 18912849Sar4jc@virginia.edu skip_diff_stat=args.skip_diff_stat, 19012849Sar4jc@virginia.edu skip_diff_out=args.skip_diff_out)) 19112849Sar4jc@virginia.edu 19212849Sar4jc@virginia.edu all_results = [] 19312849Sar4jc@virginia.edu print "Running %i tests" % len(tests) 19412849Sar4jc@virginia.edu for testno, test in enumerate(tests): 19511723Sar4jc@virginia.edu print "%i: Running '%s'..." % (testno, test) 19611723Sar4jc@virginia.edu 19712849Sar4jc@virginia.edu all_results.append(test.run()) 19812849Sar4jc@virginia.edu 19912849Sar4jc@virginia.edu formatter.dump_suites(all_results) 20011723Sar4jc@virginia.edu 20112849Sar4jc@virginia.edudef _show_args(subparsers): 20212849Sar4jc@virginia.edu parser = subparsers.add_parser( 20312849Sar4jc@virginia.edu "show", 20412849Sar4jc@virginia.edu formatter_class=ParagraphHelpFormatter, 20512849Sar4jc@virginia.edu help='Display pickled test results', 20612848Sar4jc@virginia.edu description='Display pickled test results', 20711723Sar4jc@virginia.edu epilog=""" 20811723Sar4jc@virginia.edu Reformat the pickled output from one or more test runs. This 20911723Sar4jc@virginia.edu command is typically used with the output from a single test 21011723Sar4jc@virginia.edu run, but it can also be used to merge the outputs from 21111723Sar4jc@virginia.edu multiple runs. 21212850Salec.roelke@gmail.com 21312850Salec.roelke@gmail.com The 'text' format is a verbose output format that provides 21412850Salec.roelke@gmail.com information about individual test units and the output from 21512850Salec.roelke@gmail.com failed tests. It's mainly useful for debugging test failures. 21612850Salec.roelke@gmail.com 21712850Salec.roelke@gmail.com The 'summary' format provides outputs the results of one test 21812850Salec.roelke@gmail.com per line with the test's overall status (OK, SKIPPED, or 21912850Salec.roelke@gmail.com FAILED). 22012850Salec.roelke@gmail.com 22112850Salec.roelke@gmail.com The 'junit' format is primarily intended for use with CI 22212850Salec.roelke@gmail.com systems. It provides an XML representation of test 22312850Salec.roelke@gmail.com status. Similar to the text format, it includes detailed 22412850Salec.roelke@gmail.com information about test failures. Since many JUnit parser make 22512850Salec.roelke@gmail.com assume that test names look like Java packet strings, the 22612850Salec.roelke@gmail.com JUnit formatter automatically to something the looks like a 22712850Salec.roelke@gmail.com Java class path ('.'->'-', '/'->'.'). 22812850Salec.roelke@gmail.com 22912850Salec.roelke@gmail.com The 'pickle' format stores the raw results in a format that 23012850Salec.roelke@gmail.com can be reformatted using this command. It's typically used 23112848Sar4jc@virginia.edu with the show command to merge multiple test results into one 23211723Sar4jc@virginia.edu pickle file.""") 23311723Sar4jc@virginia.edu 23411723Sar4jc@virginia.edu _add_format_args(parser) 23511723Sar4jc@virginia.edu 23612850Salec.roelke@gmail.com parser.add_argument("result", type=argparse.FileType("rb"), nargs="*", 237 help="Pickled test results") 238 239def _show(args): 240 formatter = _create_formatter(args) 241 suites = sum([ pickle.load(f) for f in args.result ], []) 242 formatter.dump_suites(suites) 243 244def _test_args(subparsers): 245 parser = subparsers.add_parser( 246 "test", 247 formatter_class=ParagraphHelpFormatter, 248 help='Probe test results and set exit code', 249 epilog=""" 250 251 Load one or more pickled test file and return an exit code 252 corresponding to the test outcome. The following exit codes 253 can be returned: 254 255 0: All tests were successful or skipped. 256 257 1: General fault in the script such as incorrect parameters or 258 failing to parse a pickle file. 259 260 2: At least one test failed to run. This is what the summary 261 formatter usually shows as a 'FAILED'. 262 263 3: All tests ran correctly, but at least one failed to 264 verify its output. When displaying test output using the 265 summary formatter, such a test would show up as 'CHANGED'. 266 """) 267 268 _add_format_args(parser) 269 270 parser.add_argument("result", type=argparse.FileType("rb"), nargs="*", 271 help="Pickled test results") 272 273def _test(args): 274 suites = sum([ pickle.load(f) for f in args.result ], []) 275 276 if all(s for s in suites): 277 sys.exit(0) 278 elif any([ s.failed_run() for s in suites ]): 279 sys.exit(2) 280 elif any([ s.changed() for s in suites ]): 281 sys.exit(3) 282 else: 283 assert False, "Unexpected return status from test" 284 285_commands = { 286 "list" : (_list_tests, _list_tests_args), 287 "run" : (_run_tests, _run_tests_args), 288 "show" : (_show, _show_args), 289 "test" : (_test, _test_args), 290} 291 292def main(): 293 parser = argparse.ArgumentParser( 294 formatter_class=ParagraphHelpFormatter, 295 description="""gem5 testing multi tool.""", 296 epilog=""" 297 This tool provides an interface to gem5's test framework that 298 doesn't depend on gem5's build system. It supports test 299 listing, running, and output formatting. 300 301 The list sub-command (e.g., "test.py list arm/quick") produces 302 a list of tests tuples that can be used by the run command 303 (e.g., "tests.py run gem5.opt 304 quick/se/00.hello/arm/linux/simple-timing"). 305 306 The run command supports several output formats. One of them, 307 pickle, contains the raw output from the tests and can be 308 re-formatted using the show command (e.g., "tests.py show 309 --format summary *.pickle"). Such pickle files are also 310 generated by the build system when scons is used to run 311 regressions. 312 313 See the usage strings for the individual sub-commands for 314 details.""") 315 316 parser.add_argument("--verbose", action="store_true", 317 help="Produce more verbose output") 318 319 subparsers = parser.add_subparsers(dest="command") 320 321 for key, (impl, cmd_parser) in _commands.items(): 322 cmd_parser(subparsers) 323 324 args = parser.parse_args() 325 impl, cmd_parser = _commands[args.command] 326 impl(args) 327 328if __name__ == "__main__": 329 main() 330