verify.py revision 13002
112870Sgabeblack@google.com#!/usr/bin/env python2 212870Sgabeblack@google.com# 312870Sgabeblack@google.com# Copyright 2018 Google, Inc. 412870Sgabeblack@google.com# 512870Sgabeblack@google.com# Redistribution and use in source and binary forms, with or without 612870Sgabeblack@google.com# modification, are permitted provided that the following conditions are 712870Sgabeblack@google.com# met: redistributions of source code must retain the above copyright 812870Sgabeblack@google.com# notice, this list of conditions and the following disclaimer; 912870Sgabeblack@google.com# redistributions in binary form must reproduce the above copyright 1012870Sgabeblack@google.com# notice, this list of conditions and the following disclaimer in the 1112870Sgabeblack@google.com# documentation and/or other materials provided with the distribution; 1212870Sgabeblack@google.com# neither the name of the copyright holders nor the names of its 1312870Sgabeblack@google.com# contributors may be used to endorse or promote products derived from 1412870Sgabeblack@google.com# this software without specific prior written permission. 1512870Sgabeblack@google.com# 1612870Sgabeblack@google.com# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS 1712870Sgabeblack@google.com# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT 1812870Sgabeblack@google.com# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR 1912870Sgabeblack@google.com# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT 2012870Sgabeblack@google.com# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, 2112870Sgabeblack@google.com# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT 2212870Sgabeblack@google.com# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, 2312870Sgabeblack@google.com# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY 2412870Sgabeblack@google.com# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT 2512870Sgabeblack@google.com# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE 2612870Sgabeblack@google.com# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 2712870Sgabeblack@google.com# 2812870Sgabeblack@google.com# Authors: Gabe Black 2912870Sgabeblack@google.com 3012870Sgabeblack@google.comfrom __future__ import print_function 3112870Sgabeblack@google.com 3212870Sgabeblack@google.comimport argparse 3312870Sgabeblack@google.comimport functools 3412870Sgabeblack@google.comimport inspect 3512870Sgabeblack@google.comimport itertools 3612870Sgabeblack@google.comimport json 3713000Sgabeblack@google.comimport multiprocessing.pool 3812870Sgabeblack@google.comimport os 3912870Sgabeblack@google.comimport subprocess 4012870Sgabeblack@google.comimport sys 4112870Sgabeblack@google.com 4212870Sgabeblack@google.comscript_path = os.path.abspath(inspect.getfile(inspect.currentframe())) 4312870Sgabeblack@google.comscript_dir = os.path.dirname(script_path) 4412870Sgabeblack@google.comconfig_path = os.path.join(script_dir, 'config.py') 4512870Sgabeblack@google.com 4612870Sgabeblack@google.comsystemc_rel_path = 'systemc' 4712870Sgabeblack@google.comtests_rel_path = os.path.join(systemc_rel_path, 'tests') 4812870Sgabeblack@google.comjson_rel_path = os.path.join(tests_rel_path, 'tests.json') 4912870Sgabeblack@google.com 5012870Sgabeblack@google.com 5112870Sgabeblack@google.com 5212870Sgabeblack@google.comdef scons(*args): 5312870Sgabeblack@google.com args = ['scons'] + list(args) 5412870Sgabeblack@google.com subprocess.check_call(args) 5512870Sgabeblack@google.com 5612870Sgabeblack@google.com 5712870Sgabeblack@google.com 5812870Sgabeblack@google.comclass Test(object): 5912870Sgabeblack@google.com def __init__(self, target, suffix, build_dir, props): 6012870Sgabeblack@google.com self.target = target 6112870Sgabeblack@google.com self.suffix = suffix 6212870Sgabeblack@google.com self.build_dir = build_dir 6312870Sgabeblack@google.com 6412870Sgabeblack@google.com for key, val in props.iteritems(): 6512870Sgabeblack@google.com setattr(self, key, val) 6612870Sgabeblack@google.com 6712870Sgabeblack@google.com def dir(self): 6812870Sgabeblack@google.com return os.path.join(self.build_dir, tests_rel_path, self.path) 6912870Sgabeblack@google.com 7012870Sgabeblack@google.com def src_dir(self): 7112897Sgabeblack@google.com return os.path.join(script_dir, self.path) 7212870Sgabeblack@google.com 7312870Sgabeblack@google.com def golden_dir(self): 7412870Sgabeblack@google.com return os.path.join(self.src_dir(), 'golden') 7512870Sgabeblack@google.com 7612870Sgabeblack@google.com def bin(self): 7712870Sgabeblack@google.com return '.'.join([self.name, self.suffix]) 7812870Sgabeblack@google.com 7912870Sgabeblack@google.com def full_path(self): 8012870Sgabeblack@google.com return os.path.join(self.dir(), self.bin()) 8112870Sgabeblack@google.com 8212870Sgabeblack@google.com def m5out_dir(self): 8312870Sgabeblack@google.com return os.path.join(self.dir(), 'm5out.' + self.suffix) 8412870Sgabeblack@google.com 8513002Sgabeblack@google.com def returncode_file(self): 8613002Sgabeblack@google.com return os.path.join(self.m5out_dir(), 'returncode') 8713002Sgabeblack@google.com 8812870Sgabeblack@google.com 8912870Sgabeblack@google.com 9012870Sgabeblack@google.comtest_phase_classes = {} 9112870Sgabeblack@google.com 9212870Sgabeblack@google.comclass TestPhaseMeta(type): 9312870Sgabeblack@google.com def __init__(cls, name, bases, d): 9412870Sgabeblack@google.com if not d.pop('abstract', False): 9512870Sgabeblack@google.com test_phase_classes[d['name']] = cls 9612870Sgabeblack@google.com 9712870Sgabeblack@google.com super(TestPhaseMeta, cls).__init__(name, bases, d) 9812870Sgabeblack@google.com 9912870Sgabeblack@google.comclass TestPhaseBase(object): 10012870Sgabeblack@google.com __metaclass__ = TestPhaseMeta 10112870Sgabeblack@google.com abstract = True 10212870Sgabeblack@google.com 10312870Sgabeblack@google.com def __init__(self, main_args, *args): 10412870Sgabeblack@google.com self.main_args = main_args 10512870Sgabeblack@google.com self.args = args 10612870Sgabeblack@google.com 10712870Sgabeblack@google.com def __lt__(self, other): 10812870Sgabeblack@google.com return self.number < other.number 10912870Sgabeblack@google.com 11012870Sgabeblack@google.comclass CompilePhase(TestPhaseBase): 11112870Sgabeblack@google.com name = 'compile' 11212870Sgabeblack@google.com number = 1 11312870Sgabeblack@google.com 11412870Sgabeblack@google.com def run(self, tests): 11512870Sgabeblack@google.com targets = list([test.full_path() for test in tests]) 11612870Sgabeblack@google.com scons_args = list(self.args) + targets 11712870Sgabeblack@google.com scons(*scons_args) 11812870Sgabeblack@google.com 11912870Sgabeblack@google.comclass RunPhase(TestPhaseBase): 12012870Sgabeblack@google.com name = 'execute' 12112870Sgabeblack@google.com number = 2 12212870Sgabeblack@google.com 12312870Sgabeblack@google.com def run(self, tests): 12413000Sgabeblack@google.com parser = argparse.ArgumentParser() 12513000Sgabeblack@google.com parser.add_argument('--timeout', type=int, metavar='SECONDS', 12613000Sgabeblack@google.com help='Time limit for each run in seconds.', 12713000Sgabeblack@google.com default=0) 12813000Sgabeblack@google.com parser.add_argument('-j', type=int, default=1, 12913000Sgabeblack@google.com help='How many tests to run in parallel.') 13013000Sgabeblack@google.com args = parser.parse_args(self.args) 13113000Sgabeblack@google.com 13213000Sgabeblack@google.com timeout_cmd = [ 13313000Sgabeblack@google.com 'timeout', 13413000Sgabeblack@google.com '--kill-after', str(args.timeout * 2), 13513000Sgabeblack@google.com str(args.timeout) 13613000Sgabeblack@google.com ] 13713000Sgabeblack@google.com def run_test(test): 13813000Sgabeblack@google.com cmd = [] 13913000Sgabeblack@google.com if args.timeout: 14013000Sgabeblack@google.com cmd.extend(timeout_cmd) 14113000Sgabeblack@google.com cmd.extend([ 14212870Sgabeblack@google.com test.full_path(), 14312870Sgabeblack@google.com '-red', test.m5out_dir(), 14412870Sgabeblack@google.com '--listener-mode=off', 14512870Sgabeblack@google.com config_path 14613000Sgabeblack@google.com ]) 14713002Sgabeblack@google.com # Ensure the output directory exists. 14813002Sgabeblack@google.com if not os.path.exists(test.m5out_dir()): 14913002Sgabeblack@google.com os.makedirs(test.m5out_dir()) 15013001Sgabeblack@google.com try: 15113001Sgabeblack@google.com subprocess.check_call(cmd) 15213001Sgabeblack@google.com except subprocess.CalledProcessError, error: 15313001Sgabeblack@google.com returncode = error.returncode 15413001Sgabeblack@google.com else: 15513001Sgabeblack@google.com returncode = 0 15613002Sgabeblack@google.com with open(test.returncode_file(), 'w') as rc: 15713001Sgabeblack@google.com rc.write('%d\n' % returncode) 15813000Sgabeblack@google.com 15913000Sgabeblack@google.com runnable = filter(lambda t: not t.compile_only, tests) 16013000Sgabeblack@google.com if args.j == 1: 16113000Sgabeblack@google.com map(run_test, runnable) 16213000Sgabeblack@google.com else: 16313000Sgabeblack@google.com tp = multiprocessing.pool.ThreadPool(args.j) 16413000Sgabeblack@google.com map(lambda t: tp.apply_async(run_test, (t,)), runnable) 16513000Sgabeblack@google.com tp.close() 16613000Sgabeblack@google.com tp.join() 16712870Sgabeblack@google.com 16812870Sgabeblack@google.comclass VerifyPhase(TestPhaseBase): 16912870Sgabeblack@google.com name = 'verify' 17012870Sgabeblack@google.com number = 3 17112870Sgabeblack@google.com 17213002Sgabeblack@google.com def reset_status(self): 17313002Sgabeblack@google.com self._passed = [] 17413002Sgabeblack@google.com self._failed = {} 17513002Sgabeblack@google.com 17613002Sgabeblack@google.com def passed(self, test): 17713002Sgabeblack@google.com self._passed.append(test) 17813002Sgabeblack@google.com 17913002Sgabeblack@google.com def failed(self, test, cause): 18013002Sgabeblack@google.com self._failed.setdefault(cause, []).append(test) 18113002Sgabeblack@google.com 18213002Sgabeblack@google.com def print_status(self): 18313002Sgabeblack@google.com total_passed = len(self._passed) 18413002Sgabeblack@google.com total_failed = sum(map(len, self._failed.values())) 18513002Sgabeblack@google.com print() 18613002Sgabeblack@google.com print('Passed: {passed:4} - Failed: {failed:4}'.format( 18713002Sgabeblack@google.com passed=total_passed, failed=total_failed)) 18813002Sgabeblack@google.com 18913002Sgabeblack@google.com def write_result_file(self, path): 19013002Sgabeblack@google.com passed = map(lambda t: t.path, self._passed) 19113002Sgabeblack@google.com passed.sort() 19213002Sgabeblack@google.com failed = { 19313002Sgabeblack@google.com cause: map(lambda t: t.path, tests) for 19413002Sgabeblack@google.com cause, tests in self._failed.iteritems() 19513002Sgabeblack@google.com } 19613002Sgabeblack@google.com for tests in failed.values(): 19713002Sgabeblack@google.com tests.sort() 19813002Sgabeblack@google.com results = { 'passed': passed, 'failed': failed } 19913002Sgabeblack@google.com with open(path, 'w') as rf: 20013002Sgabeblack@google.com json.dump(results, rf) 20113002Sgabeblack@google.com 20213002Sgabeblack@google.com def print_results(self): 20313002Sgabeblack@google.com passed = map(lambda t: t.path, self._passed) 20413002Sgabeblack@google.com passed.sort() 20513002Sgabeblack@google.com failed = { 20613002Sgabeblack@google.com cause: map(lambda t: t.path, tests) for 20713002Sgabeblack@google.com cause, tests in self._failed.iteritems() 20813002Sgabeblack@google.com } 20913002Sgabeblack@google.com for tests in failed.values(): 21013002Sgabeblack@google.com tests.sort() 21113002Sgabeblack@google.com 21213002Sgabeblack@google.com print() 21313002Sgabeblack@google.com print('Passed:') 21413002Sgabeblack@google.com map(lambda t: print(' ', t), passed) 21513002Sgabeblack@google.com 21613002Sgabeblack@google.com print() 21713002Sgabeblack@google.com print('Failed:') 21813002Sgabeblack@google.com categories = failed.items() 21913002Sgabeblack@google.com categories.sort() 22013002Sgabeblack@google.com 22113002Sgabeblack@google.com def cat_str((cause, tests)): 22213002Sgabeblack@google.com heading = ' ' + cause.capitalize() + ':\n' 22313002Sgabeblack@google.com test_lines = [' ' + test + '\n'for test in tests] 22413002Sgabeblack@google.com return heading + ''.join(test_lines) 22513002Sgabeblack@google.com blocks = map(cat_str, categories) 22613002Sgabeblack@google.com 22713002Sgabeblack@google.com print('\n'.join(blocks)) 22813002Sgabeblack@google.com 22912870Sgabeblack@google.com def run(self, tests): 23013002Sgabeblack@google.com parser = argparse.ArgumentParser() 23113002Sgabeblack@google.com result_opts = parser.add_mutually_exclusive_group() 23213002Sgabeblack@google.com result_opts.add_argument('--result-file', action='store_true', 23313002Sgabeblack@google.com help='Create a results.json file in the current directory.') 23413002Sgabeblack@google.com result_opts.add_argument('--result-file-at', metavar='PATH', 23513002Sgabeblack@google.com help='Create a results json file at the given path.') 23613002Sgabeblack@google.com parser.add_argument('--print-results', action='store_true', 23713002Sgabeblack@google.com help='Print a list of tests that passed or failed') 23813002Sgabeblack@google.com args = parser.parse_args(self.args) 23912870Sgabeblack@google.com 24013002Sgabeblack@google.com self.reset_status() 24113002Sgabeblack@google.com 24213002Sgabeblack@google.com runnable = filter(lambda t: not t.compile_only, tests) 24313002Sgabeblack@google.com compile_only = filter(lambda t: t.compile_only, tests) 24413002Sgabeblack@google.com 24513002Sgabeblack@google.com for test in compile_only: 24613002Sgabeblack@google.com if os.path.exists(test.full_path()): 24713002Sgabeblack@google.com self.passed(test) 24813002Sgabeblack@google.com else: 24913002Sgabeblack@google.com self.failed(test, 'compile failed') 25013002Sgabeblack@google.com 25113002Sgabeblack@google.com for test in runnable: 25213002Sgabeblack@google.com with open(test.returncode_file()) as rc: 25313002Sgabeblack@google.com returncode = int(rc.read()) 25413002Sgabeblack@google.com 25513002Sgabeblack@google.com if returncode == 0: 25613002Sgabeblack@google.com self.passed(test) 25713002Sgabeblack@google.com elif returncode == 124: 25813002Sgabeblack@google.com self.failed(test, 'time out') 25913002Sgabeblack@google.com else: 26013002Sgabeblack@google.com self.failed(test, 'abort') 26113002Sgabeblack@google.com 26213002Sgabeblack@google.com if args.print_results: 26313002Sgabeblack@google.com self.print_results() 26413002Sgabeblack@google.com 26513002Sgabeblack@google.com self.print_status() 26613002Sgabeblack@google.com 26713002Sgabeblack@google.com result_path = None 26813002Sgabeblack@google.com if args.result_file: 26913002Sgabeblack@google.com result_path = os.path.join(os.getcwd(), 'results.json') 27013002Sgabeblack@google.com elif args.result_file_at: 27113002Sgabeblack@google.com result_path = args.result_file_at 27213002Sgabeblack@google.com 27313002Sgabeblack@google.com if result_path: 27413002Sgabeblack@google.com self.write_result_file(result_path) 27512870Sgabeblack@google.com 27612870Sgabeblack@google.com 27712870Sgabeblack@google.comparser = argparse.ArgumentParser(description='SystemC test utility') 27812870Sgabeblack@google.com 27912870Sgabeblack@google.comparser.add_argument('build_dir', metavar='BUILD_DIR', 28012870Sgabeblack@google.com help='The build directory (ie. build/ARM).') 28112870Sgabeblack@google.com 28212870Sgabeblack@google.comparser.add_argument('--update-json', action='store_true', 28312870Sgabeblack@google.com help='Update the json manifest of tests.') 28412870Sgabeblack@google.com 28512870Sgabeblack@google.comparser.add_argument('--flavor', choices=['debug', 'opt', 'fast'], 28612870Sgabeblack@google.com default='opt', 28712870Sgabeblack@google.com help='Flavor of binary to test.') 28812870Sgabeblack@google.com 28912870Sgabeblack@google.comparser.add_argument('--list', action='store_true', 29012870Sgabeblack@google.com help='List the available tests') 29112870Sgabeblack@google.com 29212903Sgabeblack@google.comfilter_opts = parser.add_mutually_exclusive_group() 29312903Sgabeblack@google.comfilter_opts.add_argument('--filter', default='True', 29412903Sgabeblack@google.com help='Python expression which filters tests based ' 29512903Sgabeblack@google.com 'on their properties') 29612903Sgabeblack@google.comfilter_opts.add_argument('--filter-file', default=None, 29712903Sgabeblack@google.com type=argparse.FileType('r'), 29812903Sgabeblack@google.com help='Same as --filter, but read from a file') 29912870Sgabeblack@google.com 30012870Sgabeblack@google.comdef collect_phases(args): 30112870Sgabeblack@google.com phase_groups = [list(g) for k, g in 30212870Sgabeblack@google.com itertools.groupby(args, lambda x: x != '--phase') if k] 30312870Sgabeblack@google.com main_args = parser.parse_args(phase_groups[0][1:]) 30412870Sgabeblack@google.com phases = [] 30512870Sgabeblack@google.com names = [] 30612870Sgabeblack@google.com for group in phase_groups[1:]: 30712870Sgabeblack@google.com name = group[0] 30812870Sgabeblack@google.com if name in names: 30912870Sgabeblack@google.com raise RuntimeException('Phase %s specified more than once' % name) 31012870Sgabeblack@google.com phase = test_phase_classes[name] 31112870Sgabeblack@google.com phases.append(phase(main_args, *group[1:])) 31212870Sgabeblack@google.com phases.sort() 31312870Sgabeblack@google.com return main_args, phases 31412870Sgabeblack@google.com 31512870Sgabeblack@google.commain_args, phases = collect_phases(sys.argv) 31612870Sgabeblack@google.com 31712870Sgabeblack@google.comif len(phases) == 0: 31812870Sgabeblack@google.com phases = [ 31912870Sgabeblack@google.com CompilePhase(main_args), 32012870Sgabeblack@google.com RunPhase(main_args), 32112870Sgabeblack@google.com VerifyPhase(main_args) 32212870Sgabeblack@google.com ] 32312870Sgabeblack@google.com 32412870Sgabeblack@google.com 32512870Sgabeblack@google.com 32612870Sgabeblack@google.comjson_path = os.path.join(main_args.build_dir, json_rel_path) 32712870Sgabeblack@google.com 32812870Sgabeblack@google.comif main_args.update_json: 32912870Sgabeblack@google.com scons(os.path.join(json_path)) 33012870Sgabeblack@google.com 33112870Sgabeblack@google.comwith open(json_path) as f: 33212870Sgabeblack@google.com test_data = json.load(f) 33312870Sgabeblack@google.com 33412903Sgabeblack@google.com if main_args.filter_file: 33512903Sgabeblack@google.com f = main_args.filter_file 33612903Sgabeblack@google.com filt = compile(f.read(), f.name, 'eval') 33712903Sgabeblack@google.com else: 33812903Sgabeblack@google.com filt = compile(main_args.filter, '<string>', 'eval') 33912903Sgabeblack@google.com 34012903Sgabeblack@google.com filtered_tests = { 34112903Sgabeblack@google.com target: props for (target, props) in 34212903Sgabeblack@google.com test_data.iteritems() if eval(filt, dict(props)) 34312903Sgabeblack@google.com } 34412903Sgabeblack@google.com 34512870Sgabeblack@google.com if main_args.list: 34612903Sgabeblack@google.com for target, props in sorted(filtered_tests.iteritems()): 34712870Sgabeblack@google.com print('%s.%s' % (target, main_args.flavor)) 34812870Sgabeblack@google.com for key, val in props.iteritems(): 34912870Sgabeblack@google.com print(' %s: %s' % (key, val)) 35012937Sgabeblack@google.com print('Total tests: %d' % len(filtered_tests)) 35112870Sgabeblack@google.com else: 35212903Sgabeblack@google.com tests_to_run = list([ 35312903Sgabeblack@google.com Test(target, main_args.flavor, main_args.build_dir, props) for 35412903Sgabeblack@google.com target, props in sorted(filtered_tests.iteritems()) 35512903Sgabeblack@google.com ]) 35612870Sgabeblack@google.com 35712870Sgabeblack@google.com for phase in phases: 35812870Sgabeblack@google.com phase.run(tests_to_run) 359