verify.py revision 12937
112870Sgabeblack@google.com#!/usr/bin/env python2 212870Sgabeblack@google.com# 312870Sgabeblack@google.com# Copyright 2018 Google, Inc. 412870Sgabeblack@google.com# 512870Sgabeblack@google.com# Redistribution and use in source and binary forms, with or without 612870Sgabeblack@google.com# modification, are permitted provided that the following conditions are 712870Sgabeblack@google.com# met: redistributions of source code must retain the above copyright 812870Sgabeblack@google.com# notice, this list of conditions and the following disclaimer; 912870Sgabeblack@google.com# redistributions in binary form must reproduce the above copyright 1012870Sgabeblack@google.com# notice, this list of conditions and the following disclaimer in the 1112870Sgabeblack@google.com# documentation and/or other materials provided with the distribution; 1212870Sgabeblack@google.com# neither the name of the copyright holders nor the names of its 1312870Sgabeblack@google.com# contributors may be used to endorse or promote products derived from 1412870Sgabeblack@google.com# this software without specific prior written permission. 1512870Sgabeblack@google.com# 1612870Sgabeblack@google.com# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS 1712870Sgabeblack@google.com# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT 1812870Sgabeblack@google.com# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR 1912870Sgabeblack@google.com# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT 2012870Sgabeblack@google.com# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, 2112870Sgabeblack@google.com# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT 2212870Sgabeblack@google.com# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, 2312870Sgabeblack@google.com# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY 2412870Sgabeblack@google.com# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT 2512870Sgabeblack@google.com# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE 2612870Sgabeblack@google.com# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 2712870Sgabeblack@google.com# 2812870Sgabeblack@google.com# Authors: Gabe Black 2912870Sgabeblack@google.com 3012870Sgabeblack@google.comfrom __future__ import print_function 3112870Sgabeblack@google.com 3212870Sgabeblack@google.comimport argparse 3312870Sgabeblack@google.comimport functools 3412870Sgabeblack@google.comimport inspect 3512870Sgabeblack@google.comimport itertools 3612870Sgabeblack@google.comimport json 3712870Sgabeblack@google.comimport logging 3812870Sgabeblack@google.comimport os 3912870Sgabeblack@google.comimport subprocess 4012870Sgabeblack@google.comimport sys 4112870Sgabeblack@google.com 4212870Sgabeblack@google.comscript_path = os.path.abspath(inspect.getfile(inspect.currentframe())) 4312870Sgabeblack@google.comscript_dir = os.path.dirname(script_path) 4412870Sgabeblack@google.comconfig_path = os.path.join(script_dir, 'config.py') 4512870Sgabeblack@google.com 4612870Sgabeblack@google.comsystemc_rel_path = 'systemc' 4712870Sgabeblack@google.comtests_rel_path = os.path.join(systemc_rel_path, 'tests') 4812870Sgabeblack@google.comjson_rel_path = os.path.join(tests_rel_path, 'tests.json') 4912870Sgabeblack@google.com 5012870Sgabeblack@google.com 5112870Sgabeblack@google.com 5212870Sgabeblack@google.comlogging.basicConfig(level=logging.INFO) 5312870Sgabeblack@google.com 5412870Sgabeblack@google.comdef scons(*args): 5512870Sgabeblack@google.com args = ['scons'] + list(args) 5612870Sgabeblack@google.com subprocess.check_call(args) 5712870Sgabeblack@google.com 5812870Sgabeblack@google.com 5912870Sgabeblack@google.com 6012870Sgabeblack@google.comclass Test(object): 6112870Sgabeblack@google.com def __init__(self, target, suffix, build_dir, props): 6212870Sgabeblack@google.com self.target = target 6312870Sgabeblack@google.com self.suffix = suffix 6412870Sgabeblack@google.com self.build_dir = build_dir 6512870Sgabeblack@google.com 6612870Sgabeblack@google.com for key, val in props.iteritems(): 6712870Sgabeblack@google.com setattr(self, key, val) 6812870Sgabeblack@google.com 6912870Sgabeblack@google.com def dir(self): 7012870Sgabeblack@google.com return os.path.join(self.build_dir, tests_rel_path, self.path) 7112870Sgabeblack@google.com 7212870Sgabeblack@google.com def src_dir(self): 7312897Sgabeblack@google.com return os.path.join(script_dir, self.path) 7412870Sgabeblack@google.com 7512870Sgabeblack@google.com def golden_dir(self): 7612870Sgabeblack@google.com return os.path.join(self.src_dir(), 'golden') 7712870Sgabeblack@google.com 7812870Sgabeblack@google.com def bin(self): 7912870Sgabeblack@google.com return '.'.join([self.name, self.suffix]) 8012870Sgabeblack@google.com 8112870Sgabeblack@google.com def full_path(self): 8212870Sgabeblack@google.com return os.path.join(self.dir(), self.bin()) 8312870Sgabeblack@google.com 8412870Sgabeblack@google.com def m5out_dir(self): 8512870Sgabeblack@google.com return os.path.join(self.dir(), 'm5out.' + self.suffix) 8612870Sgabeblack@google.com 8712870Sgabeblack@google.com 8812870Sgabeblack@google.com 8912870Sgabeblack@google.comtest_phase_classes = {} 9012870Sgabeblack@google.com 9112870Sgabeblack@google.comclass TestPhaseMeta(type): 9212870Sgabeblack@google.com def __init__(cls, name, bases, d): 9312870Sgabeblack@google.com if not d.pop('abstract', False): 9412870Sgabeblack@google.com test_phase_classes[d['name']] = cls 9512870Sgabeblack@google.com 9612870Sgabeblack@google.com super(TestPhaseMeta, cls).__init__(name, bases, d) 9712870Sgabeblack@google.com 9812870Sgabeblack@google.comclass TestPhaseBase(object): 9912870Sgabeblack@google.com __metaclass__ = TestPhaseMeta 10012870Sgabeblack@google.com abstract = True 10112870Sgabeblack@google.com 10212870Sgabeblack@google.com def __init__(self, main_args, *args): 10312870Sgabeblack@google.com self.main_args = main_args 10412870Sgabeblack@google.com self.args = args 10512870Sgabeblack@google.com 10612870Sgabeblack@google.com def __lt__(self, other): 10712870Sgabeblack@google.com return self.number < other.number 10812870Sgabeblack@google.com 10912870Sgabeblack@google.comclass CompilePhase(TestPhaseBase): 11012870Sgabeblack@google.com name = 'compile' 11112870Sgabeblack@google.com number = 1 11212870Sgabeblack@google.com 11312870Sgabeblack@google.com def run(self, tests): 11412870Sgabeblack@google.com targets = list([test.full_path() for test in tests]) 11512870Sgabeblack@google.com scons_args = list(self.args) + targets 11612870Sgabeblack@google.com scons(*scons_args) 11712870Sgabeblack@google.com 11812870Sgabeblack@google.comclass RunPhase(TestPhaseBase): 11912870Sgabeblack@google.com name = 'execute' 12012870Sgabeblack@google.com number = 2 12112870Sgabeblack@google.com 12212870Sgabeblack@google.com def run(self, tests): 12312870Sgabeblack@google.com for test in tests: 12412870Sgabeblack@google.com if test.compile_only: 12512870Sgabeblack@google.com continue 12612870Sgabeblack@google.com args = [ 12712870Sgabeblack@google.com test.full_path(), 12812870Sgabeblack@google.com '-red', test.m5out_dir(), 12912870Sgabeblack@google.com '--listener-mode=off', 13012870Sgabeblack@google.com config_path 13112870Sgabeblack@google.com ] 13212870Sgabeblack@google.com subprocess.check_call(args) 13312870Sgabeblack@google.com 13412870Sgabeblack@google.comclass VerifyPhase(TestPhaseBase): 13512870Sgabeblack@google.com name = 'verify' 13612870Sgabeblack@google.com number = 3 13712870Sgabeblack@google.com 13812870Sgabeblack@google.com def run(self, tests): 13912870Sgabeblack@google.com for test in tests: 14012870Sgabeblack@google.com if test.compile_only: 14112870Sgabeblack@google.com continue 14212870Sgabeblack@google.com logging.info("Would verify %s", test.m5out_dir()) 14312870Sgabeblack@google.com 14412870Sgabeblack@google.com 14512870Sgabeblack@google.com 14612870Sgabeblack@google.comparser = argparse.ArgumentParser(description='SystemC test utility') 14712870Sgabeblack@google.com 14812870Sgabeblack@google.comparser.add_argument('build_dir', metavar='BUILD_DIR', 14912870Sgabeblack@google.com help='The build directory (ie. build/ARM).') 15012870Sgabeblack@google.com 15112870Sgabeblack@google.comparser.add_argument('--update-json', action='store_true', 15212870Sgabeblack@google.com help='Update the json manifest of tests.') 15312870Sgabeblack@google.com 15412870Sgabeblack@google.comparser.add_argument('--flavor', choices=['debug', 'opt', 'fast'], 15512870Sgabeblack@google.com default='opt', 15612870Sgabeblack@google.com help='Flavor of binary to test.') 15712870Sgabeblack@google.com 15812870Sgabeblack@google.comparser.add_argument('--list', action='store_true', 15912870Sgabeblack@google.com help='List the available tests') 16012870Sgabeblack@google.com 16112903Sgabeblack@google.comfilter_opts = parser.add_mutually_exclusive_group() 16212903Sgabeblack@google.comfilter_opts.add_argument('--filter', default='True', 16312903Sgabeblack@google.com help='Python expression which filters tests based ' 16412903Sgabeblack@google.com 'on their properties') 16512903Sgabeblack@google.comfilter_opts.add_argument('--filter-file', default=None, 16612903Sgabeblack@google.com type=argparse.FileType('r'), 16712903Sgabeblack@google.com help='Same as --filter, but read from a file') 16812870Sgabeblack@google.com 16912870Sgabeblack@google.comdef collect_phases(args): 17012870Sgabeblack@google.com phase_groups = [list(g) for k, g in 17112870Sgabeblack@google.com itertools.groupby(args, lambda x: x != '--phase') if k] 17212870Sgabeblack@google.com main_args = parser.parse_args(phase_groups[0][1:]) 17312870Sgabeblack@google.com phases = [] 17412870Sgabeblack@google.com names = [] 17512870Sgabeblack@google.com for group in phase_groups[1:]: 17612870Sgabeblack@google.com name = group[0] 17712870Sgabeblack@google.com if name in names: 17812870Sgabeblack@google.com raise RuntimeException('Phase %s specified more than once' % name) 17912870Sgabeblack@google.com phase = test_phase_classes[name] 18012870Sgabeblack@google.com phases.append(phase(main_args, *group[1:])) 18112870Sgabeblack@google.com phases.sort() 18212870Sgabeblack@google.com return main_args, phases 18312870Sgabeblack@google.com 18412870Sgabeblack@google.commain_args, phases = collect_phases(sys.argv) 18512870Sgabeblack@google.com 18612870Sgabeblack@google.comif len(phases) == 0: 18712870Sgabeblack@google.com phases = [ 18812870Sgabeblack@google.com CompilePhase(main_args), 18912870Sgabeblack@google.com RunPhase(main_args), 19012870Sgabeblack@google.com VerifyPhase(main_args) 19112870Sgabeblack@google.com ] 19212870Sgabeblack@google.com 19312870Sgabeblack@google.com 19412870Sgabeblack@google.com 19512870Sgabeblack@google.comjson_path = os.path.join(main_args.build_dir, json_rel_path) 19612870Sgabeblack@google.com 19712870Sgabeblack@google.comif main_args.update_json: 19812870Sgabeblack@google.com scons(os.path.join(json_path)) 19912870Sgabeblack@google.com 20012870Sgabeblack@google.comwith open(json_path) as f: 20112870Sgabeblack@google.com test_data = json.load(f) 20212870Sgabeblack@google.com 20312903Sgabeblack@google.com if main_args.filter_file: 20412903Sgabeblack@google.com f = main_args.filter_file 20512903Sgabeblack@google.com filt = compile(f.read(), f.name, 'eval') 20612903Sgabeblack@google.com else: 20712903Sgabeblack@google.com filt = compile(main_args.filter, '<string>', 'eval') 20812903Sgabeblack@google.com 20912903Sgabeblack@google.com filtered_tests = { 21012903Sgabeblack@google.com target: props for (target, props) in 21112903Sgabeblack@google.com test_data.iteritems() if eval(filt, dict(props)) 21212903Sgabeblack@google.com } 21312903Sgabeblack@google.com 21412870Sgabeblack@google.com if main_args.list: 21512903Sgabeblack@google.com for target, props in sorted(filtered_tests.iteritems()): 21612870Sgabeblack@google.com print('%s.%s' % (target, main_args.flavor)) 21712870Sgabeblack@google.com for key, val in props.iteritems(): 21812870Sgabeblack@google.com print(' %s: %s' % (key, val)) 21912937Sgabeblack@google.com print('Total tests: %d' % len(filtered_tests)) 22012870Sgabeblack@google.com else: 22112903Sgabeblack@google.com tests_to_run = list([ 22212903Sgabeblack@google.com Test(target, main_args.flavor, main_args.build_dir, props) for 22312903Sgabeblack@google.com target, props in sorted(filtered_tests.iteritems()) 22412903Sgabeblack@google.com ]) 22512870Sgabeblack@google.com 22612870Sgabeblack@google.com for phase in phases: 22712870Sgabeblack@google.com phase.run(tests_to_run) 228