verify.py revision 13139
1955SN/A#!/usr/bin/env python2 2955SN/A# 35871Snate@binkert.org# Copyright 2018 Google, Inc. 41762SN/A# 5955SN/A# Redistribution and use in source and binary forms, with or without 6955SN/A# modification, are permitted provided that the following conditions are 7955SN/A# met: redistributions of source code must retain the above copyright 8955SN/A# notice, this list of conditions and the following disclaimer; 9955SN/A# redistributions in binary form must reproduce the above copyright 10955SN/A# notice, this list of conditions and the following disclaimer in the 11955SN/A# documentation and/or other materials provided with the distribution; 12955SN/A# neither the name of the copyright holders nor the names of its 13955SN/A# contributors may be used to endorse or promote products derived from 14955SN/A# this software without specific prior written permission. 15955SN/A# 16955SN/A# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS 17955SN/A# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT 18955SN/A# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR 19955SN/A# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT 20955SN/A# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, 21955SN/A# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT 22955SN/A# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, 23955SN/A# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY 24955SN/A# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT 25955SN/A# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE 26955SN/A# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 27955SN/A# 28955SN/A# Authors: Gabe Black 292665Ssaidi@eecs.umich.edu 302665Ssaidi@eecs.umich.edufrom __future__ import print_function 315863Snate@binkert.org 32955SN/Aimport argparse 33955SN/Aimport collections 34955SN/Aimport difflib 35955SN/Aimport functools 36955SN/Aimport inspect 372632Sstever@eecs.umich.eduimport itertools 382632Sstever@eecs.umich.eduimport json 392632Sstever@eecs.umich.eduimport multiprocessing.pool 402632Sstever@eecs.umich.eduimport os 41955SN/Aimport re 422632Sstever@eecs.umich.eduimport subprocess 432632Sstever@eecs.umich.eduimport sys 442761Sstever@eecs.umich.edu 452632Sstever@eecs.umich.eduscript_path = os.path.abspath(inspect.getfile(inspect.currentframe())) 462632Sstever@eecs.umich.eduscript_dir = os.path.dirname(script_path) 472632Sstever@eecs.umich.educonfig_path = os.path.join(script_dir, 'config.py') 482761Sstever@eecs.umich.edu 492761Sstever@eecs.umich.edusystemc_rel_path = 'systemc' 502761Sstever@eecs.umich.edutests_rel_path = os.path.join(systemc_rel_path, 'tests') 512632Sstever@eecs.umich.edujson_rel_path = os.path.join(tests_rel_path, 'tests.json') 522632Sstever@eecs.umich.edu 532761Sstever@eecs.umich.edu 542761Sstever@eecs.umich.edu 552761Sstever@eecs.umich.edudef scons(*args): 562761Sstever@eecs.umich.edu args = ['scons'] + list(args) 572761Sstever@eecs.umich.edu subprocess.check_call(args) 582632Sstever@eecs.umich.edu 592632Sstever@eecs.umich.edu 602632Sstever@eecs.umich.edu 612632Sstever@eecs.umich.educlass Test(object): 622632Sstever@eecs.umich.edu def __init__(self, target, suffix, build_dir, props): 632632Sstever@eecs.umich.edu self.target = target 642632Sstever@eecs.umich.edu self.suffix = suffix 65955SN/A self.build_dir = build_dir 66955SN/A self.props = {} 67955SN/A 685863Snate@binkert.org for key, val in props.iteritems(): 695863Snate@binkert.org self.set_prop(key, val) 705863Snate@binkert.org 715863Snate@binkert.org def set_prop(self, key, val): 725863Snate@binkert.org setattr(self, key, val) 735863Snate@binkert.org self.props[key] = val 745863Snate@binkert.org 755863Snate@binkert.org def dir(self): 765863Snate@binkert.org return os.path.join(self.build_dir, tests_rel_path, self.path) 775863Snate@binkert.org 785863Snate@binkert.org def src_dir(self): 795863Snate@binkert.org return os.path.join(script_dir, self.path) 805863Snate@binkert.org 815863Snate@binkert.org def expected_returncode_file(self): 825863Snate@binkert.org return os.path.join(self.src_dir(), 'expected_returncode') 835863Snate@binkert.org 845863Snate@binkert.org def golden_dir(self): 855863Snate@binkert.org return os.path.join(self.src_dir(), 'golden') 865863Snate@binkert.org 875863Snate@binkert.org def bin(self): 885863Snate@binkert.org return '.'.join([self.name, self.suffix]) 895863Snate@binkert.org 905863Snate@binkert.org def full_path(self): 915863Snate@binkert.org return os.path.join(self.dir(), self.bin()) 925863Snate@binkert.org 935863Snate@binkert.org def m5out_dir(self): 945863Snate@binkert.org return os.path.join(self.dir(), 'm5out.' + self.suffix) 955863Snate@binkert.org 965863Snate@binkert.org def returncode_file(self): 975863Snate@binkert.org return os.path.join(self.m5out_dir(), 'returncode') 985863Snate@binkert.org 996654Snate@binkert.org 100955SN/A 1015396Ssaidi@eecs.umich.edutest_phase_classes = {} 1025863Snate@binkert.org 1035863Snate@binkert.orgclass TestPhaseMeta(type): 1044202Sbinkertn@umich.edu def __init__(cls, name, bases, d): 1055863Snate@binkert.org if not d.pop('abstract', False): 1065863Snate@binkert.org test_phase_classes[d['name']] = cls 1075863Snate@binkert.org 1085863Snate@binkert.org super(TestPhaseMeta, cls).__init__(name, bases, d) 109955SN/A 1106654Snate@binkert.orgclass TestPhaseBase(object): 1115273Sstever@gmail.com __metaclass__ = TestPhaseMeta 1125871Snate@binkert.org abstract = True 1135273Sstever@gmail.com 1146655Snate@binkert.org def __init__(self, main_args, *args): 1156655Snate@binkert.org self.main_args = main_args 1166655Snate@binkert.org self.args = args 1176655Snate@binkert.org 1186655Snate@binkert.org def __lt__(self, other): 1196655Snate@binkert.org return self.number < other.number 1205871Snate@binkert.org 1216654Snate@binkert.orgclass CompilePhase(TestPhaseBase): 1225396Ssaidi@eecs.umich.edu name = 'compile' 1235871Snate@binkert.org number = 1 1245871Snate@binkert.org 1256121Snate@binkert.org def run(self, tests): 1265871Snate@binkert.org targets = list([test.full_path() for test in tests]) 1275871Snate@binkert.org scons_args = [ 'USE_SYSTEMC=1' ] + list(self.args) + targets 1286003Snate@binkert.org scons(*scons_args) 1296655Snate@binkert.org 130955SN/Aclass RunPhase(TestPhaseBase): 1315871Snate@binkert.org name = 'execute' 1325871Snate@binkert.org number = 2 1335871Snate@binkert.org 1345871Snate@binkert.org def run(self, tests): 135955SN/A parser = argparse.ArgumentParser() 1366121Snate@binkert.org parser.add_argument('--timeout', type=int, metavar='SECONDS', 1376121Snate@binkert.org help='Time limit for each run in seconds.', 1386121Snate@binkert.org default=0) 1391533SN/A parser.add_argument('-j', type=int, default=1, 1406655Snate@binkert.org help='How many tests to run in parallel.') 1416655Snate@binkert.org args = parser.parse_args(self.args) 1426655Snate@binkert.org 1436655Snate@binkert.org timeout_cmd = [ 1445871Snate@binkert.org 'timeout', 1455871Snate@binkert.org '--kill-after', str(args.timeout * 2), 1465863Snate@binkert.org str(args.timeout) 1475871Snate@binkert.org ] 1485871Snate@binkert.org curdir = os.getcwd() 1495871Snate@binkert.org def run_test(test): 1505871Snate@binkert.org cmd = [] 1515871Snate@binkert.org if args.timeout: 1525863Snate@binkert.org cmd.extend(timeout_cmd) 1536121Snate@binkert.org cmd.extend([ 1545863Snate@binkert.org test.full_path(), 1555871Snate@binkert.org '-red', os.path.abspath(test.m5out_dir()), 1564678Snate@binkert.org '--listener-mode=off', 1574678Snate@binkert.org '--quiet', 1584678Snate@binkert.org config_path, 1594678Snate@binkert.org '--working-dir', 1604678Snate@binkert.org os.path.dirname(test.src_dir()) 1614678Snate@binkert.org ]) 1624678Snate@binkert.org # Ensure the output directory exists. 1634678Snate@binkert.org if not os.path.exists(test.m5out_dir()): 1644678Snate@binkert.org os.makedirs(test.m5out_dir()) 1654678Snate@binkert.org try: 1664678Snate@binkert.org subprocess.check_call(cmd) 1674678Snate@binkert.org except subprocess.CalledProcessError, error: 1686121Snate@binkert.org returncode = error.returncode 1694678Snate@binkert.org else: 1705871Snate@binkert.org returncode = 0 1715871Snate@binkert.org os.chdir(curdir) 1725871Snate@binkert.org with open(test.returncode_file(), 'w') as rc: 1735871Snate@binkert.org rc.write('%d\n' % returncode) 1745871Snate@binkert.org 1755871Snate@binkert.org runnable = filter(lambda t: not t.compile_only, tests) 1765871Snate@binkert.org if args.j == 1: 1775871Snate@binkert.org map(run_test, runnable) 1785871Snate@binkert.org else: 1795871Snate@binkert.org tp = multiprocessing.pool.ThreadPool(args.j) 1805871Snate@binkert.org map(lambda t: tp.apply_async(run_test, (t,)), runnable) 1815871Snate@binkert.org tp.close() 1825871Snate@binkert.org tp.join() 1835990Ssaidi@eecs.umich.edu 1845871Snate@binkert.orgclass Checker(object): 1855871Snate@binkert.org def __init__(self, ref, test, tag): 1865871Snate@binkert.org self.ref = ref 1874678Snate@binkert.org self.test = test 1886654Snate@binkert.org self.tag = tag 1895871Snate@binkert.org 1905871Snate@binkert.org def check(self): 1915871Snate@binkert.org with open(self.text) as test_f, open(self.ref) as ref_f: 1925871Snate@binkert.org return test_f.read() == ref_f.read() 1935871Snate@binkert.org 1945871Snate@binkert.orgdef tagged_filt(tag, num): 1955871Snate@binkert.org return (r'^\n{}: \({}{}\) .*\n(In file: .*\n)?' 1965871Snate@binkert.org r'(In process: [\w.]* @ .*\n)?').format(tag, tag[0], num) 1975871Snate@binkert.org 1984678Snate@binkert.orgdef error_filt(num): 1995871Snate@binkert.org return tagged_filt('Error', num) 2004678Snate@binkert.org 2015871Snate@binkert.orgdef warning_filt(num): 2025871Snate@binkert.org return tagged_filt('Warning', num) 2035871Snate@binkert.org 2045871Snate@binkert.orgdef info_filt(num): 2055871Snate@binkert.org return tagged_filt('Info', num) 2065871Snate@binkert.org 2075871Snate@binkert.orgclass LogChecker(Checker): 2085871Snate@binkert.org def merge_filts(*filts): 2095871Snate@binkert.org filts = map(lambda f: '(' + f + ')', filts) 2106121Snate@binkert.org filts = '|'.join(filts) 2116121Snate@binkert.org return re.compile(filts, flags=re.MULTILINE) 2125863Snate@binkert.org 213955SN/A # The reporting mechanism will print the actual filename when running in 214955SN/A # gem5, and the "golden" output will say "<removed by verify.py>". We want 2152632Sstever@eecs.umich.edu # to strip out both versions to make comparing the output sensible. 2162632Sstever@eecs.umich.edu in_file_filt = r'^In file: ((<removed by verify\.pl>)|([a-zA-Z0-9.:_/]*))$' 217955SN/A 218955SN/A ref_filt = merge_filts( 219955SN/A r'^\nInfo: /OSCI/SystemC: Simulation stopped by user.\n', 220955SN/A r'^SystemC Simulation\n', 2215863Snate@binkert.org r'^\nInfo: \(I804\) /IEEE_Std_1666/deprecated: ' + 222955SN/A r'You can turn off(.*\n){7}', 2232632Sstever@eecs.umich.edu r'^\nInfo: \(I804\) /IEEE_Std_1666/deprecated: \n' + 2242632Sstever@eecs.umich.edu r' sc_clock\(const char(.*\n){3}', 2252632Sstever@eecs.umich.edu warning_filt(540), 2262632Sstever@eecs.umich.edu warning_filt(569), 2272632Sstever@eecs.umich.edu warning_filt(571), 2282632Sstever@eecs.umich.edu error_filt(541), 2292632Sstever@eecs.umich.edu error_filt(542), 2302632Sstever@eecs.umich.edu error_filt(543), 2312632Sstever@eecs.umich.edu info_filt(804), 2322632Sstever@eecs.umich.edu in_file_filt, 2332632Sstever@eecs.umich.edu ) 2342632Sstever@eecs.umich.edu test_filt = merge_filts( 2352632Sstever@eecs.umich.edu r'^Global frequency set at \d* ticks per second\n', 2363718Sstever@eecs.umich.edu info_filt(804), 2373718Sstever@eecs.umich.edu in_file_filt, 2383718Sstever@eecs.umich.edu ) 2393718Sstever@eecs.umich.edu 2403718Sstever@eecs.umich.edu def __init__(self, ref, test, tag, out_dir): 2415863Snate@binkert.org super(LogChecker, self).__init__(ref, test, tag) 2425863Snate@binkert.org self.out_dir = out_dir 2433718Sstever@eecs.umich.edu 2443718Sstever@eecs.umich.edu def apply_filters(self, data, filts): 2456121Snate@binkert.org re.sub(filt, '', data) 2465863Snate@binkert.org 2473718Sstever@eecs.umich.edu def check(self): 2483718Sstever@eecs.umich.edu test_file = os.path.basename(self.test) 2492634Sstever@eecs.umich.edu ref_file = os.path.basename(self.ref) 2502634Sstever@eecs.umich.edu with open(self.test) as test_f, open(self.ref) as ref_f: 2515863Snate@binkert.org test = re.sub(self.test_filt, '', test_f.read()) 2522638Sstever@eecs.umich.edu ref = re.sub(self.ref_filt, '', ref_f.read()) 2532632Sstever@eecs.umich.edu diff_file = '.'.join([ref_file, 'diff']) 2542632Sstever@eecs.umich.edu diff_path = os.path.join(self.out_dir, diff_file) 2552632Sstever@eecs.umich.edu if test != ref: 2562632Sstever@eecs.umich.edu with open(diff_path, 'w') as diff_f: 2572632Sstever@eecs.umich.edu for line in difflib.unified_diff( 2582632Sstever@eecs.umich.edu ref.splitlines(True), test.splitlines(True), 2591858SN/A fromfile=ref_file, 2603716Sstever@eecs.umich.edu tofile=test_file): 2612638Sstever@eecs.umich.edu diff_f.write(line) 2622638Sstever@eecs.umich.edu return False 2632638Sstever@eecs.umich.edu else: 2642638Sstever@eecs.umich.edu if os.path.exists(diff_path): 2652638Sstever@eecs.umich.edu os.unlink(diff_path) 2662638Sstever@eecs.umich.edu return True 2672638Sstever@eecs.umich.edu 2685863Snate@binkert.orgclass GoldenDir(object): 2695863Snate@binkert.org def __init__(self, path, platform): 2705863Snate@binkert.org self.path = path 271955SN/A self.platform = platform 2725341Sstever@gmail.com 2735341Sstever@gmail.com contents = os.listdir(path) 2745863Snate@binkert.org suffix = '.' + platform 2755341Sstever@gmail.com suffixed = filter(lambda c: c.endswith(suffix), contents) 2766121Snate@binkert.org bases = map(lambda t: t[:-len(platform)], suffixed) 2774494Ssaidi@eecs.umich.edu common = filter(lambda t: not t.startswith(tuple(bases)), contents) 2786121Snate@binkert.org 2791105SN/A self.entries = {} 2802667Sstever@eecs.umich.edu class Entry(object): 2812667Sstever@eecs.umich.edu def __init__(self, e_path): 2822667Sstever@eecs.umich.edu self.used = False 2832667Sstever@eecs.umich.edu self.path = os.path.join(path, e_path) 2846121Snate@binkert.org 2852667Sstever@eecs.umich.edu def use(self): 2865341Sstever@gmail.com self.used = True 2875863Snate@binkert.org 2885341Sstever@gmail.com for entry in contents: 2895341Sstever@gmail.com self.entries[entry] = Entry(entry) 2905341Sstever@gmail.com 2915863Snate@binkert.org def entry(self, name): 2925341Sstever@gmail.com def match(n): 2935341Sstever@gmail.com return (n == name) or n.startswith(name + '.') 2945341Sstever@gmail.com matches = { n: e for n, e in self.entries.items() if match(n) } 2955863Snate@binkert.org 2965341Sstever@gmail.com for match in matches.values(): 2975341Sstever@gmail.com match.use() 2985341Sstever@gmail.com 2995341Sstever@gmail.com platform_name = '.'.join([ name, self.platform ]) 3005341Sstever@gmail.com if platform_name in matches: 3015341Sstever@gmail.com return matches[platform_name].path 3025341Sstever@gmail.com if name in matches: 3035341Sstever@gmail.com return matches[name].path 3045341Sstever@gmail.com else: 3055341Sstever@gmail.com return None 3065863Snate@binkert.org 3075341Sstever@gmail.com def unused(self): 3085863Snate@binkert.org items = self.entries.items() 3095341Sstever@gmail.com items = filter(lambda i: not i[1].used, items) 3105863Snate@binkert.org 3116121Snate@binkert.org items.sort() 3126121Snate@binkert.org sources = [] 3135397Ssaidi@eecs.umich.edu i = 0 3145397Ssaidi@eecs.umich.edu while i < len(items): 3155341Sstever@gmail.com root = items[i][0] 3166168Snate@binkert.org sources.append(root) 3176168Snate@binkert.org i += 1 3185341Sstever@gmail.com while i < len(items) and items[i][0].startswith(root): 3195341Sstever@gmail.com i += 1 3205341Sstever@gmail.com return sources 3215341Sstever@gmail.com 3225341Sstever@gmail.comclass VerifyPhase(TestPhaseBase): 3235863Snate@binkert.org name = 'verify' 3245341Sstever@gmail.com number = 3 3255341Sstever@gmail.com 3266121Snate@binkert.org def reset_status(self): 3276121Snate@binkert.org self._passed = [] 3285341Sstever@gmail.com self._failed = {} 3296814Sgblack@eecs.umich.edu 3306814Sgblack@eecs.umich.edu def passed(self, test): 3315863Snate@binkert.org self._passed.append(test) 3326121Snate@binkert.org 3335341Sstever@gmail.com def failed(self, test, cause, note=''): 3345863Snate@binkert.org test.set_prop('note', note) 3355341Sstever@gmail.com self._failed.setdefault(cause, []).append(test) 3366121Snate@binkert.org 3376121Snate@binkert.org def print_status(self): 3386121Snate@binkert.org total_passed = len(self._passed) 3395742Snate@binkert.org total_failed = sum(map(len, self._failed.values())) 3405742Snate@binkert.org print() 3415341Sstever@gmail.com print('Passed: {passed:4} - Failed: {failed:4}'.format( 3425742Snate@binkert.org passed=total_passed, failed=total_failed)) 3435742Snate@binkert.org 3445341Sstever@gmail.com def write_result_file(self, path): 3456017Snate@binkert.org results = { 3466121Snate@binkert.org 'passed': map(lambda t: t.props, self._passed), 3476017Snate@binkert.org 'failed': { 3486654Snate@binkert.org cause: map(lambda t: t.props, tests) for 3496654Snate@binkert.org cause, tests in self._failed.iteritems() 3505871Snate@binkert.org } 3516121Snate@binkert.org } 3526121Snate@binkert.org with open(path, 'w') as rf: 3536121Snate@binkert.org json.dump(results, rf) 3546121Snate@binkert.org 3553940Ssaidi@eecs.umich.edu def print_results(self): 3563918Ssaidi@eecs.umich.edu print() 3573918Ssaidi@eecs.umich.edu print('Passed:') 3581858SN/A for path in sorted(list([ t.path for t in self._passed ])): 3596121Snate@binkert.org print(' ', path) 3606121Snate@binkert.org 3616121Snate@binkert.org print() 3626143Snate@binkert.org print('Failed:') 3636121Snate@binkert.org 3646121Snate@binkert.org causes = [] 3653940Ssaidi@eecs.umich.edu for cause, tests in sorted(self._failed.items()): 3666121Snate@binkert.org block = ' ' + cause.capitalize() + ':\n' 3676121Snate@binkert.org for test in sorted(tests, key=lambda t: t.path): 3686121Snate@binkert.org block += ' ' + test.path 3696121Snate@binkert.org if test.note: 3706121Snate@binkert.org block += ' - ' + test.note 3716121Snate@binkert.org block += '\n' 3726121Snate@binkert.org causes.append(block) 3733918Ssaidi@eecs.umich.edu 3743918Ssaidi@eecs.umich.edu print('\n'.join(causes)) 3753940Ssaidi@eecs.umich.edu 3763918Ssaidi@eecs.umich.edu def run(self, tests): 3773918Ssaidi@eecs.umich.edu parser = argparse.ArgumentParser() 3786157Snate@binkert.org result_opts = parser.add_mutually_exclusive_group() 3796157Snate@binkert.org result_opts.add_argument('--result-file', action='store_true', 3806157Snate@binkert.org help='Create a results.json file in the current directory.') 3816157Snate@binkert.org result_opts.add_argument('--result-file-at', metavar='PATH', 3825397Ssaidi@eecs.umich.edu help='Create a results json file at the given path.') 3835397Ssaidi@eecs.umich.edu parser.add_argument('--print-results', action='store_true', 3846121Snate@binkert.org help='Print a list of tests that passed or failed') 3856121Snate@binkert.org args = parser.parse_args(self.args) 3866121Snate@binkert.org 3876121Snate@binkert.org self.reset_status() 3886121Snate@binkert.org 3896121Snate@binkert.org runnable = filter(lambda t: not t.compile_only, tests) 3905397Ssaidi@eecs.umich.edu compile_only = filter(lambda t: t.compile_only, tests) 3911851SN/A 3921851SN/A for test in compile_only: 3936655Snate@binkert.org if os.path.exists(test.full_path()): 394955SN/A self.passed(test) 3953053Sstever@eecs.umich.edu else: 3966121Snate@binkert.org self.failed(test, 'compile failed') 3973053Sstever@eecs.umich.edu 3983053Sstever@eecs.umich.edu for test in runnable: 3993053Sstever@eecs.umich.edu with open(test.returncode_file()) as rc: 4003053Sstever@eecs.umich.edu returncode = int(rc.read()) 4013053Sstever@eecs.umich.edu 4026654Snate@binkert.org expected_returncode = 0 4033053Sstever@eecs.umich.edu if os.path.exists(test.expected_returncode_file()): 4044742Sstever@eecs.umich.edu with open(test.expected_returncode_file()) as erc: 4054742Sstever@eecs.umich.edu expected_returncode = int(erc.read()) 4063053Sstever@eecs.umich.edu 4073053Sstever@eecs.umich.edu if returncode == 124: 4083053Sstever@eecs.umich.edu self.failed(test, 'time out') 4093053Sstever@eecs.umich.edu continue 4106654Snate@binkert.org elif returncode != expected_returncode: 4113053Sstever@eecs.umich.edu if expected_returncode == 0: 4123053Sstever@eecs.umich.edu self.failed(test, 'abort') 4133053Sstever@eecs.umich.edu else: 4143053Sstever@eecs.umich.edu self.failed(test, 'missed abort') 4152667Sstever@eecs.umich.edu continue 4164554Sbinkertn@umich.edu 4176121Snate@binkert.org out_dir = test.m5out_dir() 4182667Sstever@eecs.umich.edu 4194554Sbinkertn@umich.edu Diff = collections.namedtuple( 4204554Sbinkertn@umich.edu 'Diff', 'ref, test, tag, ref_filter') 4214554Sbinkertn@umich.edu 4226121Snate@binkert.org diffs = [] 4234554Sbinkertn@umich.edu 4244554Sbinkertn@umich.edu gd = GoldenDir(test.golden_dir(), 'linux64') 4254554Sbinkertn@umich.edu 4264781Snate@binkert.org missing = [] 4274554Sbinkertn@umich.edu log_file = '.'.join([test.name, 'log']) 4284554Sbinkertn@umich.edu log_path = gd.entry(log_file) 4292667Sstever@eecs.umich.edu simout_path = os.path.join(out_dir, 'simout') 4304554Sbinkertn@umich.edu if not os.path.exists(simout_path): 4314554Sbinkertn@umich.edu missing.append('log output') 4324554Sbinkertn@umich.edu elif log_path: 4334554Sbinkertn@umich.edu diffs.append(LogChecker(log_path, simout_path, 4342667Sstever@eecs.umich.edu log_file, out_dir)) 4354554Sbinkertn@umich.edu 4362667Sstever@eecs.umich.edu for name in gd.unused(): 4374554Sbinkertn@umich.edu test_path = os.path.join(out_dir, name) 4386121Snate@binkert.org ref_path = gd.entry(name) 4392667Sstever@eecs.umich.edu if not os.path.exists(test_path): 4405522Snate@binkert.org missing.append(name) 4415522Snate@binkert.org else: 4425522Snate@binkert.org diffs.append(Checker(ref_path, test_path, name)) 4435522Snate@binkert.org 4445522Snate@binkert.org if missing: 4455522Snate@binkert.org self.failed(test, 'missing output', ' '.join(missing)) 4465522Snate@binkert.org continue 4475522Snate@binkert.org 4485522Snate@binkert.org failed_diffs = filter(lambda d: not d.check(), diffs) 4495522Snate@binkert.org if failed_diffs: 4505522Snate@binkert.org tags = map(lambda d: d.tag, failed_diffs) 4515522Snate@binkert.org self.failed(test, 'failed diffs', ' '.join(tags)) 4525522Snate@binkert.org continue 4535522Snate@binkert.org 4545522Snate@binkert.org self.passed(test) 4555522Snate@binkert.org 4565522Snate@binkert.org if args.print_results: 4575522Snate@binkert.org self.print_results() 4585522Snate@binkert.org 4595522Snate@binkert.org self.print_status() 4605522Snate@binkert.org 4615522Snate@binkert.org result_path = None 4625522Snate@binkert.org if args.result_file: 4635522Snate@binkert.org result_path = os.path.join(os.getcwd(), 'results.json') 4645522Snate@binkert.org elif args.result_file_at: 4655522Snate@binkert.org result_path = args.result_file_at 4662638Sstever@eecs.umich.edu 4672638Sstever@eecs.umich.edu if result_path: 4686121Snate@binkert.org self.write_result_file(result_path) 4693716Sstever@eecs.umich.edu 4705522Snate@binkert.org 4715522Snate@binkert.orgparser = argparse.ArgumentParser(description='SystemC test utility') 4725522Snate@binkert.org 4735522Snate@binkert.orgparser.add_argument('build_dir', metavar='BUILD_DIR', 4745522Snate@binkert.org help='The build directory (ie. build/ARM).') 4755522Snate@binkert.org 4761858SN/Aparser.add_argument('--update-json', action='store_true', 4775227Ssaidi@eecs.umich.edu help='Update the json manifest of tests.') 4785227Ssaidi@eecs.umich.edu 4795227Ssaidi@eecs.umich.eduparser.add_argument('--flavor', choices=['debug', 'opt', 'fast'], 4805227Ssaidi@eecs.umich.edu default='opt', 4816654Snate@binkert.org help='Flavor of binary to test.') 4826654Snate@binkert.org 4836121Snate@binkert.orgparser.add_argument('--list', action='store_true', 4846121Snate@binkert.org help='List the available tests') 4856121Snate@binkert.org 4866121Snate@binkert.orgfilter_opts = parser.add_mutually_exclusive_group() 4875227Ssaidi@eecs.umich.edufilter_opts.add_argument('--filter', default='True', 4885227Ssaidi@eecs.umich.edu help='Python expression which filters tests based ' 4895227Ssaidi@eecs.umich.edu 'on their properties') 4905204Sstever@gmail.comfilter_opts.add_argument('--filter-file', default=None, 4915204Sstever@gmail.com type=argparse.FileType('r'), 4925204Sstever@gmail.com help='Same as --filter, but read from a file') 4935204Sstever@gmail.com 4945204Sstever@gmail.comdef collect_phases(args): 4955204Sstever@gmail.com phase_groups = [list(g) for k, g in 4965204Sstever@gmail.com itertools.groupby(args, lambda x: x != '--phase') if k] 4975204Sstever@gmail.com main_args = parser.parse_args(phase_groups[0][1:]) 4985204Sstever@gmail.com phases = [] 4995204Sstever@gmail.com names = [] 5005204Sstever@gmail.com for group in phase_groups[1:]: 5015204Sstever@gmail.com name = group[0] 5025204Sstever@gmail.com if name in names: 5035204Sstever@gmail.com raise RuntimeException('Phase %s specified more than once' % name) 5045204Sstever@gmail.com phase = test_phase_classes[name] 5055204Sstever@gmail.com phases.append(phase(main_args, *group[1:])) 5065204Sstever@gmail.com phases.sort() 5076121Snate@binkert.org return main_args, phases 5085204Sstever@gmail.com 5093118Sstever@eecs.umich.edumain_args, phases = collect_phases(sys.argv) 5103118Sstever@eecs.umich.edu 5113118Sstever@eecs.umich.eduif len(phases) == 0: 5123118Sstever@eecs.umich.edu phases = [ 5133118Sstever@eecs.umich.edu CompilePhase(main_args), 5145863Snate@binkert.org RunPhase(main_args), 5153118Sstever@eecs.umich.edu VerifyPhase(main_args) 5165863Snate@binkert.org ] 5173118Sstever@eecs.umich.edu 5185863Snate@binkert.org 5195863Snate@binkert.org 5205863Snate@binkert.orgjson_path = os.path.join(main_args.build_dir, json_rel_path) 5215863Snate@binkert.org 5225863Snate@binkert.orgif main_args.update_json: 5235863Snate@binkert.org scons(os.path.join(json_path)) 5245863Snate@binkert.org 5255863Snate@binkert.orgwith open(json_path) as f: 5266003Snate@binkert.org test_data = json.load(f) 5275863Snate@binkert.org 5285863Snate@binkert.org if main_args.filter_file: 5295863Snate@binkert.org f = main_args.filter_file 5306120Snate@binkert.org filt = compile(f.read(), f.name, 'eval') 5315863Snate@binkert.org else: 5325863Snate@binkert.org filt = compile(main_args.filter, '<string>', 'eval') 5335863Snate@binkert.org 5346120Snate@binkert.org filtered_tests = { 5356120Snate@binkert.org target: props for (target, props) in 5365863Snate@binkert.org test_data.iteritems() if eval(filt, dict(props)) 5375863Snate@binkert.org } 5386120Snate@binkert.org 5395863Snate@binkert.org if len(filtered_tests) == 0: 5406121Snate@binkert.org print('All tests were filtered out.') 5416121Snate@binkert.org exit() 5425863Snate@binkert.org 5435863Snate@binkert.org if main_args.list: 5443118Sstever@eecs.umich.edu for target, props in sorted(filtered_tests.iteritems()): 5455863Snate@binkert.org print('%s.%s' % (target, main_args.flavor)) 5463118Sstever@eecs.umich.edu for key, val in props.iteritems(): 5473118Sstever@eecs.umich.edu print(' %s: %s' % (key, val)) 5485863Snate@binkert.org print('Total tests: %d' % len(filtered_tests)) 5495863Snate@binkert.org else: 5505863Snate@binkert.org tests_to_run = list([ 5515863Snate@binkert.org Test(target, main_args.flavor, main_args.build_dir, props) for 5523118Sstever@eecs.umich.edu target, props in sorted(filtered_tests.iteritems()) 5533483Ssaidi@eecs.umich.edu ]) 5543494Ssaidi@eecs.umich.edu 5553494Ssaidi@eecs.umich.edu for phase in phases: 5563483Ssaidi@eecs.umich.edu phase.run(tests_to_run) 5573483Ssaidi@eecs.umich.edu