verify.py revision 13101
11689SN/A#!/usr/bin/env python2 22325SN/A# 31689SN/A# Copyright 2018 Google, Inc. 41689SN/A# 51689SN/A# Redistribution and use in source and binary forms, with or without 61689SN/A# modification, are permitted provided that the following conditions are 71689SN/A# met: redistributions of source code must retain the above copyright 81689SN/A# notice, this list of conditions and the following disclaimer; 91689SN/A# redistributions in binary form must reproduce the above copyright 101689SN/A# notice, this list of conditions and the following disclaimer in the 111689SN/A# documentation and/or other materials provided with the distribution; 121689SN/A# neither the name of the copyright holders nor the names of its 131689SN/A# contributors may be used to endorse or promote products derived from 141689SN/A# this software without specific prior written permission. 151689SN/A# 161689SN/A# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS 171689SN/A# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT 181689SN/A# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR 191689SN/A# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT 201689SN/A# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, 211689SN/A# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT 221689SN/A# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, 231689SN/A# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY 241689SN/A# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT 251689SN/A# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE 261689SN/A# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 272665Ssaidi@eecs.umich.edu# 282665Ssaidi@eecs.umich.edu# Authors: Gabe Black 292756Sksewell@umich.edu 301689SN/Afrom __future__ import print_function 311689SN/A 321858SN/Aimport argparse 332733Sktlim@umich.eduimport collections 341858SN/Aimport difflib 351858SN/Aimport functools 361060SN/Aimport inspect 371060SN/Aimport itertools 381060SN/Aimport json 391060SN/Aimport multiprocessing.pool 401060SN/Aimport os 412325SN/Aimport re 422683Sktlim@umich.eduimport subprocess 432680Sktlim@umich.eduimport sys 442817Sksewell@umich.edu 451717SN/Ascript_path = os.path.abspath(inspect.getfile(inspect.currentframe())) 461060SN/Ascript_dir = os.path.dirname(script_path) 472325SN/Aconfig_path = os.path.join(script_dir, 'config.py') 482292SN/A 492292SN/Asystemc_rel_path = 'systemc' 502794Sktlim@umich.edutests_rel_path = os.path.join(systemc_rel_path, 'tests') 512794Sktlim@umich.edujson_rel_path = os.path.join(tests_rel_path, 'tests.json') 522794Sktlim@umich.edu 532794Sktlim@umich.edu 541060SN/A 552669Sktlim@umich.edudef scons(*args): 561060SN/A args = ['scons'] + list(args) 572733Sktlim@umich.edu subprocess.check_call(args) 582292SN/A 591060SN/A 601060SN/A 611060SN/Aclass Test(object): 622292SN/A def __init__(self, target, suffix, build_dir, props): 632733Sktlim@umich.edu self.target = target 642292SN/A self.suffix = suffix 652292SN/A self.build_dir = build_dir 662292SN/A self.props = {} 672292SN/A 681060SN/A for key, val in props.iteritems(): 691755SN/A self.set_prop(key, val) 701060SN/A 711060SN/A def set_prop(self, key, val): 721060SN/A setattr(self, key, val) 731060SN/A self.props[key] = val 741060SN/A 751060SN/A def dir(self): 761755SN/A return os.path.join(self.build_dir, tests_rel_path, self.path) 771060SN/A 781060SN/A def src_dir(self): 791060SN/A return os.path.join(script_dir, self.path) 801060SN/A 811060SN/A def golden_dir(self): 821060SN/A return os.path.join(self.src_dir(), 'golden') 831755SN/A 841060SN/A def bin(self): 851755SN/A return '.'.join([self.name, self.suffix]) 861060SN/A 871060SN/A def full_path(self): 881060SN/A return os.path.join(self.dir(), self.bin()) 892829Sksewell@umich.edu 902829Sksewell@umich.edu def m5out_dir(self): 912829Sksewell@umich.edu return os.path.join(self.dir(), 'm5out.' + self.suffix) 922829Sksewell@umich.edu 932829Sksewell@umich.edu def returncode_file(self): 942829Sksewell@umich.edu return os.path.join(self.m5out_dir(), 'returncode') 952829Sksewell@umich.edu 962829Sksewell@umich.edu 972829Sksewell@umich.edu 982829Sksewell@umich.edutest_phase_classes = {} 992829Sksewell@umich.edu 1002829Sksewell@umich.educlass TestPhaseMeta(type): 1012829Sksewell@umich.edu def __init__(cls, name, bases, d): 1022829Sksewell@umich.edu if not d.pop('abstract', False): 1032829Sksewell@umich.edu test_phase_classes[d['name']] = cls 1042829Sksewell@umich.edu 1052829Sksewell@umich.edu super(TestPhaseMeta, cls).__init__(name, bases, d) 1062829Sksewell@umich.edu 1072829Sksewell@umich.educlass TestPhaseBase(object): 1082829Sksewell@umich.edu __metaclass__ = TestPhaseMeta 1092829Sksewell@umich.edu abstract = True 1102829Sksewell@umich.edu 1112829Sksewell@umich.edu def __init__(self, main_args, *args): 1122829Sksewell@umich.edu self.main_args = main_args 1132829Sksewell@umich.edu self.args = args 1142829Sksewell@umich.edu 1152829Sksewell@umich.edu def __lt__(self, other): 1162829Sksewell@umich.edu return self.number < other.number 1172829Sksewell@umich.edu 1182875Sksewell@umich.educlass CompilePhase(TestPhaseBase): 1192875Sksewell@umich.edu name = 'compile' 1202875Sksewell@umich.edu number = 1 1212875Sksewell@umich.edu 1222875Sksewell@umich.edu def run(self, tests): 1232875Sksewell@umich.edu targets = list([test.full_path() for test in tests]) 1242875Sksewell@umich.edu scons_args = [ 'USE_SYSTEMC=1' ] + list(self.args) + targets 1252875Sksewell@umich.edu scons(*scons_args) 1262875Sksewell@umich.edu 1272875Sksewell@umich.educlass RunPhase(TestPhaseBase): 1282875Sksewell@umich.edu name = 'execute' 1292875Sksewell@umich.edu number = 2 1302875Sksewell@umich.edu 1312875Sksewell@umich.edu def run(self, tests): 1322875Sksewell@umich.edu parser = argparse.ArgumentParser() 1332875Sksewell@umich.edu parser.add_argument('--timeout', type=int, metavar='SECONDS', 1342875Sksewell@umich.edu help='Time limit for each run in seconds.', 1352875Sksewell@umich.edu default=0) 1362875Sksewell@umich.edu parser.add_argument('-j', type=int, default=1, 1372875Sksewell@umich.edu help='How many tests to run in parallel.') 1382875Sksewell@umich.edu args = parser.parse_args(self.args) 1392875Sksewell@umich.edu 1402875Sksewell@umich.edu timeout_cmd = [ 1412875Sksewell@umich.edu 'timeout', 1422875Sksewell@umich.edu '--kill-after', str(args.timeout * 2), 1432875Sksewell@umich.edu str(args.timeout) 1442875Sksewell@umich.edu ] 1452875Sksewell@umich.edu curdir = os.getcwd() 1462875Sksewell@umich.edu def run_test(test): 1472875Sksewell@umich.edu cmd = [] 1482292SN/A if args.timeout: 1492733Sktlim@umich.edu cmd.extend(timeout_cmd) 1501060SN/A cmd.extend([ 1512292SN/A test.full_path(), 1521060SN/A '-red', os.path.abspath(test.m5out_dir()), 1531060SN/A '--listener-mode=off', 1541060SN/A '--quiet', 1551060SN/A config_path, 1561060SN/A '--working-dir', 1571060SN/A os.path.dirname(test.src_dir()) 1582292SN/A ]) 1591060SN/A # Ensure the output directory exists. 1602831Sksewell@umich.edu if not os.path.exists(test.m5out_dir()): 1612292SN/A os.makedirs(test.m5out_dir()) 1622292SN/A try: 1631060SN/A subprocess.check_call(cmd) 1642292SN/A except subprocess.CalledProcessError, error: 1652292SN/A returncode = error.returncode 1662292SN/A else: 1671060SN/A returncode = 0 1682831Sksewell@umich.edu os.chdir(curdir) 1692292SN/A with open(test.returncode_file(), 'w') as rc: 1702292SN/A rc.write('%d\n' % returncode) 1712292SN/A 1722292SN/A runnable = filter(lambda t: not t.compile_only, tests) 1731060SN/A if args.j == 1: 1742873Sktlim@umich.edu map(run_test, runnable) 1752873Sktlim@umich.edu else: 1762873Sktlim@umich.edu tp = multiprocessing.pool.ThreadPool(args.j) 1772873Sktlim@umich.edu map(lambda t: tp.apply_async(run_test, (t,)), runnable) 1782873Sktlim@umich.edu tp.close() 1792873Sktlim@umich.edu tp.join() 1802873Sktlim@umich.edu 1812873Sktlim@umich.educlass Checker(object): 1821060SN/A def __init__(self, ref, test, tag): 1831060SN/A self.ref = ref 1841060SN/A self.test = test 1851858SN/A self.tag = tag 1862292SN/A 1871060SN/A def check(self): 1881060SN/A with open(self.text) as test_f, open(self.ref) as ref_f: 1892292SN/A return test_f.read() == ref_f.read() 1902843Sktlim@umich.edu 1912316SN/Adef tagged_filt(tag, num): 1922316SN/A return (r'^\n{}: \({}{}\) .*\n(In file: .*\n)?' 1931060SN/A r'(In process: [\w.]* @ .*\n)?').format(tag, tag[0], num) 1941060SN/A 1951681SN/Adef warning_filt(num): 1962733Sktlim@umich.edu return tagged_filt('Warning', num) 1972733Sktlim@umich.edu 1982794Sktlim@umich.edudef info_filt(num): 1992733Sktlim@umich.edu return tagged_filt('Info', num) 2002316SN/A 2012316SN/Aclass LogChecker(Checker): 2022316SN/A def merge_filts(*filts): 2032316SN/A filts = map(lambda f: '(' + f + ')', filts) 2042316SN/A filts = '|'.join(filts) 2052316SN/A return re.compile(filts, flags=re.MULTILINE) 2062794Sktlim@umich.edu 2072794Sktlim@umich.edu ref_filt = merge_filts( 2082794Sktlim@umich.edu r'^\nInfo: /OSCI/SystemC: Simulation stopped by user.\n', 2092316SN/A r'^SystemC Simulation\n', 2102316SN/A r'^\nInfo: \(I804\) /IEEE_Std_1666/deprecated: ' + 2111858SN/A r'You can turn off(.*\n){7}', 2122292SN/A r'^\nInfo: \(I804\) /IEEE_Std_1666/deprecated: \n' + 2132292SN/A r' sc_clock\(const char(.*\n){3}', 2141681SN/A warning_filt(540), 2151681SN/A warning_filt(569), 2162325SN/A warning_filt(571), 2172325SN/A info_filt(804), 2182325SN/A ) 2191060SN/A test_filt = merge_filts( 2202292SN/A r'^Global frequency set at \d* ticks per second\n', 2212292SN/A info_filt(804), 2222292SN/A ) 2232292SN/A 2242292SN/A def __init__(self, ref, test, tag, out_dir): 2252292SN/A super(LogChecker, self).__init__(ref, test, tag) 2261060SN/A self.out_dir = out_dir 2271060SN/A 2281060SN/A def apply_filters(self, data, filts): 2291060SN/A re.sub(filt, '', data) 2301060SN/A 2311060SN/A def check(self): 2321060SN/A test_file = os.path.basename(self.test) 2331060SN/A ref_file = os.path.basename(self.ref) 2341060SN/A with open(self.test) as test_f, open(self.ref) as ref_f: 2351060SN/A test = re.sub(self.test_filt, '', test_f.read()) 2361060SN/A ref = re.sub(self.ref_filt, '', ref_f.read()) 2372292SN/A diff_file = '.'.join([ref_file, 'diff']) 2381060SN/A diff_path = os.path.join(self.out_dir, diff_file) 2391060SN/A if test != ref: 2401060SN/A with open(diff_path, 'w') as diff_f: 2411060SN/A for line in difflib.unified_diff( 2421060SN/A ref.splitlines(True), test.splitlines(True), 2431060SN/A fromfile=ref_file, 2441060SN/A tofile=test_file): 2451060SN/A diff_f.write(line) 2462292SN/A return False 2472292SN/A else: 2482292SN/A if os.path.exists(diff_path): 2492292SN/A os.unlink(diff_path) 2502292SN/A return True 2512307SN/A 2522831Sksewell@umich.educlass GoldenDir(object): 2532831Sksewell@umich.edu def __init__(self, path, platform): 2542831Sksewell@umich.edu self.path = path 2552831Sksewell@umich.edu self.platform = platform 2562831Sksewell@umich.edu 2572831Sksewell@umich.edu contents = os.listdir(path) 2582292SN/A suffix = '.' + platform 2592307SN/A suffixed = filter(lambda c: c.endswith(suffix), contents) 2602292SN/A bases = map(lambda t: t[:-len(platform)], suffixed) 2612292SN/A common = filter(lambda t: not t.startswith(tuple(bases)), contents) 2622316SN/A 2632292SN/A self.entries = {} 2642292SN/A class Entry(object): 2652292SN/A def __init__(self, e_path): 2662292SN/A self.used = False 2672292SN/A self.path = os.path.join(path, e_path) 2682292SN/A 2691060SN/A def use(self): 2702292SN/A self.used = True 2712292SN/A 2721060SN/A for entry in contents: 2732292SN/A self.entries[entry] = Entry(entry) 2742307SN/A 2752292SN/A def entry(self, name): 2762292SN/A def match(n): 2772292SN/A return (n == name) or n.startswith(name + '.') 2782325SN/A matches = { n: e for n, e in self.entries.items() if match(n) } 2792292SN/A 2802292SN/A for match in matches.values(): 2812292SN/A match.use() 2822325SN/A 2832292SN/A platform_name = '.'.join([ name, self.platform ]) 2842292SN/A if platform_name in matches: 2852292SN/A return matches[platform_name].path 2862292SN/A if name in matches: 2872292SN/A return matches[name].path 2882292SN/A else: 2892292SN/A return None 2902292SN/A 2912292SN/A def unused(self): 2922292SN/A items = self.entries.items() 2932292SN/A items = filter(lambda i: not i[1].used, items) 2942325SN/A 2952292SN/A items.sort() 2962292SN/A sources = [] 2972292SN/A i = 0 2982325SN/A while i < len(items): 2992292SN/A root = items[i][0] 3002292SN/A sources.append(root) 3012292SN/A i += 1 3022292SN/A while i < len(items) and items[i][0].startswith(root): 3032292SN/A i += 1 3042292SN/A return sources 3052292SN/A 3062292SN/Aclass VerifyPhase(TestPhaseBase): 3072292SN/A name = 'verify' 3082292SN/A number = 3 3092292SN/A 3102292SN/A def reset_status(self): 3112292SN/A self._passed = [] 3122292SN/A self._failed = {} 3132292SN/A 3142292SN/A def passed(self, test): 3152292SN/A self._passed.append(test) 3161060SN/A 3172292SN/A def failed(self, test, cause, note=''): 3181060SN/A test.set_prop('note', note) 3191060SN/A self._failed.setdefault(cause, []).append(test) 3202292SN/A 3212292SN/A def print_status(self): 3222292SN/A total_passed = len(self._passed) 3232829Sksewell@umich.edu total_failed = sum(map(len, self._failed.values())) 3242829Sksewell@umich.edu print() 3252292SN/A print('Passed: {passed:4} - Failed: {failed:4}'.format( 3261060SN/A passed=total_passed, failed=total_failed)) 3271060SN/A 3281060SN/A def write_result_file(self, path): 3291755SN/A results = { 3301060SN/A 'passed': map(lambda t: t.props, self._passed), 3311060SN/A 'failed': { 3321060SN/A cause: map(lambda t: t.props, tests) for 3331060SN/A cause, tests in self._failed.iteritems() 3341060SN/A } 3351755SN/A } 3361062SN/A with open(path, 'w') as rf: 3372733Sktlim@umich.edu json.dump(results, rf) 3382292SN/A 3392733Sktlim@umich.edu def print_results(self): 3402292SN/A print() 3412292SN/A print('Passed:') 3422292SN/A for path in sorted(list([ t.path for t in self._passed ])): 3432292SN/A print(' ', path) 3442292SN/A 3452292SN/A print() 3462292SN/A print('Failed:') 3472292SN/A 3482292SN/A causes = [] 3492292SN/A for cause, tests in sorted(self._failed.items()): 3502292SN/A block = ' ' + cause.capitalize() + ':\n' 3512292SN/A for test in sorted(tests, key=lambda t: t.path): 3522292SN/A block += ' ' + test.path 3532292SN/A if test.note: 3542292SN/A block += ' - ' + test.note 3552292SN/A block += '\n' 3562292SN/A causes.append(block) 3572292SN/A 3582292SN/A print('\n'.join(causes)) 3592292SN/A 3602292SN/A def run(self, tests): 3612292SN/A parser = argparse.ArgumentParser() 3622292SN/A result_opts = parser.add_mutually_exclusive_group() 3632292SN/A result_opts.add_argument('--result-file', action='store_true', 3642292SN/A help='Create a results.json file in the current directory.') 3652292SN/A result_opts.add_argument('--result-file-at', metavar='PATH', 3662292SN/A help='Create a results json file at the given path.') 3672292SN/A parser.add_argument('--print-results', action='store_true', 3682292SN/A help='Print a list of tests that passed or failed') 3692292SN/A args = parser.parse_args(self.args) 3702292SN/A 3712292SN/A self.reset_status() 3722292SN/A 3732292SN/A runnable = filter(lambda t: not t.compile_only, tests) 3742292SN/A compile_only = filter(lambda t: t.compile_only, tests) 3752292SN/A 3762292SN/A for test in compile_only: 3772292SN/A if os.path.exists(test.full_path()): 3782292SN/A self.passed(test) 3792292SN/A else: 3802292SN/A self.failed(test, 'compile failed') 3812292SN/A 3822292SN/A for test in runnable: 3832292SN/A with open(test.returncode_file()) as rc: 3842292SN/A returncode = int(rc.read()) 3852292SN/A 3862292SN/A if returncode == 124: 3872292SN/A self.failed(test, 'time out') 3882292SN/A continue 3891062SN/A elif returncode != 0: 3901062SN/A self.failed(test, 'abort') 3911062SN/A continue 3922871Sktlim@umich.edu 3932871Sktlim@umich.edu out_dir = test.m5out_dir() 3942871Sktlim@umich.edu 3952871Sktlim@umich.edu Diff = collections.namedtuple( 3962871Sktlim@umich.edu 'Diff', 'ref, test, tag, ref_filter') 3972871Sktlim@umich.edu 3982871Sktlim@umich.edu diffs = [] 3992871Sktlim@umich.edu 4002871Sktlim@umich.edu gd = GoldenDir(test.golden_dir(), 'linux64') 4012871Sktlim@umich.edu 4022871Sktlim@umich.edu missing = [] 4032871Sktlim@umich.edu log_file = '.'.join([test.name, 'log']) 4041062SN/A log_path = gd.entry(log_file) 4051755SN/A simout_path = os.path.join(out_dir, 'simout') 4061060SN/A if not os.path.exists(simout_path): 4072733Sktlim@umich.edu missing.append('log output') 4081060SN/A elif log_path: 4092292SN/A diffs.append(LogChecker(log_path, simout_path, 4102292SN/A log_file, out_dir)) 4112325SN/A 4122292SN/A for name in gd.unused(): 4132292SN/A test_path = os.path.join(out_dir, name) 4141060SN/A ref_path = gd.entry(name) 4151060SN/A if not os.path.exists(test_path): 4161060SN/A missing.append(name) 4171060SN/A else: 4181060SN/A diffs.append(Checker(ref_path, test_path, name)) 4191060SN/A 4201060SN/A if missing: 4211060SN/A self.failed(test, 'missing output', ' '.join(missing)) 4221060SN/A continue 4231060SN/A 4242292SN/A failed_diffs = filter(lambda d: not d.check(), diffs) 4252292SN/A if failed_diffs: 4262292SN/A tags = map(lambda d: d.tag, failed_diffs) 4272292SN/A self.failed(test, 'failed diffs', ' '.join(tags)) 4282292SN/A continue 4291060SN/A 4301060SN/A self.passed(test) 4311060SN/A 4321060SN/A if args.print_results: 4331060SN/A self.print_results() 4341060SN/A 4351060SN/A self.print_status() 4362325SN/A 4372292SN/A result_path = None 4382292SN/A if args.result_file: 4392292SN/A result_path = os.path.join(os.getcwd(), 'results.json') 4402292SN/A elif args.result_file_at: 4412292SN/A result_path = args.result_file_at 4422325SN/A 4432867Sktlim@umich.edu if result_path: 4442905Sktlim@umich.edu self.write_result_file(result_path) 4452325SN/A 4462325SN/A 4472325SN/Aparser = argparse.ArgumentParser(description='SystemC test utility') 4482325SN/A 4492325SN/Aparser.add_argument('build_dir', metavar='BUILD_DIR', 4502325SN/A help='The build directory (ie. build/ARM).') 4512325SN/A 4522325SN/Aparser.add_argument('--update-json', action='store_true', 4532292SN/A help='Update the json manifest of tests.') 4542292SN/A 4552292SN/Aparser.add_argument('--flavor', choices=['debug', 'opt', 'fast'], 4562292SN/A default='opt', 4572292SN/A help='Flavor of binary to test.') 4582292SN/A 4591060SN/Aparser.add_argument('--list', action='store_true', 4601060SN/A help='List the available tests') 4611060SN/A 4621060SN/Afilter_opts = parser.add_mutually_exclusive_group() 4631755SN/Afilter_opts.add_argument('--filter', default='True', 4641060SN/A help='Python expression which filters tests based ' 4652307SN/A 'on their properties') 4662680Sktlim@umich.edufilter_opts.add_argument('--filter-file', default=None, 4672292SN/A type=argparse.FileType('r'), 4681060SN/A help='Same as --filter, but read from a file') 4692292SN/A 4702292SN/Adef collect_phases(args): 4712292SN/A phase_groups = [list(g) for k, g in 4722292SN/A itertools.groupby(args, lambda x: x != '--phase') if k] 4732292SN/A main_args = parser.parse_args(phase_groups[0][1:]) 4742292SN/A phases = [] 4751858SN/A names = [] 4762680Sktlim@umich.edu for group in phase_groups[1:]: 4771681SN/A name = group[0] 4782680Sktlim@umich.edu if name in names: 4791681SN/A raise RuntimeException('Phase %s specified more than once' % name) 4802292SN/A phase = test_phase_classes[name] 4812680Sktlim@umich.edu phases.append(phase(main_args, *group[1:])) 4822292SN/A phases.sort() 4831060SN/A return main_args, phases 4841060SN/A 4852292SN/Amain_args, phases = collect_phases(sys.argv) 4862680Sktlim@umich.edu 4872292SN/Aif len(phases) == 0: 4882292SN/A phases = [ 4892292SN/A CompilePhase(main_args), 4902292SN/A RunPhase(main_args), 4912292SN/A VerifyPhase(main_args) 4922292SN/A ] 4932292SN/A 4942316SN/A 4952292SN/A 4962292SN/Ajson_path = os.path.join(main_args.build_dir, json_rel_path) 4972292SN/A 4982292SN/Aif main_args.update_json: 4992292SN/A scons(os.path.join(json_path)) 5002292SN/A 5012292SN/Awith open(json_path) as f: 5022292SN/A test_data = json.load(f) 5032292SN/A 5042292SN/A if main_args.filter_file: 5052875Sksewell@umich.edu f = main_args.filter_file 5062875Sksewell@umich.edu filt = compile(f.read(), f.name, 'eval') 5072875Sksewell@umich.edu else: 5082875Sksewell@umich.edu filt = compile(main_args.filter, '<string>', 'eval') 5092875Sksewell@umich.edu 5102875Sksewell@umich.edu filtered_tests = { 5112875Sksewell@umich.edu target: props for (target, props) in 5122875Sksewell@umich.edu test_data.iteritems() if eval(filt, dict(props)) 5132875Sksewell@umich.edu } 5142875Sksewell@umich.edu 5152875Sksewell@umich.edu if len(filtered_tests) == 0: 5162875Sksewell@umich.edu print('All tests were filtered out.') 5172875Sksewell@umich.edu exit() 5182875Sksewell@umich.edu 5192875Sksewell@umich.edu if main_args.list: 5202875Sksewell@umich.edu for target, props in sorted(filtered_tests.iteritems()): 5212875Sksewell@umich.edu print('%s.%s' % (target, main_args.flavor)) 5222875Sksewell@umich.edu for key, val in props.iteritems(): 5232875Sksewell@umich.edu print(' %s: %s' % (key, val)) 5242875Sksewell@umich.edu print('Total tests: %d' % len(filtered_tests)) 5252875Sksewell@umich.edu else: 5262875Sksewell@umich.edu tests_to_run = list([ 5272875Sksewell@umich.edu Test(target, main_args.flavor, main_args.build_dir, props) for 5282875Sksewell@umich.edu target, props in sorted(filtered_tests.iteritems()) 5292875Sksewell@umich.edu ]) 5302875Sksewell@umich.edu 5312875Sksewell@umich.edu for phase in phases: 5322875Sksewell@umich.edu phase.run(tests_to_run) 5332875Sksewell@umich.edu