verify.py revision 13000:c53f8c81369b
1#!/usr/bin/env python2 2# 3# Copyright 2018 Google, Inc. 4# 5# Redistribution and use in source and binary forms, with or without 6# modification, are permitted provided that the following conditions are 7# met: redistributions of source code must retain the above copyright 8# notice, this list of conditions and the following disclaimer; 9# redistributions in binary form must reproduce the above copyright 10# notice, this list of conditions and the following disclaimer in the 11# documentation and/or other materials provided with the distribution; 12# neither the name of the copyright holders nor the names of its 13# contributors may be used to endorse or promote products derived from 14# this software without specific prior written permission. 15# 16# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS 17# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT 18# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR 19# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT 20# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, 21# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT 22# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, 23# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY 24# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT 25# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE 26# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 27# 28# Authors: Gabe Black 29 30from __future__ import print_function 31 32import argparse 33import functools 34import inspect 35import itertools 36import json 37import logging 38import multiprocessing.pool 39import os 40import subprocess 41import sys 42 43script_path = os.path.abspath(inspect.getfile(inspect.currentframe())) 44script_dir = os.path.dirname(script_path) 45config_path = os.path.join(script_dir, 'config.py') 46 47systemc_rel_path = 'systemc' 48tests_rel_path = os.path.join(systemc_rel_path, 'tests') 49json_rel_path = os.path.join(tests_rel_path, 'tests.json') 50 51 52 53logging.basicConfig(level=logging.INFO) 54 55def scons(*args): 56 args = ['scons'] + list(args) 57 subprocess.check_call(args) 58 59 60 61class Test(object): 62 def __init__(self, target, suffix, build_dir, props): 63 self.target = target 64 self.suffix = suffix 65 self.build_dir = build_dir 66 67 for key, val in props.iteritems(): 68 setattr(self, key, val) 69 70 def dir(self): 71 return os.path.join(self.build_dir, tests_rel_path, self.path) 72 73 def src_dir(self): 74 return os.path.join(script_dir, self.path) 75 76 def golden_dir(self): 77 return os.path.join(self.src_dir(), 'golden') 78 79 def bin(self): 80 return '.'.join([self.name, self.suffix]) 81 82 def full_path(self): 83 return os.path.join(self.dir(), self.bin()) 84 85 def m5out_dir(self): 86 return os.path.join(self.dir(), 'm5out.' + self.suffix) 87 88 89 90test_phase_classes = {} 91 92class TestPhaseMeta(type): 93 def __init__(cls, name, bases, d): 94 if not d.pop('abstract', False): 95 test_phase_classes[d['name']] = cls 96 97 super(TestPhaseMeta, cls).__init__(name, bases, d) 98 99class TestPhaseBase(object): 100 __metaclass__ = TestPhaseMeta 101 abstract = True 102 103 def __init__(self, main_args, *args): 104 self.main_args = main_args 105 self.args = args 106 107 def __lt__(self, other): 108 return self.number < other.number 109 110class CompilePhase(TestPhaseBase): 111 name = 'compile' 112 number = 1 113 114 def run(self, tests): 115 targets = list([test.full_path() for test in tests]) 116 scons_args = list(self.args) + targets 117 scons(*scons_args) 118 119class RunPhase(TestPhaseBase): 120 name = 'execute' 121 number = 2 122 123 def run(self, tests): 124 parser = argparse.ArgumentParser() 125 parser.add_argument('--timeout', type=int, metavar='SECONDS', 126 help='Time limit for each run in seconds.', 127 default=0) 128 parser.add_argument('-j', type=int, default=1, 129 help='How many tests to run in parallel.') 130 args = parser.parse_args(self.args) 131 132 timeout_cmd = [ 133 'timeout', 134 '--kill-after', str(args.timeout * 2), 135 str(args.timeout) 136 ] 137 def run_test(test): 138 cmd = [] 139 if args.timeout: 140 cmd.extend(timeout_cmd) 141 cmd.extend([ 142 test.full_path(), 143 '-red', test.m5out_dir(), 144 '--listener-mode=off', 145 config_path 146 ]) 147 subprocess.check_call(cmd) 148 149 runnable = filter(lambda t: not t.compile_only, tests) 150 if args.j == 1: 151 map(run_test, runnable) 152 else: 153 tp = multiprocessing.pool.ThreadPool(args.j) 154 map(lambda t: tp.apply_async(run_test, (t,)), runnable) 155 tp.close() 156 tp.join() 157 158class VerifyPhase(TestPhaseBase): 159 name = 'verify' 160 number = 3 161 162 def run(self, tests): 163 for test in tests: 164 if test.compile_only: 165 continue 166 logging.info("Would verify %s", test.m5out_dir()) 167 168 169 170parser = argparse.ArgumentParser(description='SystemC test utility') 171 172parser.add_argument('build_dir', metavar='BUILD_DIR', 173 help='The build directory (ie. build/ARM).') 174 175parser.add_argument('--update-json', action='store_true', 176 help='Update the json manifest of tests.') 177 178parser.add_argument('--flavor', choices=['debug', 'opt', 'fast'], 179 default='opt', 180 help='Flavor of binary to test.') 181 182parser.add_argument('--list', action='store_true', 183 help='List the available tests') 184 185filter_opts = parser.add_mutually_exclusive_group() 186filter_opts.add_argument('--filter', default='True', 187 help='Python expression which filters tests based ' 188 'on their properties') 189filter_opts.add_argument('--filter-file', default=None, 190 type=argparse.FileType('r'), 191 help='Same as --filter, but read from a file') 192 193def collect_phases(args): 194 phase_groups = [list(g) for k, g in 195 itertools.groupby(args, lambda x: x != '--phase') if k] 196 main_args = parser.parse_args(phase_groups[0][1:]) 197 phases = [] 198 names = [] 199 for group in phase_groups[1:]: 200 name = group[0] 201 if name in names: 202 raise RuntimeException('Phase %s specified more than once' % name) 203 phase = test_phase_classes[name] 204 phases.append(phase(main_args, *group[1:])) 205 phases.sort() 206 return main_args, phases 207 208main_args, phases = collect_phases(sys.argv) 209 210if len(phases) == 0: 211 phases = [ 212 CompilePhase(main_args), 213 RunPhase(main_args), 214 VerifyPhase(main_args) 215 ] 216 217 218 219json_path = os.path.join(main_args.build_dir, json_rel_path) 220 221if main_args.update_json: 222 scons(os.path.join(json_path)) 223 224with open(json_path) as f: 225 test_data = json.load(f) 226 227 if main_args.filter_file: 228 f = main_args.filter_file 229 filt = compile(f.read(), f.name, 'eval') 230 else: 231 filt = compile(main_args.filter, '<string>', 'eval') 232 233 filtered_tests = { 234 target: props for (target, props) in 235 test_data.iteritems() if eval(filt, dict(props)) 236 } 237 238 if main_args.list: 239 for target, props in sorted(filtered_tests.iteritems()): 240 print('%s.%s' % (target, main_args.flavor)) 241 for key, val in props.iteritems(): 242 print(' %s: %s' % (key, val)) 243 print('Total tests: %d' % len(filtered_tests)) 244 else: 245 tests_to_run = list([ 246 Test(target, main_args.flavor, main_args.build_dir, props) for 247 target, props in sorted(filtered_tests.iteritems()) 248 ]) 249 250 for phase in phases: 251 phase.run(tests_to_run) 252