verify.py revision 13001:acf4fd41ba76
1#!/usr/bin/env python2 2# 3# Copyright 2018 Google, Inc. 4# 5# Redistribution and use in source and binary forms, with or without 6# modification, are permitted provided that the following conditions are 7# met: redistributions of source code must retain the above copyright 8# notice, this list of conditions and the following disclaimer; 9# redistributions in binary form must reproduce the above copyright 10# notice, this list of conditions and the following disclaimer in the 11# documentation and/or other materials provided with the distribution; 12# neither the name of the copyright holders nor the names of its 13# contributors may be used to endorse or promote products derived from 14# this software without specific prior written permission. 15# 16# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS 17# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT 18# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR 19# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT 20# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, 21# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT 22# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, 23# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY 24# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT 25# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE 26# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 27# 28# Authors: Gabe Black 29 30from __future__ import print_function 31 32import argparse 33import functools 34import inspect 35import itertools 36import json 37import logging 38import multiprocessing.pool 39import os 40import subprocess 41import sys 42 43script_path = os.path.abspath(inspect.getfile(inspect.currentframe())) 44script_dir = os.path.dirname(script_path) 45config_path = os.path.join(script_dir, 'config.py') 46 47systemc_rel_path = 'systemc' 48tests_rel_path = os.path.join(systemc_rel_path, 'tests') 49json_rel_path = os.path.join(tests_rel_path, 'tests.json') 50 51 52 53logging.basicConfig(level=logging.INFO) 54 55def scons(*args): 56 args = ['scons'] + list(args) 57 subprocess.check_call(args) 58 59 60 61class Test(object): 62 def __init__(self, target, suffix, build_dir, props): 63 self.target = target 64 self.suffix = suffix 65 self.build_dir = build_dir 66 67 for key, val in props.iteritems(): 68 setattr(self, key, val) 69 70 def dir(self): 71 return os.path.join(self.build_dir, tests_rel_path, self.path) 72 73 def src_dir(self): 74 return os.path.join(script_dir, self.path) 75 76 def golden_dir(self): 77 return os.path.join(self.src_dir(), 'golden') 78 79 def bin(self): 80 return '.'.join([self.name, self.suffix]) 81 82 def full_path(self): 83 return os.path.join(self.dir(), self.bin()) 84 85 def m5out_dir(self): 86 return os.path.join(self.dir(), 'm5out.' + self.suffix) 87 88 89 90test_phase_classes = {} 91 92class TestPhaseMeta(type): 93 def __init__(cls, name, bases, d): 94 if not d.pop('abstract', False): 95 test_phase_classes[d['name']] = cls 96 97 super(TestPhaseMeta, cls).__init__(name, bases, d) 98 99class TestPhaseBase(object): 100 __metaclass__ = TestPhaseMeta 101 abstract = True 102 103 def __init__(self, main_args, *args): 104 self.main_args = main_args 105 self.args = args 106 107 def __lt__(self, other): 108 return self.number < other.number 109 110class CompilePhase(TestPhaseBase): 111 name = 'compile' 112 number = 1 113 114 def run(self, tests): 115 targets = list([test.full_path() for test in tests]) 116 scons_args = list(self.args) + targets 117 scons(*scons_args) 118 119class RunPhase(TestPhaseBase): 120 name = 'execute' 121 number = 2 122 123 def run(self, tests): 124 parser = argparse.ArgumentParser() 125 parser.add_argument('--timeout', type=int, metavar='SECONDS', 126 help='Time limit for each run in seconds.', 127 default=0) 128 parser.add_argument('-j', type=int, default=1, 129 help='How many tests to run in parallel.') 130 args = parser.parse_args(self.args) 131 132 timeout_cmd = [ 133 'timeout', 134 '--kill-after', str(args.timeout * 2), 135 str(args.timeout) 136 ] 137 def run_test(test): 138 cmd = [] 139 if args.timeout: 140 cmd.extend(timeout_cmd) 141 cmd.extend([ 142 test.full_path(), 143 '-red', test.m5out_dir(), 144 '--listener-mode=off', 145 config_path 146 ]) 147 try: 148 subprocess.check_call(cmd) 149 except subprocess.CalledProcessError, error: 150 returncode = error.returncode 151 else: 152 returncode = 0 153 with open(os.path.join(test.m5out_dir(), 'returncode'), 'w') as rc: 154 rc.write('%d\n' % returncode) 155 156 runnable = filter(lambda t: not t.compile_only, tests) 157 if args.j == 1: 158 map(run_test, runnable) 159 else: 160 tp = multiprocessing.pool.ThreadPool(args.j) 161 map(lambda t: tp.apply_async(run_test, (t,)), runnable) 162 tp.close() 163 tp.join() 164 165class VerifyPhase(TestPhaseBase): 166 name = 'verify' 167 number = 3 168 169 def run(self, tests): 170 for test in tests: 171 if test.compile_only: 172 continue 173 logging.info("Would verify %s", test.m5out_dir()) 174 175 176 177parser = argparse.ArgumentParser(description='SystemC test utility') 178 179parser.add_argument('build_dir', metavar='BUILD_DIR', 180 help='The build directory (ie. build/ARM).') 181 182parser.add_argument('--update-json', action='store_true', 183 help='Update the json manifest of tests.') 184 185parser.add_argument('--flavor', choices=['debug', 'opt', 'fast'], 186 default='opt', 187 help='Flavor of binary to test.') 188 189parser.add_argument('--list', action='store_true', 190 help='List the available tests') 191 192filter_opts = parser.add_mutually_exclusive_group() 193filter_opts.add_argument('--filter', default='True', 194 help='Python expression which filters tests based ' 195 'on their properties') 196filter_opts.add_argument('--filter-file', default=None, 197 type=argparse.FileType('r'), 198 help='Same as --filter, but read from a file') 199 200def collect_phases(args): 201 phase_groups = [list(g) for k, g in 202 itertools.groupby(args, lambda x: x != '--phase') if k] 203 main_args = parser.parse_args(phase_groups[0][1:]) 204 phases = [] 205 names = [] 206 for group in phase_groups[1:]: 207 name = group[0] 208 if name in names: 209 raise RuntimeException('Phase %s specified more than once' % name) 210 phase = test_phase_classes[name] 211 phases.append(phase(main_args, *group[1:])) 212 phases.sort() 213 return main_args, phases 214 215main_args, phases = collect_phases(sys.argv) 216 217if len(phases) == 0: 218 phases = [ 219 CompilePhase(main_args), 220 RunPhase(main_args), 221 VerifyPhase(main_args) 222 ] 223 224 225 226json_path = os.path.join(main_args.build_dir, json_rel_path) 227 228if main_args.update_json: 229 scons(os.path.join(json_path)) 230 231with open(json_path) as f: 232 test_data = json.load(f) 233 234 if main_args.filter_file: 235 f = main_args.filter_file 236 filt = compile(f.read(), f.name, 'eval') 237 else: 238 filt = compile(main_args.filter, '<string>', 'eval') 239 240 filtered_tests = { 241 target: props for (target, props) in 242 test_data.iteritems() if eval(filt, dict(props)) 243 } 244 245 if main_args.list: 246 for target, props in sorted(filtered_tests.iteritems()): 247 print('%s.%s' % (target, main_args.flavor)) 248 for key, val in props.iteritems(): 249 print(' %s: %s' % (key, val)) 250 print('Total tests: %d' % len(filtered_tests)) 251 else: 252 tests_to_run = list([ 253 Test(target, main_args.flavor, main_args.build_dir, props) for 254 target, props in sorted(filtered_tests.iteritems()) 255 ]) 256 257 for phase in phases: 258 phase.run(tests_to_run) 259