verify.py (13002:b8d58d5f25a5) verify.py (13003:3a164f2f8103)
1#!/usr/bin/env python2
2#
3# Copyright 2018 Google, Inc.
4#
5# Redistribution and use in source and binary forms, with or without
6# modification, are permitted provided that the following conditions are
7# met: redistributions of source code must retain the above copyright
8# notice, this list of conditions and the following disclaimer;

--- 16 unchanged lines hidden (view full) ---

25# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
26# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
27#
28# Authors: Gabe Black
29
30from __future__ import print_function
31
32import argparse
1#!/usr/bin/env python2
2#
3# Copyright 2018 Google, Inc.
4#
5# Redistribution and use in source and binary forms, with or without
6# modification, are permitted provided that the following conditions are
7# met: redistributions of source code must retain the above copyright
8# notice, this list of conditions and the following disclaimer;

--- 16 unchanged lines hidden (view full) ---

25# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
26# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
27#
28# Authors: Gabe Black
29
30from __future__ import print_function
31
32import argparse
33import collections
34import difflib
33import functools
34import inspect
35import itertools
36import json
37import multiprocessing.pool
38import os
35import functools
36import inspect
37import itertools
38import json
39import multiprocessing.pool
40import os
41import re
39import subprocess
40import sys
41
42script_path = os.path.abspath(inspect.getfile(inspect.currentframe()))
43script_dir = os.path.dirname(script_path)
44config_path = os.path.join(script_dir, 'config.py')
45
46systemc_rel_path = 'systemc'

--- 8 unchanged lines hidden (view full) ---

55
56
57
58class Test(object):
59 def __init__(self, target, suffix, build_dir, props):
60 self.target = target
61 self.suffix = suffix
62 self.build_dir = build_dir
42import subprocess
43import sys
44
45script_path = os.path.abspath(inspect.getfile(inspect.currentframe()))
46script_dir = os.path.dirname(script_path)
47config_path = os.path.join(script_dir, 'config.py')
48
49systemc_rel_path = 'systemc'

--- 8 unchanged lines hidden (view full) ---

58
59
60
61class Test(object):
62 def __init__(self, target, suffix, build_dir, props):
63 self.target = target
64 self.suffix = suffix
65 self.build_dir = build_dir
66 self.props = {}
63
64 for key, val in props.iteritems():
67
68 for key, val in props.iteritems():
65 setattr(self, key, val)
69 self.set_prop(key, val)
66
70
71 def set_prop(self, key, val):
72 setattr(self, key, val)
73 self.props[key] = val
74
67 def dir(self):
68 return os.path.join(self.build_dir, tests_rel_path, self.path)
69
70 def src_dir(self):
71 return os.path.join(script_dir, self.path)
72
73 def golden_dir(self):
74 return os.path.join(self.src_dir(), 'golden')

--- 62 unchanged lines hidden (view full) ---

137 def run_test(test):
138 cmd = []
139 if args.timeout:
140 cmd.extend(timeout_cmd)
141 cmd.extend([
142 test.full_path(),
143 '-red', test.m5out_dir(),
144 '--listener-mode=off',
75 def dir(self):
76 return os.path.join(self.build_dir, tests_rel_path, self.path)
77
78 def src_dir(self):
79 return os.path.join(script_dir, self.path)
80
81 def golden_dir(self):
82 return os.path.join(self.src_dir(), 'golden')

--- 62 unchanged lines hidden (view full) ---

145 def run_test(test):
146 cmd = []
147 if args.timeout:
148 cmd.extend(timeout_cmd)
149 cmd.extend([
150 test.full_path(),
151 '-red', test.m5out_dir(),
152 '--listener-mode=off',
153 '--quiet',
145 config_path
146 ])
147 # Ensure the output directory exists.
148 if not os.path.exists(test.m5out_dir()):
149 os.makedirs(test.m5out_dir())
150 try:
151 subprocess.check_call(cmd)
152 except subprocess.CalledProcessError, error:

--- 7 unchanged lines hidden (view full) ---

160 if args.j == 1:
161 map(run_test, runnable)
162 else:
163 tp = multiprocessing.pool.ThreadPool(args.j)
164 map(lambda t: tp.apply_async(run_test, (t,)), runnable)
165 tp.close()
166 tp.join()
167
154 config_path
155 ])
156 # Ensure the output directory exists.
157 if not os.path.exists(test.m5out_dir()):
158 os.makedirs(test.m5out_dir())
159 try:
160 subprocess.check_call(cmd)
161 except subprocess.CalledProcessError, error:

--- 7 unchanged lines hidden (view full) ---

169 if args.j == 1:
170 map(run_test, runnable)
171 else:
172 tp = multiprocessing.pool.ThreadPool(args.j)
173 map(lambda t: tp.apply_async(run_test, (t,)), runnable)
174 tp.close()
175 tp.join()
176
177class Checker(object):
178 def __init__(self, ref, test, tag):
179 self.ref = ref
180 self.test = test
181 self.tag = tag
182
183 def check(self):
184 with open(self.text) as test_f, open(self.ref) as ref_f:
185 return test_f.read() == ref_f.read()
186
187class LogChecker(Checker):
188 def merge_filts(*filts):
189 filts = map(lambda f: '(' + f + ')', filts)
190 filts = '|'.join(filts)
191 return re.compile(filts, flags=re.MULTILINE)
192
193 ref_filt = merge_filts(
194 r'^\nInfo: /OSCI/SystemC: Simulation stopped by user.\n',
195 r'^SystemC Simulation\n'
196 )
197 test_filt = merge_filts(
198 r'^Global frequency set at \d* ticks per second\n'
199 )
200
201 def __init__(self, ref, test, tag, out_dir):
202 super(LogChecker, self).__init__(ref, test, tag)
203 self.out_dir = out_dir
204
205 def apply_filters(self, data, filts):
206 re.sub(filt, '', data)
207
208 def check(self):
209 test_file = os.path.basename(self.test)
210 ref_file = os.path.basename(self.ref)
211 with open(self.test) as test_f, open(self.ref) as ref_f:
212 test = re.sub(self.test_filt, '', test_f.read())
213 ref = re.sub(self.ref_filt, '', ref_f.read())
214 if test != ref:
215 diff_file = '.'.join([ref_file, 'diff'])
216 diff_path = os.path.join(self.out_dir, diff_file)
217 with open(diff_path, 'w') as diff_f:
218 for line in difflib.unified_diff(
219 ref.splitlines(True), test.splitlines(True),
220 fromfile=ref_file,
221 tofile=test_file):
222 diff_f.write(line)
223 return False
224 return True
225
168class VerifyPhase(TestPhaseBase):
169 name = 'verify'
170 number = 3
171
172 def reset_status(self):
173 self._passed = []
174 self._failed = {}
175
176 def passed(self, test):
177 self._passed.append(test)
178
226class VerifyPhase(TestPhaseBase):
227 name = 'verify'
228 number = 3
229
230 def reset_status(self):
231 self._passed = []
232 self._failed = {}
233
234 def passed(self, test):
235 self._passed.append(test)
236
179 def failed(self, test, cause):
237 def failed(self, test, cause, note=''):
238 test.set_prop('note', note)
180 self._failed.setdefault(cause, []).append(test)
181
182 def print_status(self):
183 total_passed = len(self._passed)
184 total_failed = sum(map(len, self._failed.values()))
185 print()
186 print('Passed: {passed:4} - Failed: {failed:4}'.format(
187 passed=total_passed, failed=total_failed))
188
189 def write_result_file(self, path):
239 self._failed.setdefault(cause, []).append(test)
240
241 def print_status(self):
242 total_passed = len(self._passed)
243 total_failed = sum(map(len, self._failed.values()))
244 print()
245 print('Passed: {passed:4} - Failed: {failed:4}'.format(
246 passed=total_passed, failed=total_failed))
247
248 def write_result_file(self, path):
190 passed = map(lambda t: t.path, self._passed)
191 passed.sort()
192 failed = {
193 cause: map(lambda t: t.path, tests) for
249 results = {
250 'passed': map(lambda t: t.props, self._passed),
251 'failed': {
252 cause: map(lambda t: t.props, tests) for
194 cause, tests in self._failed.iteritems()
253 cause, tests in self._failed.iteritems()
254 }
195 }
255 }
196 for tests in failed.values():
197 tests.sort()
198 results = { 'passed': passed, 'failed': failed }
199 with open(path, 'w') as rf:
200 json.dump(results, rf)
201
202 def print_results(self):
256 with open(path, 'w') as rf:
257 json.dump(results, rf)
258
259 def print_results(self):
203 passed = map(lambda t: t.path, self._passed)
204 passed.sort()
205 failed = {
206 cause: map(lambda t: t.path, tests) for
207 cause, tests in self._failed.iteritems()
208 }
209 for tests in failed.values():
210 tests.sort()
211
212 print()
213 print('Passed:')
260 print()
261 print('Passed:')
214 map(lambda t: print(' ', t), passed)
262 for path in sorted(list([ t.path for t in self._passed ])):
263 print(' ', path)
215
216 print()
217 print('Failed:')
264
265 print()
266 print('Failed:')
218 categories = failed.items()
219 categories.sort()
220
267
221 def cat_str((cause, tests)):
222 heading = ' ' + cause.capitalize() + ':\n'
223 test_lines = [' ' + test + '\n'for test in tests]
224 return heading + ''.join(test_lines)
225 blocks = map(cat_str, categories)
268 causes = []
269 for cause, tests in sorted(self._failed.items()):
270 block = ' ' + cause.capitalize() + ':\n'
271 for test in sorted(tests, key=lambda t: t.path):
272 block += ' ' + test.path
273 if test.note:
274 block += ' - ' + test.note
275 block += '\n'
276 causes.append(block)
226
277
227 print('\n'.join(blocks))
278 print('\n'.join(causes))
228
229 def run(self, tests):
230 parser = argparse.ArgumentParser()
231 result_opts = parser.add_mutually_exclusive_group()
232 result_opts.add_argument('--result-file', action='store_true',
233 help='Create a results.json file in the current directory.')
234 result_opts.add_argument('--result-file-at', metavar='PATH',
235 help='Create a results json file at the given path.')

--- 11 unchanged lines hidden (view full) ---

247 self.passed(test)
248 else:
249 self.failed(test, 'compile failed')
250
251 for test in runnable:
252 with open(test.returncode_file()) as rc:
253 returncode = int(rc.read())
254
279
280 def run(self, tests):
281 parser = argparse.ArgumentParser()
282 result_opts = parser.add_mutually_exclusive_group()
283 result_opts.add_argument('--result-file', action='store_true',
284 help='Create a results.json file in the current directory.')
285 result_opts.add_argument('--result-file-at', metavar='PATH',
286 help='Create a results json file at the given path.')

--- 11 unchanged lines hidden (view full) ---

298 self.passed(test)
299 else:
300 self.failed(test, 'compile failed')
301
302 for test in runnable:
303 with open(test.returncode_file()) as rc:
304 returncode = int(rc.read())
305
255 if returncode == 0:
256 self.passed(test)
257 elif returncode == 124:
306 if returncode == 124:
258 self.failed(test, 'time out')
307 self.failed(test, 'time out')
259 else:
308 continue
309 elif returncode != 0:
260 self.failed(test, 'abort')
310 self.failed(test, 'abort')
311 continue
261
312
313 out_dir = test.m5out_dir()
314
315 Diff = collections.namedtuple(
316 'Diff', 'ref, test, tag, ref_filter')
317
318 diffs = []
319
320 log_file = '.'.join([test.name, 'log'])
321 log_path = os.path.join(test.golden_dir(), log_file)
322 simout_path = os.path.join(out_dir, 'simout')
323 if not os.path.exists(simout_path):
324 self.failed(test, 'no log output')
325 if os.path.exists(log_path):
326 diffs.append(LogChecker(
327 log_path, simout_path, log_file, out_dir))
328
329 failed_diffs = filter(lambda d: not d.check(), diffs)
330 if failed_diffs:
331 tags = map(lambda d: d.tag, failed_diffs)
332 self.failed(test, 'failed diffs', ' '.join(tags))
333 continue
334
335 self.passed(test)
336
262 if args.print_results:
263 self.print_results()
264
265 self.print_status()
266
267 result_path = None
268 if args.result_file:
269 result_path = os.path.join(os.getcwd(), 'results.json')

--- 89 unchanged lines hidden --- 		337 if args.print_results:
338 self.print_results()
339
340 self.print_status()
341
342 result_path = None
343 if args.result_file:
344 result_path = os.path.join(os.getcwd(), 'results.json')

--- 89 unchanged lines hidden ---