units.py revision 13540
17639Sgblack@eecs.umich.edu#!/usr/bin/env python2.7 27639Sgblack@eecs.umich.edu# 310037SARM gem5 Developers# Copyright (c) 2016 ARM Limited 47639Sgblack@eecs.umich.edu# All rights reserved 57639Sgblack@eecs.umich.edu# 67639Sgblack@eecs.umich.edu# The license below extends only to copyright in the software and shall 77639Sgblack@eecs.umich.edu# not be construed as granting a license to any other intellectual 87639Sgblack@eecs.umich.edu# property including but not limited to intellectual property relating 97639Sgblack@eecs.umich.edu# to a hardware implementation of the functionality of the software 107639Sgblack@eecs.umich.edu# licensed hereunder. You may use the software subject to the license 117639Sgblack@eecs.umich.edu# terms below provided that you ensure that this notice is replicated 127639Sgblack@eecs.umich.edu# unmodified and in its entirety in all distributions of the software, 137639Sgblack@eecs.umich.edu# modified or unmodified, in source code or in binary form. 147639Sgblack@eecs.umich.edu# 157639Sgblack@eecs.umich.edu# Redistribution and use in source and binary forms, with or without 167639Sgblack@eecs.umich.edu# modification, are permitted provided that the following conditions are 177639Sgblack@eecs.umich.edu# met: redistributions of source code must retain the above copyright 187639Sgblack@eecs.umich.edu# notice, this list of conditions and the following disclaimer; 197639Sgblack@eecs.umich.edu# redistributions in binary form must reproduce the above copyright 207639Sgblack@eecs.umich.edu# notice, this list of conditions and the following disclaimer in the 217639Sgblack@eecs.umich.edu# documentation and/or other materials provided with the distribution; 227639Sgblack@eecs.umich.edu# neither the name of the copyright holders nor the names of its 237639Sgblack@eecs.umich.edu# contributors may be used to endorse or promote products derived from 247639Sgblack@eecs.umich.edu# this software without specific prior written permission. 257639Sgblack@eecs.umich.edu# 267639Sgblack@eecs.umich.edu# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS 277639Sgblack@eecs.umich.edu# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT 287639Sgblack@eecs.umich.edu# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR 297639Sgblack@eecs.umich.edu# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT 307639Sgblack@eecs.umich.edu# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, 317639Sgblack@eecs.umich.edu# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT 327639Sgblack@eecs.umich.edu# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, 337639Sgblack@eecs.umich.edu# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY 347639Sgblack@eecs.umich.edu# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT 357639Sgblack@eecs.umich.edu# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE 367639Sgblack@eecs.umich.edu# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 377639Sgblack@eecs.umich.edu# 387639Sgblack@eecs.umich.edu# Authors: Andreas Sandberg 397639Sgblack@eecs.umich.edu 407639Sgblack@eecs.umich.edufrom abc import ABCMeta, abstractmethod 417639Sgblack@eecs.umich.edufrom datetime import datetime 427639Sgblack@eecs.umich.eduimport difflib 437639Sgblack@eecs.umich.eduimport functools 447639Sgblack@eecs.umich.eduimport os 457639Sgblack@eecs.umich.eduimport re 467639Sgblack@eecs.umich.eduimport subprocess 477639Sgblack@eecs.umich.eduimport sys 487639Sgblack@eecs.umich.eduimport traceback 497639Sgblack@eecs.umich.edu 507639Sgblack@eecs.umich.edufrom results import UnitResult 517639Sgblack@eecs.umich.edufrom helpers import * 527639Sgblack@eecs.umich.edu 537639Sgblack@eecs.umich.edu_test_base = os.path.join(os.path.dirname(__file__), "..") 547639Sgblack@eecs.umich.edu 557639Sgblack@eecs.umich.educlass TestUnit(object): 567639Sgblack@eecs.umich.edu """Base class for all test units. 577639Sgblack@eecs.umich.edu 587639Sgblack@eecs.umich.edu A test unit is a part of a larger test case. Test cases usually 597639Sgblack@eecs.umich.edu contain two types of units, run units (run gem5) and verify units 607639Sgblack@eecs.umich.edu (diff output files). All unit implementations inherit from this 617639Sgblack@eecs.umich.edu class. 627639Sgblack@eecs.umich.edu 637639Sgblack@eecs.umich.edu A unit implementation overrides the _run() method. The test runner 647639Sgblack@eecs.umich.edu calls the run() method, which wraps _run() to protect against 657639Sgblack@eecs.umich.edu exceptions. 667639Sgblack@eecs.umich.edu 677639Sgblack@eecs.umich.edu """ 687639Sgblack@eecs.umich.edu 697639Sgblack@eecs.umich.edu __metaclass__ = ABCMeta 707639Sgblack@eecs.umich.edu 717639Sgblack@eecs.umich.edu def __init__(self, name, ref_dir, test_dir, skip=False): 727639Sgblack@eecs.umich.edu self.name = name 737639Sgblack@eecs.umich.edu self.ref_dir = ref_dir 747639Sgblack@eecs.umich.edu self.test_dir = test_dir 757639Sgblack@eecs.umich.edu self.force_skip = skip 767639Sgblack@eecs.umich.edu self.start_time = None 777639Sgblack@eecs.umich.edu self.stop_time = None 787639Sgblack@eecs.umich.edu 797639Sgblack@eecs.umich.edu def result(self, state, **kwargs): 807639Sgblack@eecs.umich.edu if self.start_time is not None and "runtime" not in kwargs: 817639Sgblack@eecs.umich.edu self.stop_time = datetime.utcnow() 827639Sgblack@eecs.umich.edu delta = self.stop_time - self.start_time 837639Sgblack@eecs.umich.edu kwargs["runtime"] = delta.total_seconds() 847639Sgblack@eecs.umich.edu 857639Sgblack@eecs.umich.edu return UnitResult(self.name, state, **kwargs) 867639Sgblack@eecs.umich.edu 877639Sgblack@eecs.umich.edu def ok(self, **kwargs): 887639Sgblack@eecs.umich.edu return self.result(UnitResult.STATE_OK, **kwargs) 897639Sgblack@eecs.umich.edu 907639Sgblack@eecs.umich.edu def skip(self, **kwargs): 917639Sgblack@eecs.umich.edu return self.result(UnitResult.STATE_SKIPPED, **kwargs) 927639Sgblack@eecs.umich.edu 937639Sgblack@eecs.umich.edu def error(self, message, **kwargs): 947639Sgblack@eecs.umich.edu return self.result(UnitResult.STATE_ERROR, message=message, **kwargs) 957639Sgblack@eecs.umich.edu 967639Sgblack@eecs.umich.edu def failure(self, message, **kwargs): 9710037SARM gem5 Developers return self.result(UnitResult.STATE_FAILURE, message=message, **kwargs) 9810037SARM gem5 Developers 997639Sgblack@eecs.umich.edu def ref_file(self, fname): 1007639Sgblack@eecs.umich.edu return os.path.join(self.ref_dir, fname) 1017639Sgblack@eecs.umich.edu 1027639Sgblack@eecs.umich.edu def out_file(self, fname): 1037639Sgblack@eecs.umich.edu return os.path.join(self.test_dir, fname) 1047639Sgblack@eecs.umich.edu 1057639Sgblack@eecs.umich.edu def _read_output(self, fname, default=""): 1067639Sgblack@eecs.umich.edu try: 1077639Sgblack@eecs.umich.edu with open(self.out_file(fname), "r") as f: 1087639Sgblack@eecs.umich.edu return f.read() 1097639Sgblack@eecs.umich.edu except IOError: 1107639Sgblack@eecs.umich.edu return default 1117639Sgblack@eecs.umich.edu 1127639Sgblack@eecs.umich.edu def run(self): 1137639Sgblack@eecs.umich.edu self.start_time = datetime.utcnow() 1147639Sgblack@eecs.umich.edu try: 11510037SARM gem5 Developers if self.force_skip: 11610037SARM gem5 Developers return self.skip() 1177639Sgblack@eecs.umich.edu else: 1187639Sgblack@eecs.umich.edu return self._run() 1197639Sgblack@eecs.umich.edu except: 1207639Sgblack@eecs.umich.edu return self.error("Python exception:\n%s" % traceback.format_exc()) 1217639Sgblack@eecs.umich.edu 1227639Sgblack@eecs.umich.edu @abstractmethod 1237639Sgblack@eecs.umich.edu def _run(self): 1247639Sgblack@eecs.umich.edu pass 1257639Sgblack@eecs.umich.edu 1267639Sgblack@eecs.umich.educlass RunGem5(TestUnit): 1277639Sgblack@eecs.umich.edu """Test unit representing a gem5 run. 1287639Sgblack@eecs.umich.edu 1297639Sgblack@eecs.umich.edu Possible failure modes: 1307639Sgblack@eecs.umich.edu - gem5 failed to run -> STATE_ERROR 1317639Sgblack@eecs.umich.edu - timeout -> STATE_ERROR 13210037SARM gem5 Developers - non-zero exit code -> STATE_ERROR 13310037SARM gem5 Developers 13410037SARM gem5 Developers Possible non-failure results: 13510037SARM gem5 Developers - exit code == 0 -> STATE_OK 13610037SARM gem5 Developers - exit code == 2 -> STATE_SKIPPED 13710037SARM gem5 Developers """ 13810037SARM gem5 Developers 13910037SARM gem5 Developers def __init__(self, gem5, gem5_args, timeout=0, **kwargs): 14010037SARM gem5 Developers super(RunGem5, self).__init__("gem5", **kwargs) 14110037SARM gem5 Developers self.gem5 = gem5 14210037SARM gem5 Developers self.args = gem5_args 14310037SARM gem5 Developers self.timeout = timeout 14410037SARM gem5 Developers 14510037SARM gem5 Developers def _run(self): 14610037SARM gem5 Developers gem5_cmd = [ 14710037SARM gem5 Developers self.gem5, 14810037SARM gem5 Developers "-d", self.test_dir, 14910037SARM gem5 Developers "--stats-file", "text://stats.txt?desc=False", 15010037SARM gem5 Developers "-re", 15110037SARM gem5 Developers ] + self.args 15210037SARM gem5 Developers 15310037SARM gem5 Developers try: 15410037SARM gem5 Developers with ProcessHelper(gem5_cmd, stdout=subprocess.PIPE, 15510037SARM gem5 Developers stderr=subprocess.PIPE) as p: 15610037SARM gem5 Developers status, gem5_stdout, gem5_stderr = p.call(timeout=self.timeout) 15710037SARM gem5 Developers except CallTimeoutException as te: 15810037SARM gem5 Developers return self.error("Timeout", stdout=te.stdout, stderr=te.stderr) 15910037SARM gem5 Developers except OSError as ose: 16010037SARM gem5 Developers return self.error("Failed to launch gem5: %s" % ose) 16110037SARM gem5 Developers 16210037SARM gem5 Developers stderr = "\n".join([ 16310037SARM gem5 Developers "*** gem5 stderr ***", 1647639Sgblack@eecs.umich.edu gem5_stderr, 1657639Sgblack@eecs.umich.edu "", 1667639Sgblack@eecs.umich.edu "*** m5out/simerr ***", 1677639Sgblack@eecs.umich.edu self._read_output("simerr"), 1687639Sgblack@eecs.umich.edu ]) 1697639Sgblack@eecs.umich.edu 1707639Sgblack@eecs.umich.edu stdout = "\n".join([ 1717639Sgblack@eecs.umich.edu "*** gem5 stdout ***", 1727639Sgblack@eecs.umich.edu gem5_stdout, 1737639Sgblack@eecs.umich.edu "", 1747639Sgblack@eecs.umich.edu "*** m5out/simout ***", 1757639Sgblack@eecs.umich.edu self._read_output("simout"), 1767639Sgblack@eecs.umich.edu ]) 1777639Sgblack@eecs.umich.edu 1787639Sgblack@eecs.umich.edu # Signal 1797639Sgblack@eecs.umich.edu if status < 0: 1807639Sgblack@eecs.umich.edu return self.error("gem5 terminated by signal %i" % (-status, ), 1817639Sgblack@eecs.umich.edu stdout=stdout, stderr=stderr) 1827639Sgblack@eecs.umich.edu elif status == 2: 1837639Sgblack@eecs.umich.edu return self.skip(stdout=stdout, stderr=stderr) 1847639Sgblack@eecs.umich.edu elif status > 0: 1857639Sgblack@eecs.umich.edu return self.error("gem5 exited with non-zero status: %i" % status, 1867639Sgblack@eecs.umich.edu stdout=stdout, stderr=stderr) 1877639Sgblack@eecs.umich.edu else: 1887639Sgblack@eecs.umich.edu return self.ok(stdout=stdout, stderr=stderr) 1897639Sgblack@eecs.umich.edu 1907639Sgblack@eecs.umich.educlass DiffOutFile(TestUnit): 1917639Sgblack@eecs.umich.edu """Test unit comparing and output file and a reference file.""" 1927639Sgblack@eecs.umich.edu 1937639Sgblack@eecs.umich.edu # regular expressions of lines to ignore when diffing outputs 1947639Sgblack@eecs.umich.edu diff_ignore_regexes = { 1957639Sgblack@eecs.umich.edu "simout" : [ 1967639Sgblack@eecs.umich.edu re.compile('^Redirecting (stdout|stderr) to'), 1977639Sgblack@eecs.umich.edu re.compile('^gem5 compiled '), 1987639Sgblack@eecs.umich.edu re.compile('^gem5 started '), 1997639Sgblack@eecs.umich.edu re.compile('^gem5 executing on '), 2007639Sgblack@eecs.umich.edu re.compile('^command line:'), 2017639Sgblack@eecs.umich.edu re.compile("^Couldn't import dot_parser,"), 2027639Sgblack@eecs.umich.edu re.compile("^info: kernel located at:"), 2037639Sgblack@eecs.umich.edu re.compile("^Couldn't unlink "), 2047639Sgblack@eecs.umich.edu re.compile("^Using GPU kernel code file\(s\) "), 2057639Sgblack@eecs.umich.edu ], 2067639Sgblack@eecs.umich.edu "simerr" : [ 2077639Sgblack@eecs.umich.edu #re.compile('^Simulation complete at'), 2087639Sgblack@eecs.umich.edu ], 2097639Sgblack@eecs.umich.edu "config.ini" : [ 2107639Sgblack@eecs.umich.edu re.compile("^(executable|readfile|kernel|image_file)="), 2117639Sgblack@eecs.umich.edu re.compile("^(cwd|input|codefile)="), 21210037SARM gem5 Developers ], 21310037SARM gem5 Developers "config.json" : [ 21410037SARM gem5 Developers re.compile(r'''^\s*"(executable|readfile|kernel|image_file)":'''), 21510037SARM gem5 Developers re.compile(r'''^\s*"(cwd|input|codefile)":'''), 21610037SARM gem5 Developers ], 21710037SARM gem5 Developers } 21810037SARM gem5 Developers 21910037SARM gem5 Developers def __init__(self, fname, **kwargs): 22010037SARM gem5 Developers super(DiffOutFile, self).__init__("diff[%s]" % fname, 22110037SARM gem5 Developers **kwargs) 22210037SARM gem5 Developers 22310037SARM gem5 Developers self.fname = fname 22410037SARM gem5 Developers self.line_filters = DiffOutFile.diff_ignore_regexes.get(fname, tuple()) 22510037SARM gem5 Developers 22610037SARM gem5 Developers def _filter_file(self, fname): 22710037SARM gem5 Developers def match_line(l): 22810037SARM gem5 Developers for r in self.line_filters: 22910037SARM gem5 Developers if r.match(l): 23010037SARM gem5 Developers return True 23110037SARM gem5 Developers return False 23210037SARM gem5 Developers 23310037SARM gem5 Developers with open(fname, "r") as f: 23410037SARM gem5 Developers for l in f: 23510037SARM gem5 Developers if not match_line(l): 23610037SARM gem5 Developers yield l 23710037SARM gem5 Developers 23810037SARM gem5 Developers 23910037SARM gem5 Developers def _run(self): 24010037SARM gem5 Developers fname = self.fname 24110037SARM gem5 Developers ref = self.ref_file(fname) 24210037SARM gem5 Developers out = self.out_file(fname) 24310037SARM gem5 Developers 2447639Sgblack@eecs.umich.edu if not os.path.exists(ref): 2457639Sgblack@eecs.umich.edu return self.error("%s doesn't exist in reference directory" \ 2467639Sgblack@eecs.umich.edu % fname) 2477639Sgblack@eecs.umich.edu 2487639Sgblack@eecs.umich.edu if not os.path.exists(out): 2497639Sgblack@eecs.umich.edu return self.error("%s doesn't exist in output directory" % fname) 2507639Sgblack@eecs.umich.edu 2517639Sgblack@eecs.umich.edu diff = difflib.unified_diff( 2527639Sgblack@eecs.umich.edu tuple(self._filter_file(ref)), 2537639Sgblack@eecs.umich.edu tuple(self._filter_file(out)), 2547639Sgblack@eecs.umich.edu fromfile="ref/%s" % fname, tofile="out/%s" % fname) 2557639Sgblack@eecs.umich.edu 2567639Sgblack@eecs.umich.edu diff = list(diff) 2577639Sgblack@eecs.umich.edu if diff: 2587639Sgblack@eecs.umich.edu return self.error("ref/%s and out/%s differ" % (fname, fname), 2597639Sgblack@eecs.umich.edu stderr="".join(diff)) 2607639Sgblack@eecs.umich.edu else: 2617639Sgblack@eecs.umich.edu return self.ok(stdout="-- ref/%s and out/%s are identical --" \ 2627639Sgblack@eecs.umich.edu % (fname, fname)) 2637639Sgblack@eecs.umich.edu 2647639Sgblack@eecs.umich.educlass DiffStatFile(TestUnit): 2657639Sgblack@eecs.umich.edu """Test unit comparing two gem5 stat files.""" 2667639Sgblack@eecs.umich.edu 2677639Sgblack@eecs.umich.edu def __init__(self, **kwargs): 2687639Sgblack@eecs.umich.edu super(DiffStatFile, self).__init__("stat_diff", **kwargs) 2697639Sgblack@eecs.umich.edu 2707639Sgblack@eecs.umich.edu self.stat_diff = os.path.join(_test_base, "diff-out") 2717639Sgblack@eecs.umich.edu 2727639Sgblack@eecs.umich.edu def _run(self): 2737639Sgblack@eecs.umich.edu STATUS_OK = 0 2747639Sgblack@eecs.umich.edu STATUS_NEW_STATS = 1 2757639Sgblack@eecs.umich.edu STATUS_FAILED = 2 2767639Sgblack@eecs.umich.edu 2777639Sgblack@eecs.umich.edu stats = "stats.txt" 2787639Sgblack@eecs.umich.edu 2797639Sgblack@eecs.umich.edu cmd = [ 2807639Sgblack@eecs.umich.edu self.stat_diff, 2817639Sgblack@eecs.umich.edu self.ref_file(stats), self.out_file(stats), 2827639Sgblack@eecs.umich.edu ] 2837639Sgblack@eecs.umich.edu with ProcessHelper(cmd, 2847639Sgblack@eecs.umich.edu stdout=subprocess.PIPE, 2857639Sgblack@eecs.umich.edu stderr=subprocess.PIPE) as p: 2867639Sgblack@eecs.umich.edu status, stdout, stderr = p.call() 2877639Sgblack@eecs.umich.edu 2887639Sgblack@eecs.umich.edu if status in (STATUS_OK, STATUS_NEW_STATS): 2897639Sgblack@eecs.umich.edu return self.ok(stdout=stdout, stderr=stderr) 2907639Sgblack@eecs.umich.edu elif status == STATUS_FAILED: 2917639Sgblack@eecs.umich.edu return self.failure("Statistics mismatch", 2927639Sgblack@eecs.umich.edu stdout=stdout, stderr=stderr) 2937639Sgblack@eecs.umich.edu else: 2947639Sgblack@eecs.umich.edu return self.error("diff-out returned an error: %i" % status, 2957639Sgblack@eecs.umich.edu stdout=stdout, stderr=stderr) 2967639Sgblack@eecs.umich.edu