units.py revision 11482
112751Sqtt2@cornell.edu#!/usr/bin/env python 212751Sqtt2@cornell.edu# 312751Sqtt2@cornell.edu# Copyright (c) 2016 ARM Limited 412751Sqtt2@cornell.edu# All rights reserved 512751Sqtt2@cornell.edu# 612751Sqtt2@cornell.edu# The license below extends only to copyright in the software and shall 712751Sqtt2@cornell.edu# not be construed as granting a license to any other intellectual 812751Sqtt2@cornell.edu# property including but not limited to intellectual property relating 912751Sqtt2@cornell.edu# to a hardware implementation of the functionality of the software 1012751Sqtt2@cornell.edu# licensed hereunder. You may use the software subject to the license 1112751Sqtt2@cornell.edu# terms below provided that you ensure that this notice is replicated 1212751Sqtt2@cornell.edu# unmodified and in its entirety in all distributions of the software, 1312751Sqtt2@cornell.edu# modified or unmodified, in source code or in binary form. 1412751Sqtt2@cornell.edu# 1512751Sqtt2@cornell.edu# Redistribution and use in source and binary forms, with or without 1612751Sqtt2@cornell.edu# modification, are permitted provided that the following conditions are 1712751Sqtt2@cornell.edu# met: redistributions of source code must retain the above copyright 1812751Sqtt2@cornell.edu# notice, this list of conditions and the following disclaimer; 1912751Sqtt2@cornell.edu# redistributions in binary form must reproduce the above copyright 2012751Sqtt2@cornell.edu# notice, this list of conditions and the following disclaimer in the 2112751Sqtt2@cornell.edu# documentation and/or other materials provided with the distribution; 2212751Sqtt2@cornell.edu# neither the name of the copyright holders nor the names of its 2312751Sqtt2@cornell.edu# contributors may be used to endorse or promote products derived from 2412751Sqtt2@cornell.edu# this software without specific prior written permission. 2512751Sqtt2@cornell.edu# 2612751Sqtt2@cornell.edu# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS 2712751Sqtt2@cornell.edu# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT 2812751Sqtt2@cornell.edu# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR 2912751Sqtt2@cornell.edu# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT 3012751Sqtt2@cornell.edu# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, 3112751Sqtt2@cornell.edu# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT 3212751Sqtt2@cornell.edu# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, 3312751Sqtt2@cornell.edu# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY 3412751Sqtt2@cornell.edu# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT 3512751Sqtt2@cornell.edu# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE 3612751Sqtt2@cornell.edu# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 3712751Sqtt2@cornell.edu# 3812751Sqtt2@cornell.edu# Authors: Andreas Sandberg 3912751Sqtt2@cornell.edu 4012751Sqtt2@cornell.edufrom abc import ABCMeta, abstractmethod 4112751Sqtt2@cornell.edufrom datetime import datetime 4212751Sqtt2@cornell.eduimport difflib 4312751Sqtt2@cornell.eduimport functools 4412751Sqtt2@cornell.eduimport os 4512751Sqtt2@cornell.eduimport re 4612751Sqtt2@cornell.eduimport subprocess 4712751Sqtt2@cornell.eduimport sys 4812751Sqtt2@cornell.eduimport traceback 4912751Sqtt2@cornell.edu 5012751Sqtt2@cornell.edufrom results import UnitResult 5112751Sqtt2@cornell.edufrom helpers import * 5212751Sqtt2@cornell.edu 5312751Sqtt2@cornell.edu_test_base = os.path.join(os.path.dirname(__file__), "..") 5412751Sqtt2@cornell.edu 5512751Sqtt2@cornell.educlass TestUnit(object): 5612751Sqtt2@cornell.edu """Base class for all test units. 5712751Sqtt2@cornell.edu 5812751Sqtt2@cornell.edu A test unit is a part of a larger test case. Test cases usually 5912751Sqtt2@cornell.edu contain two types of units, run units (run gem5) and verify units 6012751Sqtt2@cornell.edu (diff output files). All unit implementations inherit from this 6112751Sqtt2@cornell.edu class. 6212751Sqtt2@cornell.edu 6312751Sqtt2@cornell.edu A unit implementation overrides the _run() method. The test runner 6412751Sqtt2@cornell.edu calls the run() method, which wraps _run() to protect against 6512751Sqtt2@cornell.edu exceptions. 6612751Sqtt2@cornell.edu 6712751Sqtt2@cornell.edu """ 6812751Sqtt2@cornell.edu 6912751Sqtt2@cornell.edu __metaclass__ = ABCMeta 7012751Sqtt2@cornell.edu 7112751Sqtt2@cornell.edu def __init__(self, name, ref_dir, test_dir, skip=False): 7212751Sqtt2@cornell.edu self.name = name 7312751Sqtt2@cornell.edu self.ref_dir = ref_dir 7412751Sqtt2@cornell.edu self.test_dir = test_dir 7512751Sqtt2@cornell.edu self.force_skip = skip 7612751Sqtt2@cornell.edu self.start_time = None 7712751Sqtt2@cornell.edu self.stop_time = None 7812751Sqtt2@cornell.edu 7912751Sqtt2@cornell.edu def result(self, state, **kwargs): 8012751Sqtt2@cornell.edu if self.start_time is not None and "runtime" not in kwargs: 8112751Sqtt2@cornell.edu self.stop_time = datetime.utcnow() 8212751Sqtt2@cornell.edu delta = self.stop_time - self.start_time 8312751Sqtt2@cornell.edu kwargs["runtime"] = delta.total_seconds() 8412751Sqtt2@cornell.edu 8512751Sqtt2@cornell.edu return UnitResult(self.name, state, **kwargs) 8612751Sqtt2@cornell.edu 8712751Sqtt2@cornell.edu def ok(self, **kwargs): 8812751Sqtt2@cornell.edu return self.result(UnitResult.STATE_OK, **kwargs) 8912751Sqtt2@cornell.edu 9012751Sqtt2@cornell.edu def skip(self, **kwargs): 9112751Sqtt2@cornell.edu return self.result(UnitResult.STATE_SKIPPED, **kwargs) 9212751Sqtt2@cornell.edu 93 def error(self, message, **kwargs): 94 return self.result(UnitResult.STATE_ERROR, message=message, **kwargs) 95 96 def failure(self, message, **kwargs): 97 return self.result(UnitResult.STATE_FAILURE, message=message, **kwargs) 98 99 def ref_file(self, fname): 100 return os.path.join(self.ref_dir, fname) 101 102 def out_file(self, fname): 103 return os.path.join(self.test_dir, fname) 104 105 def _read_output(self, fname, default=""): 106 try: 107 with open(self.out_file(fname), "r") as f: 108 return f.read() 109 except IOError: 110 return default 111 112 def run(self): 113 self.start_time = datetime.utcnow() 114 try: 115 if self.force_skip: 116 return self.skip() 117 else: 118 return self._run() 119 except: 120 return self.error("Python exception:\n%s" % traceback.format_exc()) 121 122 @abstractmethod 123 def _run(self): 124 pass 125 126class RunGem5(TestUnit): 127 """Test unit representing a gem5 run. 128 129 Possible failure modes: 130 - gem5 failed to run -> STATE_ERROR 131 - timeout -> STATE_ERROR 132 - non-zero exit code -> STATE_ERROR 133 134 Possible non-failure results: 135 - exit code == 0 -> STATE_OK 136 - exit code == 2 -> STATE_SKIPPED 137 """ 138 139 def __init__(self, gem5, gem5_args, timeout=0, **kwargs): 140 super(RunGem5, self).__init__("gem5", **kwargs) 141 self.gem5 = gem5 142 self.args = gem5_args 143 self.timeout = timeout 144 145 def _run(self): 146 gem5_cmd = [ 147 self.gem5, 148 "-d", self.test_dir, 149 "-re", 150 ] + self.args 151 152 try: 153 with ProcessHelper(gem5_cmd, stdout=subprocess.PIPE, 154 stderr=subprocess.PIPE) as p: 155 status, gem5_stdout, gem5_stderr = p.call(timeout=self.timeout) 156 except CallTimeoutException as te: 157 return self.error("Timeout", stdout=te.stdout, stderr=te.stderr) 158 except OSError as ose: 159 return self.error("Failed to launch gem5: %s" % ose) 160 161 stderr = "\n".join([ 162 "*** gem5 stderr ***", 163 gem5_stderr, 164 "", 165 "*** m5out/simerr ***", 166 self._read_output("simerr"), 167 ]) 168 169 stdout = "\n".join([ 170 "*** gem5 stdout ***", 171 gem5_stdout, 172 "", 173 "*** m5out/simout ***", 174 self._read_output("simout"), 175 ]) 176 177 # Signal 178 if status < 0: 179 return self.error("gem5 terminated by signal %i" % (-status, ), 180 stdout=stdout, stderr=stderr) 181 elif status == 2: 182 return self.skip(stdout=stdout, stderr=stderr) 183 elif status > 0: 184 return self.error("gem5 exited with non-zero status: %i" % status, 185 stdout=stdout, stderr=stderr) 186 else: 187 return self.ok(stdout=stdout, stderr=stderr) 188 189class DiffOutFile(TestUnit): 190 """Test unit comparing and output file and a reference file.""" 191 192 # regular expressions of lines to ignore when diffing outputs 193 diff_ignore_regexes = { 194 "simout" : [ 195 re.compile('^Redirecting (stdout|stderr) to'), 196 re.compile('^gem5 compiled '), 197 re.compile('^gem5 started '), 198 re.compile('^gem5 executing on '), 199 re.compile('^command line:'), 200 re.compile("^Couldn't import dot_parser,"), 201 re.compile("^info: kernel located at:"), 202 re.compile("^Couldn't unlink "), 203 re.compile("^Using GPU kernel code file\(s\) "), 204 ], 205 "simerr" : [ 206 #re.compile('^Simulation complete at'), 207 ], 208 "config.ini" : [ 209 re.compile("^(executable|readfile|kernel|image_file)="), 210 re.compile("^(cwd|input|codefile)="), 211 ], 212 "config.json" : [ 213 re.compile(r'''^\s*"(executable|readfile|kernel|image_file)":'''), 214 re.compile(r'''^\s*"(cwd|input|codefile)":'''), 215 ], 216 } 217 218 def __init__(self, fname, **kwargs): 219 super(DiffOutFile, self).__init__("diff[%s]" % fname, 220 **kwargs) 221 222 self.fname = fname 223 self.line_filters = DiffOutFile.diff_ignore_regexes.get(fname, tuple()) 224 225 def _filter_file(self, fname): 226 def match_line(l): 227 for r in self.line_filters: 228 if r.match(l): 229 return True 230 return False 231 232 with open(fname, "r") as f: 233 for l in f: 234 if not match_line(l): 235 yield l 236 237 238 def _run(self): 239 fname = self.fname 240 ref = self.ref_file(fname) 241 out = self.out_file(fname) 242 243 if not os.path.exists(ref): 244 return self.error("%s doesn't exist in reference directory" \ 245 % fname) 246 247 if not os.path.exists(out): 248 return self.error("%s doesn't exist in output directory" % fname) 249 250 diff = difflib.unified_diff( 251 tuple(self._filter_file(ref)), 252 tuple(self._filter_file(out)), 253 fromfile="ref/%s" % fname, tofile="out/%s" % fname) 254 255 diff = list(diff) 256 if diff: 257 return self.error("ref/%s and out/%s differ" % (fname, fname), 258 stderr="".join(diff)) 259 else: 260 return self.ok(stdout="-- ref/%s and out/%s are identical --" \ 261 % (fname, fname)) 262 263class DiffStatFile(TestUnit): 264 """Test unit comparing two gem5 stat files.""" 265 266 def __init__(self, **kwargs): 267 super(DiffStatFile, self).__init__("stat_diff", **kwargs) 268 269 self.stat_diff = os.path.join(_test_base, "diff-out") 270 271 def _run(self): 272 stats = "stats.txt" 273 274 cmd = [ 275 self.stat_diff, 276 self.ref_file(stats), self.out_file(stats), 277 ] 278 with ProcessHelper(cmd, 279 stdout=subprocess.PIPE, 280 stderr=subprocess.PIPE) as p: 281 status, stdout, stderr = p.call() 282 283 if status == 0: 284 return self.ok(stdout=stdout, stderr=stderr) 285 if status == 1: 286 return self.failure("Statistics mismatch", 287 stdout=stdout, stderr=stderr) 288 else: 289 return self.error("diff-out returned an error: %i" % status, 290 stdout=stdout, stderr=stderr) 291