results.py revision 11542
17149Sgblack@eecs.umich.edu#!/usr/bin/env python 27149Sgblack@eecs.umich.edu# 37149Sgblack@eecs.umich.edu# Copyright (c) 2016 ARM Limited 47149Sgblack@eecs.umich.edu# All rights reserved 57149Sgblack@eecs.umich.edu# 67149Sgblack@eecs.umich.edu# The license below extends only to copyright in the software and shall 77149Sgblack@eecs.umich.edu# not be construed as granting a license to any other intellectual 87149Sgblack@eecs.umich.edu# property including but not limited to intellectual property relating 97149Sgblack@eecs.umich.edu# to a hardware implementation of the functionality of the software 107149Sgblack@eecs.umich.edu# licensed hereunder. You may use the software subject to the license 117149Sgblack@eecs.umich.edu# terms below provided that you ensure that this notice is replicated 127149Sgblack@eecs.umich.edu# unmodified and in its entirety in all distributions of the software, 137149Sgblack@eecs.umich.edu# modified or unmodified, in source code or in binary form. 147149Sgblack@eecs.umich.edu# 156253Sgblack@eecs.umich.edu# Redistribution and use in source and binary forms, with or without 166253Sgblack@eecs.umich.edu# modification, are permitted provided that the following conditions are 176253Sgblack@eecs.umich.edu# met: redistributions of source code must retain the above copyright 186253Sgblack@eecs.umich.edu# notice, this list of conditions and the following disclaimer; 196253Sgblack@eecs.umich.edu# redistributions in binary form must reproduce the above copyright 206253Sgblack@eecs.umich.edu# notice, this list of conditions and the following disclaimer in the 216253Sgblack@eecs.umich.edu# documentation and/or other materials provided with the distribution; 226253Sgblack@eecs.umich.edu# neither the name of the copyright holders nor the names of its 236253Sgblack@eecs.umich.edu# contributors may be used to endorse or promote products derived from 246253Sgblack@eecs.umich.edu# this software without specific prior written permission. 256253Sgblack@eecs.umich.edu# 266253Sgblack@eecs.umich.edu# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS 276253Sgblack@eecs.umich.edu# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT 286253Sgblack@eecs.umich.edu# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR 296253Sgblack@eecs.umich.edu# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT 306253Sgblack@eecs.umich.edu# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, 316253Sgblack@eecs.umich.edu# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT 326253Sgblack@eecs.umich.edu# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, 336253Sgblack@eecs.umich.edu# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY 346253Sgblack@eecs.umich.edu# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT 356253Sgblack@eecs.umich.edu# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE 366253Sgblack@eecs.umich.edu# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 376253Sgblack@eecs.umich.edu# 386253Sgblack@eecs.umich.edu# Authors: Andreas Sandberg 396253Sgblack@eecs.umich.edu 406253Sgblack@eecs.umich.edufrom abc import ABCMeta, abstractmethod 416253Sgblack@eecs.umich.eduimport inspect 426253Sgblack@eecs.umich.eduimport pickle 436253Sgblack@eecs.umich.eduimport string 446253Sgblack@eecs.umich.eduimport sys 456253Sgblack@eecs.umich.edu 466253Sgblack@eecs.umich.eduimport xml.etree.cElementTree as ET 476253Sgblack@eecs.umich.edu 486253Sgblack@eecs.umich.educlass UnitResult(object): 497149Sgblack@eecs.umich.edu """Results of a single test unit. 507149Sgblack@eecs.umich.edu 517149Sgblack@eecs.umich.edu A test result can be one of: 527149Sgblack@eecs.umich.edu - STATE_OK: Test ran successfully. 537149Sgblack@eecs.umich.edu - STATE_SKIPPED: The test was skipped. 547149Sgblack@eecs.umich.edu - STATE_ERROR: The test failed to run. 557149Sgblack@eecs.umich.edu - STATE_FAILED: Test ran, but failed. 567149Sgblack@eecs.umich.edu 577149Sgblack@eecs.umich.edu The difference between STATE_ERROR and STATE_FAILED is very 587149Sgblack@eecs.umich.edu subtle. In a gem5 context, STATE_ERROR would mean that gem5 failed 597149Sgblack@eecs.umich.edu to start or crashed, while STATE_FAILED would mean that a test 608146SAli.Saidi@ARM.com failed (e.g., statistics mismatch). 617149Sgblack@eecs.umich.edu 627149Sgblack@eecs.umich.edu """ 637149Sgblack@eecs.umich.edu 647149Sgblack@eecs.umich.edu STATE_OK = 0 657149Sgblack@eecs.umich.edu STATE_SKIPPED = 1 667149Sgblack@eecs.umich.edu STATE_ERROR = 2 677149Sgblack@eecs.umich.edu STATE_FAILURE = 3 687149Sgblack@eecs.umich.edu 698909SAli.Saidi@ARM.com state_names = { 708909SAli.Saidi@ARM.com STATE_OK : "OK", 718909SAli.Saidi@ARM.com STATE_SKIPPED : "SKIPPED", 728909SAli.Saidi@ARM.com STATE_ERROR : "ERROR", 738909SAli.Saidi@ARM.com STATE_FAILURE : "FAILURE", 748909SAli.Saidi@ARM.com } 757149Sgblack@eecs.umich.edu 767149Sgblack@eecs.umich.edu def __init__(self, name, state, message="", stderr="", stdout="", 777149Sgblack@eecs.umich.edu runtime=0.0): 787149Sgblack@eecs.umich.edu self.name = name 797149Sgblack@eecs.umich.edu self.state = state 807149Sgblack@eecs.umich.edu self.message = message 817149Sgblack@eecs.umich.edu self.stdout = stdout 827149Sgblack@eecs.umich.edu self.stderr = stderr 837149Sgblack@eecs.umich.edu self.runtime = runtime 847149Sgblack@eecs.umich.edu 857149Sgblack@eecs.umich.edu def skipped(self): 867149Sgblack@eecs.umich.edu return self.state == UnitResult.STATE_SKIPPED 877149Sgblack@eecs.umich.edu 887149Sgblack@eecs.umich.edu def success(self): 897149Sgblack@eecs.umich.edu return self.state == UnitResult.STATE_OK 907149Sgblack@eecs.umich.edu 917149Sgblack@eecs.umich.edu def state_name(self): 927149Sgblack@eecs.umich.edu return UnitResult.state_names[self.state] 937149Sgblack@eecs.umich.edu 947149Sgblack@eecs.umich.edu def __nonzero__(self): 957149Sgblack@eecs.umich.edu return self.success() or self.skipped() 968909SAli.Saidi@ARM.com 978909SAli.Saidi@ARM.com def __str__(self): 988909SAli.Saidi@ARM.com state_name = self.state_name() 998909SAli.Saidi@ARM.com 1008909SAli.Saidi@ARM.com status = "%s: %s" % (state_name, self.message) if self.message else \ 1018909SAli.Saidi@ARM.com state_name 1027149Sgblack@eecs.umich.edu 1037149Sgblack@eecs.umich.edu return "%s: %s" % (self.name, status) 1047149Sgblack@eecs.umich.edu 1057149Sgblack@eecs.umich.educlass TestResult(object): 1067149Sgblack@eecs.umich.edu """Results for from a single test consisting of one or more units.""" 1077149Sgblack@eecs.umich.edu 1087149Sgblack@eecs.umich.edu def __init__(self, name, run_results=[], verify_results=[]): 1097149Sgblack@eecs.umich.edu self.name = name 1107149Sgblack@eecs.umich.edu self.results = run_results + verify_results 1117149Sgblack@eecs.umich.edu self.run_results = run_results 1127149Sgblack@eecs.umich.edu self.verify_results = verify_results 1137149Sgblack@eecs.umich.edu 1147149Sgblack@eecs.umich.edu def success(self): 1157149Sgblack@eecs.umich.edu return self.success_run() and self.success_verify() 1167149Sgblack@eecs.umich.edu 1177149Sgblack@eecs.umich.edu def success_run(self): 1187149Sgblack@eecs.umich.edu return all([ r.success() for r in self.run_results ]) 1197149Sgblack@eecs.umich.edu 1207149Sgblack@eecs.umich.edu def success_verify(self): 1217149Sgblack@eecs.umich.edu return all([ r.success() for r in self.verify_results ]) 1227149Sgblack@eecs.umich.edu 1237149Sgblack@eecs.umich.edu def failed(self): 1247149Sgblack@eecs.umich.edu return self.failed_run() or self.failed_verify() 1257149Sgblack@eecs.umich.edu 1267149Sgblack@eecs.umich.edu def failed_run(self): 1277149Sgblack@eecs.umich.edu return any([ not r for r in self.run_results ]) 1287149Sgblack@eecs.umich.edu 1297149Sgblack@eecs.umich.edu def failed_verify(self): 1307149Sgblack@eecs.umich.edu return any([ not r for r in self.verify_results ]) 1317149Sgblack@eecs.umich.edu 1326253Sgblack@eecs.umich.edu def skipped(self): 1336253Sgblack@eecs.umich.edu return all([ r.skipped() for r in self.run_results ]) 1346253Sgblack@eecs.umich.edu 135 def changed(self): 136 return self.success_run() and self.failed_verify() 137 138 def runtime(self): 139 return sum([ r.runtime for r in self.results ]) 140 141 def __nonzero__(self): 142 return all([ r for r in self.results ]) 143 144class ResultFormatter(object): 145 __metaclass__ = ABCMeta 146 147 def __init__(self, fout=sys.stdout, verbose=False): 148 self.verbose = verbose 149 self.fout = fout 150 151 @abstractmethod 152 def dump_suites(self, suites): 153 pass 154 155class Pickle(ResultFormatter): 156 """Save test results as a binary using Python's pickle 157 functionality. 158 159 """ 160 161 def __init__(self, **kwargs): 162 super(Pickle, self).__init__(**kwargs) 163 164 def dump_suites(self, suites): 165 pickle.dump(suites, self.fout, pickle.HIGHEST_PROTOCOL) 166 167class Text(ResultFormatter): 168 """Output test results as text.""" 169 170 def __init__(self, **kwargs): 171 super(Text, self).__init__(**kwargs) 172 173 def dump_suites(self, suites): 174 fout = self.fout 175 for suite in suites: 176 print >> fout, "--- %s ---" % suite.name 177 178 for t in suite.results: 179 print >> fout, "*** %s" % t 180 181 if t and not self.verbose: 182 continue 183 184 if t.message: 185 print >> fout, t.message 186 187 if t.stderr: 188 print >> fout, t.stderr 189 if t.stdout: 190 print >> fout, t.stdout 191 192class TextSummary(ResultFormatter): 193 """Output test results as a text summary""" 194 195 def __init__(self, **kwargs): 196 super(TextSummary, self).__init__(**kwargs) 197 198 def test_status(self, suite): 199 if suite.skipped(): 200 return "SKIPPED" 201 elif suite.changed(): 202 return "CHANGED" 203 elif suite: 204 return "OK" 205 else: 206 return "FAILED" 207 208 def dump_suites(self, suites): 209 fout = self.fout 210 for suite in suites: 211 status = self.test_status(suite) 212 print >> fout, "%s: %s" % (suite.name, status) 213 214class JUnit(ResultFormatter): 215 """Output test results as JUnit XML""" 216 217 def __init__(self, translate_names=True, **kwargs): 218 super(JUnit, self).__init__(**kwargs) 219 220 if translate_names: 221 self.name_table = string.maketrans( 222 "/.", 223 ".-", 224 ) 225 else: 226 self.name_table = string.maketrans("", "") 227 228 def convert_unit(self, x_suite, test): 229 x_test = ET.SubElement(x_suite, "testcase", 230 name=test.name, 231 time="%f" % test.runtime) 232 233 x_state = None 234 if test.state == UnitResult.STATE_OK: 235 pass 236 elif test.state == UnitResult.STATE_SKIPPED: 237 x_state = ET.SubElement(x_test, "skipped") 238 elif test.state == UnitResult.STATE_FAILURE: 239 x_state = ET.SubElement(x_test, "failure") 240 elif test.state == UnitResult.STATE_ERROR: 241 x_state = ET.SubElement(x_test, "error") 242 else: 243 assert False, "Unknown test state" 244 245 if x_state is not None: 246 if test.message: 247 x_state.set("message", test.message) 248 249 msg = [] 250 if test.stderr: 251 msg.append("*** Standard Errror: ***") 252 msg.append(test.stderr) 253 if test.stdout: 254 msg.append("*** Standard Out: ***") 255 msg.append(test.stdout) 256 257 x_state.text = "\n".join(msg) 258 259 return x_test 260 261 def convert_suite(self, x_suites, suite): 262 x_suite = ET.SubElement(x_suites, "testsuite", 263 name=suite.name.translate(self.name_table), 264 time="%f" % suite.runtime()) 265 errors = 0 266 failures = 0 267 skipped = 0 268 269 for test in suite.results: 270 if test.state != UnitResult.STATE_OK: 271 if test.state == UnitResult.STATE_SKIPPED: 272 skipped += 1 273 elif test.state == UnitResult.STATE_ERROR: 274 errors += 1 275 elif test.state == UnitResult.STATE_FAILURE: 276 failures += 1 277 278 x_test = self.convert_unit(x_suite, test) 279 280 x_suite.set("errors", str(errors)) 281 x_suite.set("failures", str(failures)) 282 x_suite.set("skipped", str(skipped)) 283 x_suite.set("tests", str(len(suite.results))) 284 285 return x_suite 286 287 def convert_suites(self, suites): 288 x_root = ET.Element("testsuites") 289 290 for suite in suites: 291 self.convert_suite(x_root, suite) 292 293 return x_root 294 295 def dump_suites(self, suites): 296 et = ET.ElementTree(self.convert_suites(suites)) 297 et.write(self.fout, encoding="UTF-8") 298