1#!/usr/bin/env python2 2# 3# Copyright (c) 2016 ARM Limited 4# All rights reserved 5# 6# The license below extends only to copyright in the software and shall 7# not be construed as granting a license to any other intellectual 8# property including but not limited to intellectual property relating 9# to a hardware implementation of the functionality of the software 10# licensed hereunder. You may use the software subject to the license 11# terms below provided that you ensure that this notice is replicated 12# unmodified and in its entirety in all distributions of the software, 13# modified or unmodified, in source code or in binary form. 14# 15# Redistribution and use in source and binary forms, with or without 16# modification, are permitted provided that the following conditions are 17# met: redistributions of source code must retain the above copyright 18# notice, this list of conditions and the following disclaimer; 19# redistributions in binary form must reproduce the above copyright 20# notice, this list of conditions and the following disclaimer in the 21# documentation and/or other materials provided with the distribution; 22# neither the name of the copyright holders nor the names of its 23# contributors may be used to endorse or promote products derived from 24# this software without specific prior written permission. 25# 26# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS 27# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT 28# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR 29# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT 30# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, 31# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT 32# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, 33# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY 34# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT 35# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE 36# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 37# 38# Authors: Andreas Sandberg 39
|
40from __future__ import print_function 41 |
42from abc import ABCMeta, abstractmethod 43import inspect 44import pickle 45import string 46import sys 47 48import xml.etree.cElementTree as ET 49 50class UnitResult(object): 51 """Results of a single test unit. 52 53 A test result can be one of: 54 - STATE_OK: Test ran successfully. 55 - STATE_SKIPPED: The test was skipped. 56 - STATE_ERROR: The test failed to run. 57 - STATE_FAILED: Test ran, but failed. 58 59 The difference between STATE_ERROR and STATE_FAILED is very 60 subtle. In a gem5 context, STATE_ERROR would mean that gem5 failed 61 to start or crashed, while STATE_FAILED would mean that a test 62 failed (e.g., statistics mismatch). 63 64 """ 65 66 STATE_OK = 0 67 STATE_SKIPPED = 1 68 STATE_ERROR = 2 69 STATE_FAILURE = 3 70 71 state_names = { 72 STATE_OK : "OK", 73 STATE_SKIPPED : "SKIPPED", 74 STATE_ERROR : "ERROR", 75 STATE_FAILURE : "FAILURE", 76 } 77 78 def __init__(self, name, state, message="", stderr="", stdout="", 79 runtime=0.0): 80 self.name = name 81 self.state = state 82 self.message = message 83 self.stdout = stdout 84 self.stderr = stderr 85 self.runtime = runtime 86 87 def skipped(self): 88 return self.state == UnitResult.STATE_SKIPPED 89 90 def success(self): 91 return self.state == UnitResult.STATE_OK 92 93 def state_name(self): 94 return UnitResult.state_names[self.state] 95 96 def __nonzero__(self): 97 return self.success() or self.skipped() 98 99 def __str__(self): 100 state_name = self.state_name() 101 102 status = "%s: %s" % (state_name, self.message) if self.message else \ 103 state_name 104 105 return "%s: %s" % (self.name, status) 106 107class TestResult(object): 108 """Results for from a single test consisting of one or more units.""" 109 110 def __init__(self, name, run_results=[], verify_results=[]): 111 self.name = name 112 self.results = run_results + verify_results 113 self.run_results = run_results 114 self.verify_results = verify_results 115 116 def success(self): 117 return self.success_run() and self.success_verify() 118 119 def success_run(self): 120 return all([ r.success() for r in self.run_results ]) 121 122 def success_verify(self): 123 return all([ r.success() for r in self.verify_results ]) 124 125 def failed(self): 126 return self.failed_run() or self.failed_verify() 127 128 def failed_run(self): 129 return any([ not r for r in self.run_results ]) 130 131 def failed_verify(self): 132 return any([ not r for r in self.verify_results ]) 133 134 def skipped(self): 135 return all([ r.skipped() for r in self.run_results ]) 136 137 def changed(self): 138 return self.success_run() and self.failed_verify() 139 140 def runtime(self): 141 return sum([ r.runtime for r in self.results ]) 142 143 def __nonzero__(self): 144 return all([ r for r in self.results ]) 145 146class ResultFormatter(object): 147 __metaclass__ = ABCMeta 148 149 def __init__(self, fout=sys.stdout, verbose=False): 150 self.verbose = verbose 151 self.fout = fout 152 153 @abstractmethod 154 def dump_suites(self, suites): 155 pass 156 157class Pickle(ResultFormatter): 158 """Save test results as a binary using Python's pickle 159 functionality. 160 161 """ 162 163 def __init__(self, **kwargs): 164 super(Pickle, self).__init__(**kwargs) 165 166 def dump_suites(self, suites): 167 pickle.dump(suites, self.fout, pickle.HIGHEST_PROTOCOL) 168 169class Text(ResultFormatter): 170 """Output test results as text.""" 171 172 def __init__(self, **kwargs): 173 super(Text, self).__init__(**kwargs) 174 175 def dump_suites(self, suites): 176 fout = self.fout 177 for suite in suites:
|
176 print >> fout, "--- %s ---" % suite.name
|
178 print("--- %s ---" % suite.name, file=fout) |
179 180 for t in suite.results:
|
179 print >> fout, "*** %s" % t
|
181 print("*** %s" % t, file=fout) |
182 183 if t and not self.verbose: 184 continue 185 186 if t.message:
|
185 print >> fout, t.message
|
187 print(t.message, file=fout) |
188 189 if t.stderr:
|
188 print >> fout, t.stderr
|
190 print(t.stderr, file=fout) |
191 if t.stdout:
|
190 print >> fout, t.stdout
|
192 print(t.stdout, file=fout) |
193 194class TextSummary(ResultFormatter): 195 """Output test results as a text summary""" 196 197 def __init__(self, **kwargs): 198 super(TextSummary, self).__init__(**kwargs) 199 200 def test_status(self, suite): 201 if suite.skipped(): 202 return "SKIPPED" 203 elif suite.changed(): 204 return "CHANGED" 205 elif suite: 206 return "OK" 207 else: 208 return "FAILED" 209 210 def dump_suites(self, suites): 211 fout = self.fout 212 for suite in suites: 213 status = self.test_status(suite)
|
212 print >> fout, "%s: %s" % (suite.name, status)
|
214 print("%s: %s" % (suite.name, status), file=fout) |
215 216class JUnit(ResultFormatter): 217 """Output test results as JUnit XML""" 218 219 def __init__(self, translate_names=True, **kwargs): 220 super(JUnit, self).__init__(**kwargs) 221 222 if translate_names: 223 self.name_table = string.maketrans( 224 "/.", 225 ".-", 226 ) 227 else: 228 self.name_table = string.maketrans("", "") 229 230 def convert_unit(self, x_suite, test): 231 x_test = ET.SubElement(x_suite, "testcase", 232 name=test.name, 233 time="%f" % test.runtime) 234 235 x_state = None 236 if test.state == UnitResult.STATE_OK: 237 pass 238 elif test.state == UnitResult.STATE_SKIPPED: 239 x_state = ET.SubElement(x_test, "skipped") 240 elif test.state == UnitResult.STATE_FAILURE: 241 x_state = ET.SubElement(x_test, "failure") 242 elif test.state == UnitResult.STATE_ERROR: 243 x_state = ET.SubElement(x_test, "error") 244 else: 245 assert False, "Unknown test state" 246 247 if x_state is not None: 248 if test.message: 249 x_state.set("message", test.message) 250 251 msg = [] 252 if test.stderr: 253 msg.append("*** Standard Errror: ***") 254 msg.append(test.stderr) 255 if test.stdout: 256 msg.append("*** Standard Out: ***") 257 msg.append(test.stdout) 258 259 x_state.text = "\n".join(msg) 260 261 return x_test 262 263 def convert_suite(self, x_suites, suite): 264 x_suite = ET.SubElement(x_suites, "testsuite", 265 name=suite.name.translate(self.name_table), 266 time="%f" % suite.runtime()) 267 errors = 0 268 failures = 0 269 skipped = 0 270 271 for test in suite.results: 272 if test.state != UnitResult.STATE_OK: 273 if test.state == UnitResult.STATE_SKIPPED: 274 skipped += 1 275 elif test.state == UnitResult.STATE_ERROR: 276 errors += 1 277 elif test.state == UnitResult.STATE_FAILURE: 278 failures += 1 279 280 x_test = self.convert_unit(x_suite, test) 281 282 x_suite.set("errors", str(errors)) 283 x_suite.set("failures", str(failures)) 284 x_suite.set("skipped", str(skipped)) 285 x_suite.set("tests", str(len(suite.results))) 286 287 return x_suite 288 289 def convert_suites(self, suites): 290 x_root = ET.Element("testsuites") 291 292 for suite in suites: 293 self.convert_suite(x_root, suite) 294 295 return x_root 296 297 def dump_suites(self, suites): 298 et = ET.ElementTree(self.convert_suites(suites)) 299 et.write(self.fout, encoding="UTF-8")
|