results.py revision 11542
111482Sandreas.sandberg@arm.com#!/usr/bin/env python 211482Sandreas.sandberg@arm.com# 311482Sandreas.sandberg@arm.com# Copyright (c) 2016 ARM Limited 411482Sandreas.sandberg@arm.com# All rights reserved 511482Sandreas.sandberg@arm.com# 611482Sandreas.sandberg@arm.com# The license below extends only to copyright in the software and shall 711482Sandreas.sandberg@arm.com# not be construed as granting a license to any other intellectual 811482Sandreas.sandberg@arm.com# property including but not limited to intellectual property relating 911482Sandreas.sandberg@arm.com# to a hardware implementation of the functionality of the software 1011482Sandreas.sandberg@arm.com# licensed hereunder. You may use the software subject to the license 1111482Sandreas.sandberg@arm.com# terms below provided that you ensure that this notice is replicated 1211482Sandreas.sandberg@arm.com# unmodified and in its entirety in all distributions of the software, 1311482Sandreas.sandberg@arm.com# modified or unmodified, in source code or in binary form. 1411482Sandreas.sandberg@arm.com# 1511482Sandreas.sandberg@arm.com# Redistribution and use in source and binary forms, with or without 1611482Sandreas.sandberg@arm.com# modification, are permitted provided that the following conditions are 1711482Sandreas.sandberg@arm.com# met: redistributions of source code must retain the above copyright 1811482Sandreas.sandberg@arm.com# notice, this list of conditions and the following disclaimer; 1911482Sandreas.sandberg@arm.com# redistributions in binary form must reproduce the above copyright 2011482Sandreas.sandberg@arm.com# notice, this list of conditions and the following disclaimer in the 2111482Sandreas.sandberg@arm.com# documentation and/or other materials provided with the distribution; 2211482Sandreas.sandberg@arm.com# neither the name of the copyright holders nor the names of its 2311482Sandreas.sandberg@arm.com# contributors may be used to endorse or promote products derived from 2411482Sandreas.sandberg@arm.com# this software without specific prior written permission. 2511482Sandreas.sandberg@arm.com# 2611482Sandreas.sandberg@arm.com# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS 2711482Sandreas.sandberg@arm.com# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT 2811482Sandreas.sandberg@arm.com# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR 2911482Sandreas.sandberg@arm.com# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT 3011482Sandreas.sandberg@arm.com# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, 3111482Sandreas.sandberg@arm.com# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT 3211482Sandreas.sandberg@arm.com# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, 3311482Sandreas.sandberg@arm.com# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY 3411482Sandreas.sandberg@arm.com# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT 3511482Sandreas.sandberg@arm.com# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE 3611482Sandreas.sandberg@arm.com# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 3711482Sandreas.sandberg@arm.com# 3811482Sandreas.sandberg@arm.com# Authors: Andreas Sandberg 3911482Sandreas.sandberg@arm.com 4011482Sandreas.sandberg@arm.comfrom abc import ABCMeta, abstractmethod 4111482Sandreas.sandberg@arm.comimport inspect 4211482Sandreas.sandberg@arm.comimport pickle 4311482Sandreas.sandberg@arm.comimport string 4411482Sandreas.sandberg@arm.comimport sys 4511482Sandreas.sandberg@arm.com 4611482Sandreas.sandberg@arm.comimport xml.etree.cElementTree as ET 4711482Sandreas.sandberg@arm.com 4811482Sandreas.sandberg@arm.comclass UnitResult(object): 4911482Sandreas.sandberg@arm.com """Results of a single test unit. 5011482Sandreas.sandberg@arm.com 5111482Sandreas.sandberg@arm.com A test result can be one of: 5211482Sandreas.sandberg@arm.com - STATE_OK: Test ran successfully. 5311482Sandreas.sandberg@arm.com - STATE_SKIPPED: The test was skipped. 5411482Sandreas.sandberg@arm.com - STATE_ERROR: The test failed to run. 5511482Sandreas.sandberg@arm.com - STATE_FAILED: Test ran, but failed. 5611482Sandreas.sandberg@arm.com 5711482Sandreas.sandberg@arm.com The difference between STATE_ERROR and STATE_FAILED is very 5811482Sandreas.sandberg@arm.com subtle. In a gem5 context, STATE_ERROR would mean that gem5 failed 5911482Sandreas.sandberg@arm.com to start or crashed, while STATE_FAILED would mean that a test 6011482Sandreas.sandberg@arm.com failed (e.g., statistics mismatch). 6111482Sandreas.sandberg@arm.com 6211482Sandreas.sandberg@arm.com """ 6311482Sandreas.sandberg@arm.com 6411482Sandreas.sandberg@arm.com STATE_OK = 0 6511482Sandreas.sandberg@arm.com STATE_SKIPPED = 1 6611482Sandreas.sandberg@arm.com STATE_ERROR = 2 6711482Sandreas.sandberg@arm.com STATE_FAILURE = 3 6811482Sandreas.sandberg@arm.com 6911482Sandreas.sandberg@arm.com state_names = { 7011482Sandreas.sandberg@arm.com STATE_OK : "OK", 7111482Sandreas.sandberg@arm.com STATE_SKIPPED : "SKIPPED", 7211482Sandreas.sandberg@arm.com STATE_ERROR : "ERROR", 7311482Sandreas.sandberg@arm.com STATE_FAILURE : "FAILURE", 7411482Sandreas.sandberg@arm.com } 7511482Sandreas.sandberg@arm.com 7611482Sandreas.sandberg@arm.com def __init__(self, name, state, message="", stderr="", stdout="", 7711482Sandreas.sandberg@arm.com runtime=0.0): 7811482Sandreas.sandberg@arm.com self.name = name 7911482Sandreas.sandberg@arm.com self.state = state 8011482Sandreas.sandberg@arm.com self.message = message 8111482Sandreas.sandberg@arm.com self.stdout = stdout 8211482Sandreas.sandberg@arm.com self.stderr = stderr 8311482Sandreas.sandberg@arm.com self.runtime = runtime 8411482Sandreas.sandberg@arm.com 8511482Sandreas.sandberg@arm.com def skipped(self): 8611482Sandreas.sandberg@arm.com return self.state == UnitResult.STATE_SKIPPED 8711482Sandreas.sandberg@arm.com 8811482Sandreas.sandberg@arm.com def success(self): 8911482Sandreas.sandberg@arm.com return self.state == UnitResult.STATE_OK 9011482Sandreas.sandberg@arm.com 9111482Sandreas.sandberg@arm.com def state_name(self): 9211482Sandreas.sandberg@arm.com return UnitResult.state_names[self.state] 9311482Sandreas.sandberg@arm.com 9411482Sandreas.sandberg@arm.com def __nonzero__(self): 9511482Sandreas.sandberg@arm.com return self.success() or self.skipped() 9611482Sandreas.sandberg@arm.com 9711482Sandreas.sandberg@arm.com def __str__(self): 9811482Sandreas.sandberg@arm.com state_name = self.state_name() 9911482Sandreas.sandberg@arm.com 10011482Sandreas.sandberg@arm.com status = "%s: %s" % (state_name, self.message) if self.message else \ 10111482Sandreas.sandberg@arm.com state_name 10211482Sandreas.sandberg@arm.com 10311482Sandreas.sandberg@arm.com return "%s: %s" % (self.name, status) 10411482Sandreas.sandberg@arm.com 10511482Sandreas.sandberg@arm.comclass TestResult(object): 10611482Sandreas.sandberg@arm.com """Results for from a single test consisting of one or more units.""" 10711482Sandreas.sandberg@arm.com 10811542Sandreas.sandberg@arm.com def __init__(self, name, run_results=[], verify_results=[]): 10911482Sandreas.sandberg@arm.com self.name = name 11011542Sandreas.sandberg@arm.com self.results = run_results + verify_results 11111542Sandreas.sandberg@arm.com self.run_results = run_results 11211542Sandreas.sandberg@arm.com self.verify_results = verify_results 11311482Sandreas.sandberg@arm.com 11411482Sandreas.sandberg@arm.com def success(self): 11511542Sandreas.sandberg@arm.com return self.success_run() and self.success_verify() 11611542Sandreas.sandberg@arm.com 11711542Sandreas.sandberg@arm.com def success_run(self): 11811542Sandreas.sandberg@arm.com return all([ r.success() for r in self.run_results ]) 11911542Sandreas.sandberg@arm.com 12011542Sandreas.sandberg@arm.com def success_verify(self): 12111542Sandreas.sandberg@arm.com return all([ r.success() for r in self.verify_results ]) 12211542Sandreas.sandberg@arm.com 12311542Sandreas.sandberg@arm.com def failed(self): 12411542Sandreas.sandberg@arm.com return self.failed_run() or self.failed_verify() 12511542Sandreas.sandberg@arm.com 12611542Sandreas.sandberg@arm.com def failed_run(self): 12711542Sandreas.sandberg@arm.com return any([ not r for r in self.run_results ]) 12811542Sandreas.sandberg@arm.com 12911542Sandreas.sandberg@arm.com def failed_verify(self): 13011542Sandreas.sandberg@arm.com return any([ not r for r in self.verify_results ]) 13111482Sandreas.sandberg@arm.com 13211482Sandreas.sandberg@arm.com def skipped(self): 13311542Sandreas.sandberg@arm.com return all([ r.skipped() for r in self.run_results ]) 13411482Sandreas.sandberg@arm.com 13511512SCurtis.Dunham@arm.com def changed(self): 13611542Sandreas.sandberg@arm.com return self.success_run() and self.failed_verify() 13711482Sandreas.sandberg@arm.com 13811482Sandreas.sandberg@arm.com def runtime(self): 13911482Sandreas.sandberg@arm.com return sum([ r.runtime for r in self.results ]) 14011482Sandreas.sandberg@arm.com 14111482Sandreas.sandberg@arm.com def __nonzero__(self): 14211542Sandreas.sandberg@arm.com return all([ r for r in self.results ]) 14311482Sandreas.sandberg@arm.com 14411482Sandreas.sandberg@arm.comclass ResultFormatter(object): 14511482Sandreas.sandberg@arm.com __metaclass__ = ABCMeta 14611482Sandreas.sandberg@arm.com 14711482Sandreas.sandberg@arm.com def __init__(self, fout=sys.stdout, verbose=False): 14811482Sandreas.sandberg@arm.com self.verbose = verbose 14911482Sandreas.sandberg@arm.com self.fout = fout 15011482Sandreas.sandberg@arm.com 15111482Sandreas.sandberg@arm.com @abstractmethod 15211482Sandreas.sandberg@arm.com def dump_suites(self, suites): 15311482Sandreas.sandberg@arm.com pass 15411482Sandreas.sandberg@arm.com 15511482Sandreas.sandberg@arm.comclass Pickle(ResultFormatter): 15611482Sandreas.sandberg@arm.com """Save test results as a binary using Python's pickle 15711482Sandreas.sandberg@arm.com functionality. 15811482Sandreas.sandberg@arm.com 15911482Sandreas.sandberg@arm.com """ 16011482Sandreas.sandberg@arm.com 16111482Sandreas.sandberg@arm.com def __init__(self, **kwargs): 16211482Sandreas.sandberg@arm.com super(Pickle, self).__init__(**kwargs) 16311482Sandreas.sandberg@arm.com 16411482Sandreas.sandberg@arm.com def dump_suites(self, suites): 16511482Sandreas.sandberg@arm.com pickle.dump(suites, self.fout, pickle.HIGHEST_PROTOCOL) 16611482Sandreas.sandberg@arm.com 16711482Sandreas.sandberg@arm.comclass Text(ResultFormatter): 16811482Sandreas.sandberg@arm.com """Output test results as text.""" 16911482Sandreas.sandberg@arm.com 17011482Sandreas.sandberg@arm.com def __init__(self, **kwargs): 17111482Sandreas.sandberg@arm.com super(Text, self).__init__(**kwargs) 17211482Sandreas.sandberg@arm.com 17311482Sandreas.sandberg@arm.com def dump_suites(self, suites): 17411482Sandreas.sandberg@arm.com fout = self.fout 17511482Sandreas.sandberg@arm.com for suite in suites: 17611482Sandreas.sandberg@arm.com print >> fout, "--- %s ---" % suite.name 17711482Sandreas.sandberg@arm.com 17811482Sandreas.sandberg@arm.com for t in suite.results: 17911482Sandreas.sandberg@arm.com print >> fout, "*** %s" % t 18011482Sandreas.sandberg@arm.com 18111482Sandreas.sandberg@arm.com if t and not self.verbose: 18211482Sandreas.sandberg@arm.com continue 18311482Sandreas.sandberg@arm.com 18411482Sandreas.sandberg@arm.com if t.message: 18511482Sandreas.sandberg@arm.com print >> fout, t.message 18611482Sandreas.sandberg@arm.com 18711482Sandreas.sandberg@arm.com if t.stderr: 18811482Sandreas.sandberg@arm.com print >> fout, t.stderr 18911482Sandreas.sandberg@arm.com if t.stdout: 19011482Sandreas.sandberg@arm.com print >> fout, t.stdout 19111482Sandreas.sandberg@arm.com 19211482Sandreas.sandberg@arm.comclass TextSummary(ResultFormatter): 19311482Sandreas.sandberg@arm.com """Output test results as a text summary""" 19411482Sandreas.sandberg@arm.com 19511482Sandreas.sandberg@arm.com def __init__(self, **kwargs): 19611482Sandreas.sandberg@arm.com super(TextSummary, self).__init__(**kwargs) 19711482Sandreas.sandberg@arm.com 19811512SCurtis.Dunham@arm.com def test_status(self, suite): 19911512SCurtis.Dunham@arm.com if suite.skipped(): 20011512SCurtis.Dunham@arm.com return "SKIPPED" 20111512SCurtis.Dunham@arm.com elif suite.changed(): 20211512SCurtis.Dunham@arm.com return "CHANGED" 20311512SCurtis.Dunham@arm.com elif suite: 20411512SCurtis.Dunham@arm.com return "OK" 20511512SCurtis.Dunham@arm.com else: 20611512SCurtis.Dunham@arm.com return "FAILED" 20711512SCurtis.Dunham@arm.com 20811482Sandreas.sandberg@arm.com def dump_suites(self, suites): 20911482Sandreas.sandberg@arm.com fout = self.fout 21011482Sandreas.sandberg@arm.com for suite in suites: 21111512SCurtis.Dunham@arm.com status = self.test_status(suite) 21211482Sandreas.sandberg@arm.com print >> fout, "%s: %s" % (suite.name, status) 21311482Sandreas.sandberg@arm.com 21411482Sandreas.sandberg@arm.comclass JUnit(ResultFormatter): 21511482Sandreas.sandberg@arm.com """Output test results as JUnit XML""" 21611482Sandreas.sandberg@arm.com 21711482Sandreas.sandberg@arm.com def __init__(self, translate_names=True, **kwargs): 21811482Sandreas.sandberg@arm.com super(JUnit, self).__init__(**kwargs) 21911482Sandreas.sandberg@arm.com 22011482Sandreas.sandberg@arm.com if translate_names: 22111482Sandreas.sandberg@arm.com self.name_table = string.maketrans( 22211482Sandreas.sandberg@arm.com "/.", 22311482Sandreas.sandberg@arm.com ".-", 22411482Sandreas.sandberg@arm.com ) 22511482Sandreas.sandberg@arm.com else: 22611482Sandreas.sandberg@arm.com self.name_table = string.maketrans("", "") 22711482Sandreas.sandberg@arm.com 22811482Sandreas.sandberg@arm.com def convert_unit(self, x_suite, test): 22911482Sandreas.sandberg@arm.com x_test = ET.SubElement(x_suite, "testcase", 23011482Sandreas.sandberg@arm.com name=test.name, 23111482Sandreas.sandberg@arm.com time="%f" % test.runtime) 23211482Sandreas.sandberg@arm.com 23311482Sandreas.sandberg@arm.com x_state = None 23411482Sandreas.sandberg@arm.com if test.state == UnitResult.STATE_OK: 23511482Sandreas.sandberg@arm.com pass 23611482Sandreas.sandberg@arm.com elif test.state == UnitResult.STATE_SKIPPED: 23711482Sandreas.sandberg@arm.com x_state = ET.SubElement(x_test, "skipped") 23811482Sandreas.sandberg@arm.com elif test.state == UnitResult.STATE_FAILURE: 23911482Sandreas.sandberg@arm.com x_state = ET.SubElement(x_test, "failure") 24011482Sandreas.sandberg@arm.com elif test.state == UnitResult.STATE_ERROR: 24111482Sandreas.sandberg@arm.com x_state = ET.SubElement(x_test, "error") 24211482Sandreas.sandberg@arm.com else: 24311482Sandreas.sandberg@arm.com assert False, "Unknown test state" 24411482Sandreas.sandberg@arm.com 24511482Sandreas.sandberg@arm.com if x_state is not None: 24611482Sandreas.sandberg@arm.com if test.message: 24711482Sandreas.sandberg@arm.com x_state.set("message", test.message) 24811482Sandreas.sandberg@arm.com 24911482Sandreas.sandberg@arm.com msg = [] 25011482Sandreas.sandberg@arm.com if test.stderr: 25111482Sandreas.sandberg@arm.com msg.append("*** Standard Errror: ***") 25211482Sandreas.sandberg@arm.com msg.append(test.stderr) 25311482Sandreas.sandberg@arm.com if test.stdout: 25411482Sandreas.sandberg@arm.com msg.append("*** Standard Out: ***") 25511482Sandreas.sandberg@arm.com msg.append(test.stdout) 25611482Sandreas.sandberg@arm.com 25711482Sandreas.sandberg@arm.com x_state.text = "\n".join(msg) 25811482Sandreas.sandberg@arm.com 25911482Sandreas.sandberg@arm.com return x_test 26011482Sandreas.sandberg@arm.com 26111482Sandreas.sandberg@arm.com def convert_suite(self, x_suites, suite): 26211482Sandreas.sandberg@arm.com x_suite = ET.SubElement(x_suites, "testsuite", 26311482Sandreas.sandberg@arm.com name=suite.name.translate(self.name_table), 26411482Sandreas.sandberg@arm.com time="%f" % suite.runtime()) 26511482Sandreas.sandberg@arm.com errors = 0 26611482Sandreas.sandberg@arm.com failures = 0 26711482Sandreas.sandberg@arm.com skipped = 0 26811482Sandreas.sandberg@arm.com 26911482Sandreas.sandberg@arm.com for test in suite.results: 27011482Sandreas.sandberg@arm.com if test.state != UnitResult.STATE_OK: 27111482Sandreas.sandberg@arm.com if test.state == UnitResult.STATE_SKIPPED: 27211482Sandreas.sandberg@arm.com skipped += 1 27311482Sandreas.sandberg@arm.com elif test.state == UnitResult.STATE_ERROR: 27411482Sandreas.sandberg@arm.com errors += 1 27511482Sandreas.sandberg@arm.com elif test.state == UnitResult.STATE_FAILURE: 27611482Sandreas.sandberg@arm.com failures += 1 27711482Sandreas.sandberg@arm.com 27811482Sandreas.sandberg@arm.com x_test = self.convert_unit(x_suite, test) 27911482Sandreas.sandberg@arm.com 28011482Sandreas.sandberg@arm.com x_suite.set("errors", str(errors)) 28111482Sandreas.sandberg@arm.com x_suite.set("failures", str(failures)) 28211482Sandreas.sandberg@arm.com x_suite.set("skipped", str(skipped)) 28311482Sandreas.sandberg@arm.com x_suite.set("tests", str(len(suite.results))) 28411482Sandreas.sandberg@arm.com 28511482Sandreas.sandberg@arm.com return x_suite 28611482Sandreas.sandberg@arm.com 28711482Sandreas.sandberg@arm.com def convert_suites(self, suites): 28811482Sandreas.sandberg@arm.com x_root = ET.Element("testsuites") 28911482Sandreas.sandberg@arm.com 29011482Sandreas.sandberg@arm.com for suite in suites: 29111482Sandreas.sandberg@arm.com self.convert_suite(x_root, suite) 29211482Sandreas.sandberg@arm.com 29311482Sandreas.sandberg@arm.com return x_root 29411482Sandreas.sandberg@arm.com 29511482Sandreas.sandberg@arm.com def dump_suites(self, suites): 29611482Sandreas.sandberg@arm.com et = ET.ElementTree(self.convert_suites(suites)) 29711482Sandreas.sandberg@arm.com et.write(self.fout, encoding="UTF-8") 298