results.py revision 11512
111482Sandreas.sandberg@arm.com#!/usr/bin/env python 211482Sandreas.sandberg@arm.com# 311482Sandreas.sandberg@arm.com# Copyright (c) 2016 ARM Limited 411482Sandreas.sandberg@arm.com# All rights reserved 511482Sandreas.sandberg@arm.com# 611482Sandreas.sandberg@arm.com# The license below extends only to copyright in the software and shall 711482Sandreas.sandberg@arm.com# not be construed as granting a license to any other intellectual 811482Sandreas.sandberg@arm.com# property including but not limited to intellectual property relating 911482Sandreas.sandberg@arm.com# to a hardware implementation of the functionality of the software 1011482Sandreas.sandberg@arm.com# licensed hereunder. You may use the software subject to the license 1111482Sandreas.sandberg@arm.com# terms below provided that you ensure that this notice is replicated 1211482Sandreas.sandberg@arm.com# unmodified and in its entirety in all distributions of the software, 1311482Sandreas.sandberg@arm.com# modified or unmodified, in source code or in binary form. 1411482Sandreas.sandberg@arm.com# 1511482Sandreas.sandberg@arm.com# Redistribution and use in source and binary forms, with or without 1611482Sandreas.sandberg@arm.com# modification, are permitted provided that the following conditions are 1711482Sandreas.sandberg@arm.com# met: redistributions of source code must retain the above copyright 1811482Sandreas.sandberg@arm.com# notice, this list of conditions and the following disclaimer; 1911482Sandreas.sandberg@arm.com# redistributions in binary form must reproduce the above copyright 2011482Sandreas.sandberg@arm.com# notice, this list of conditions and the following disclaimer in the 2111482Sandreas.sandberg@arm.com# documentation and/or other materials provided with the distribution; 2211482Sandreas.sandberg@arm.com# neither the name of the copyright holders nor the names of its 2311482Sandreas.sandberg@arm.com# contributors may be used to endorse or promote products derived from 2411482Sandreas.sandberg@arm.com# this software without specific prior written permission. 2511482Sandreas.sandberg@arm.com# 2611482Sandreas.sandberg@arm.com# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS 2711482Sandreas.sandberg@arm.com# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT 2811482Sandreas.sandberg@arm.com# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR 2911482Sandreas.sandberg@arm.com# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT 3011482Sandreas.sandberg@arm.com# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, 3111482Sandreas.sandberg@arm.com# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT 3211482Sandreas.sandberg@arm.com# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, 3311482Sandreas.sandberg@arm.com# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY 3411482Sandreas.sandberg@arm.com# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT 3511482Sandreas.sandberg@arm.com# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE 3611482Sandreas.sandberg@arm.com# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 3711482Sandreas.sandberg@arm.com# 3811482Sandreas.sandberg@arm.com# Authors: Andreas Sandberg 3911482Sandreas.sandberg@arm.com 4011482Sandreas.sandberg@arm.comfrom abc import ABCMeta, abstractmethod 4111482Sandreas.sandberg@arm.comimport inspect 4211482Sandreas.sandberg@arm.comimport pickle 4311482Sandreas.sandberg@arm.comimport string 4411482Sandreas.sandberg@arm.comimport sys 4511482Sandreas.sandberg@arm.com 4611482Sandreas.sandberg@arm.comimport xml.etree.cElementTree as ET 4711482Sandreas.sandberg@arm.com 4811482Sandreas.sandberg@arm.comclass UnitResult(object): 4911482Sandreas.sandberg@arm.com """Results of a single test unit. 5011482Sandreas.sandberg@arm.com 5111482Sandreas.sandberg@arm.com A test result can be one of: 5211482Sandreas.sandberg@arm.com - STATE_OK: Test ran successfully. 5311482Sandreas.sandberg@arm.com - STATE_SKIPPED: The test was skipped. 5411482Sandreas.sandberg@arm.com - STATE_ERROR: The test failed to run. 5511482Sandreas.sandberg@arm.com - STATE_FAILED: Test ran, but failed. 5611482Sandreas.sandberg@arm.com 5711482Sandreas.sandberg@arm.com The difference between STATE_ERROR and STATE_FAILED is very 5811482Sandreas.sandberg@arm.com subtle. In a gem5 context, STATE_ERROR would mean that gem5 failed 5911482Sandreas.sandberg@arm.com to start or crashed, while STATE_FAILED would mean that a test 6011482Sandreas.sandberg@arm.com failed (e.g., statistics mismatch). 6111482Sandreas.sandberg@arm.com 6211482Sandreas.sandberg@arm.com """ 6311482Sandreas.sandberg@arm.com 6411482Sandreas.sandberg@arm.com STATE_OK = 0 6511482Sandreas.sandberg@arm.com STATE_SKIPPED = 1 6611482Sandreas.sandberg@arm.com STATE_ERROR = 2 6711482Sandreas.sandberg@arm.com STATE_FAILURE = 3 6811482Sandreas.sandberg@arm.com 6911482Sandreas.sandberg@arm.com state_names = { 7011482Sandreas.sandberg@arm.com STATE_OK : "OK", 7111482Sandreas.sandberg@arm.com STATE_SKIPPED : "SKIPPED", 7211482Sandreas.sandberg@arm.com STATE_ERROR : "ERROR", 7311482Sandreas.sandberg@arm.com STATE_FAILURE : "FAILURE", 7411482Sandreas.sandberg@arm.com } 7511482Sandreas.sandberg@arm.com 7611482Sandreas.sandberg@arm.com def __init__(self, name, state, message="", stderr="", stdout="", 7711482Sandreas.sandberg@arm.com runtime=0.0): 7811482Sandreas.sandberg@arm.com self.name = name 7911482Sandreas.sandberg@arm.com self.state = state 8011482Sandreas.sandberg@arm.com self.message = message 8111482Sandreas.sandberg@arm.com self.stdout = stdout 8211482Sandreas.sandberg@arm.com self.stderr = stderr 8311482Sandreas.sandberg@arm.com self.runtime = runtime 8411482Sandreas.sandberg@arm.com 8511482Sandreas.sandberg@arm.com def skipped(self): 8611482Sandreas.sandberg@arm.com return self.state == UnitResult.STATE_SKIPPED 8711482Sandreas.sandberg@arm.com 8811482Sandreas.sandberg@arm.com def success(self): 8911482Sandreas.sandberg@arm.com return self.state == UnitResult.STATE_OK 9011482Sandreas.sandberg@arm.com 9111482Sandreas.sandberg@arm.com def state_name(self): 9211482Sandreas.sandberg@arm.com return UnitResult.state_names[self.state] 9311482Sandreas.sandberg@arm.com 9411482Sandreas.sandberg@arm.com def __nonzero__(self): 9511482Sandreas.sandberg@arm.com return self.success() or self.skipped() 9611482Sandreas.sandberg@arm.com 9711482Sandreas.sandberg@arm.com def __str__(self): 9811482Sandreas.sandberg@arm.com state_name = self.state_name() 9911482Sandreas.sandberg@arm.com 10011482Sandreas.sandberg@arm.com status = "%s: %s" % (state_name, self.message) if self.message else \ 10111482Sandreas.sandberg@arm.com state_name 10211482Sandreas.sandberg@arm.com 10311482Sandreas.sandberg@arm.com return "%s: %s" % (self.name, status) 10411482Sandreas.sandberg@arm.com 10511482Sandreas.sandberg@arm.comclass TestResult(object): 10611482Sandreas.sandberg@arm.com """Results for from a single test consisting of one or more units.""" 10711482Sandreas.sandberg@arm.com 10811482Sandreas.sandberg@arm.com def __init__(self, name, results=[]): 10911482Sandreas.sandberg@arm.com self.name = name 11011482Sandreas.sandberg@arm.com self.results = results 11111482Sandreas.sandberg@arm.com 11211482Sandreas.sandberg@arm.com def success(self): 11311482Sandreas.sandberg@arm.com return all([ r.success() for r in self.results]) 11411482Sandreas.sandberg@arm.com 11511482Sandreas.sandberg@arm.com def skipped(self): 11611482Sandreas.sandberg@arm.com return all([ r.skipped() for r in self.results]) 11711482Sandreas.sandberg@arm.com 11811512SCurtis.Dunham@arm.com def changed(self): 11911512SCurtis.Dunham@arm.com return self.results[0].success() and self.failed() 12011512SCurtis.Dunham@arm.com 12111482Sandreas.sandberg@arm.com def failed(self): 12211482Sandreas.sandberg@arm.com return any([ not r for r in self.results]) 12311482Sandreas.sandberg@arm.com 12411482Sandreas.sandberg@arm.com def runtime(self): 12511482Sandreas.sandberg@arm.com return sum([ r.runtime for r in self.results ]) 12611482Sandreas.sandberg@arm.com 12711482Sandreas.sandberg@arm.com def __nonzero__(self): 12811482Sandreas.sandberg@arm.com return all([r for r in self.results]) 12911482Sandreas.sandberg@arm.com 13011482Sandreas.sandberg@arm.comclass ResultFormatter(object): 13111482Sandreas.sandberg@arm.com __metaclass__ = ABCMeta 13211482Sandreas.sandberg@arm.com 13311482Sandreas.sandberg@arm.com def __init__(self, fout=sys.stdout, verbose=False): 13411482Sandreas.sandberg@arm.com self.verbose = verbose 13511482Sandreas.sandberg@arm.com self.fout = fout 13611482Sandreas.sandberg@arm.com 13711482Sandreas.sandberg@arm.com @abstractmethod 13811482Sandreas.sandberg@arm.com def dump_suites(self, suites): 13911482Sandreas.sandberg@arm.com pass 14011482Sandreas.sandberg@arm.com 14111482Sandreas.sandberg@arm.comclass Pickle(ResultFormatter): 14211482Sandreas.sandberg@arm.com """Save test results as a binary using Python's pickle 14311482Sandreas.sandberg@arm.com functionality. 14411482Sandreas.sandberg@arm.com 14511482Sandreas.sandberg@arm.com """ 14611482Sandreas.sandberg@arm.com 14711482Sandreas.sandberg@arm.com def __init__(self, **kwargs): 14811482Sandreas.sandberg@arm.com super(Pickle, self).__init__(**kwargs) 14911482Sandreas.sandberg@arm.com 15011482Sandreas.sandberg@arm.com def dump_suites(self, suites): 15111482Sandreas.sandberg@arm.com pickle.dump(suites, self.fout, pickle.HIGHEST_PROTOCOL) 15211482Sandreas.sandberg@arm.com 15311482Sandreas.sandberg@arm.comclass Text(ResultFormatter): 15411482Sandreas.sandberg@arm.com """Output test results as text.""" 15511482Sandreas.sandberg@arm.com 15611482Sandreas.sandberg@arm.com def __init__(self, **kwargs): 15711482Sandreas.sandberg@arm.com super(Text, self).__init__(**kwargs) 15811482Sandreas.sandberg@arm.com 15911482Sandreas.sandberg@arm.com def dump_suites(self, suites): 16011482Sandreas.sandberg@arm.com fout = self.fout 16111482Sandreas.sandberg@arm.com for suite in suites: 16211482Sandreas.sandberg@arm.com print >> fout, "--- %s ---" % suite.name 16311482Sandreas.sandberg@arm.com 16411482Sandreas.sandberg@arm.com for t in suite.results: 16511482Sandreas.sandberg@arm.com print >> fout, "*** %s" % t 16611482Sandreas.sandberg@arm.com 16711482Sandreas.sandberg@arm.com if t and not self.verbose: 16811482Sandreas.sandberg@arm.com continue 16911482Sandreas.sandberg@arm.com 17011482Sandreas.sandberg@arm.com if t.message: 17111482Sandreas.sandberg@arm.com print >> fout, t.message 17211482Sandreas.sandberg@arm.com 17311482Sandreas.sandberg@arm.com if t.stderr: 17411482Sandreas.sandberg@arm.com print >> fout, t.stderr 17511482Sandreas.sandberg@arm.com if t.stdout: 17611482Sandreas.sandberg@arm.com print >> fout, t.stdout 17711482Sandreas.sandberg@arm.com 17811482Sandreas.sandberg@arm.comclass TextSummary(ResultFormatter): 17911482Sandreas.sandberg@arm.com """Output test results as a text summary""" 18011482Sandreas.sandberg@arm.com 18111482Sandreas.sandberg@arm.com def __init__(self, **kwargs): 18211482Sandreas.sandberg@arm.com super(TextSummary, self).__init__(**kwargs) 18311482Sandreas.sandberg@arm.com 18411512SCurtis.Dunham@arm.com def test_status(self, suite): 18511512SCurtis.Dunham@arm.com if suite.skipped(): 18611512SCurtis.Dunham@arm.com return "SKIPPED" 18711512SCurtis.Dunham@arm.com elif suite.changed(): 18811512SCurtis.Dunham@arm.com return "CHANGED" 18911512SCurtis.Dunham@arm.com elif suite: 19011512SCurtis.Dunham@arm.com return "OK" 19111512SCurtis.Dunham@arm.com else: 19211512SCurtis.Dunham@arm.com return "FAILED" 19311512SCurtis.Dunham@arm.com 19411482Sandreas.sandberg@arm.com def dump_suites(self, suites): 19511482Sandreas.sandberg@arm.com fout = self.fout 19611482Sandreas.sandberg@arm.com for suite in suites: 19711512SCurtis.Dunham@arm.com status = self.test_status(suite) 19811482Sandreas.sandberg@arm.com print >> fout, "%s: %s" % (suite.name, status) 19911482Sandreas.sandberg@arm.com 20011482Sandreas.sandberg@arm.comclass JUnit(ResultFormatter): 20111482Sandreas.sandberg@arm.com """Output test results as JUnit XML""" 20211482Sandreas.sandberg@arm.com 20311482Sandreas.sandberg@arm.com def __init__(self, translate_names=True, **kwargs): 20411482Sandreas.sandberg@arm.com super(JUnit, self).__init__(**kwargs) 20511482Sandreas.sandberg@arm.com 20611482Sandreas.sandberg@arm.com if translate_names: 20711482Sandreas.sandberg@arm.com self.name_table = string.maketrans( 20811482Sandreas.sandberg@arm.com "/.", 20911482Sandreas.sandberg@arm.com ".-", 21011482Sandreas.sandberg@arm.com ) 21111482Sandreas.sandberg@arm.com else: 21211482Sandreas.sandberg@arm.com self.name_table = string.maketrans("", "") 21311482Sandreas.sandberg@arm.com 21411482Sandreas.sandberg@arm.com def convert_unit(self, x_suite, test): 21511482Sandreas.sandberg@arm.com x_test = ET.SubElement(x_suite, "testcase", 21611482Sandreas.sandberg@arm.com name=test.name, 21711482Sandreas.sandberg@arm.com time="%f" % test.runtime) 21811482Sandreas.sandberg@arm.com 21911482Sandreas.sandberg@arm.com x_state = None 22011482Sandreas.sandberg@arm.com if test.state == UnitResult.STATE_OK: 22111482Sandreas.sandberg@arm.com pass 22211482Sandreas.sandberg@arm.com elif test.state == UnitResult.STATE_SKIPPED: 22311482Sandreas.sandberg@arm.com x_state = ET.SubElement(x_test, "skipped") 22411482Sandreas.sandberg@arm.com elif test.state == UnitResult.STATE_FAILURE: 22511482Sandreas.sandberg@arm.com x_state = ET.SubElement(x_test, "failure") 22611482Sandreas.sandberg@arm.com elif test.state == UnitResult.STATE_ERROR: 22711482Sandreas.sandberg@arm.com x_state = ET.SubElement(x_test, "error") 22811482Sandreas.sandberg@arm.com else: 22911482Sandreas.sandberg@arm.com assert False, "Unknown test state" 23011482Sandreas.sandberg@arm.com 23111482Sandreas.sandberg@arm.com if x_state is not None: 23211482Sandreas.sandberg@arm.com if test.message: 23311482Sandreas.sandberg@arm.com x_state.set("message", test.message) 23411482Sandreas.sandberg@arm.com 23511482Sandreas.sandberg@arm.com msg = [] 23611482Sandreas.sandberg@arm.com if test.stderr: 23711482Sandreas.sandberg@arm.com msg.append("*** Standard Errror: ***") 23811482Sandreas.sandberg@arm.com msg.append(test.stderr) 23911482Sandreas.sandberg@arm.com if test.stdout: 24011482Sandreas.sandberg@arm.com msg.append("*** Standard Out: ***") 24111482Sandreas.sandberg@arm.com msg.append(test.stdout) 24211482Sandreas.sandberg@arm.com 24311482Sandreas.sandberg@arm.com x_state.text = "\n".join(msg) 24411482Sandreas.sandberg@arm.com 24511482Sandreas.sandberg@arm.com return x_test 24611482Sandreas.sandberg@arm.com 24711482Sandreas.sandberg@arm.com def convert_suite(self, x_suites, suite): 24811482Sandreas.sandberg@arm.com x_suite = ET.SubElement(x_suites, "testsuite", 24911482Sandreas.sandberg@arm.com name=suite.name.translate(self.name_table), 25011482Sandreas.sandberg@arm.com time="%f" % suite.runtime()) 25111482Sandreas.sandberg@arm.com errors = 0 25211482Sandreas.sandberg@arm.com failures = 0 25311482Sandreas.sandberg@arm.com skipped = 0 25411482Sandreas.sandberg@arm.com 25511482Sandreas.sandberg@arm.com for test in suite.results: 25611482Sandreas.sandberg@arm.com if test.state != UnitResult.STATE_OK: 25711482Sandreas.sandberg@arm.com if test.state == UnitResult.STATE_SKIPPED: 25811482Sandreas.sandberg@arm.com skipped += 1 25911482Sandreas.sandberg@arm.com elif test.state == UnitResult.STATE_ERROR: 26011482Sandreas.sandberg@arm.com errors += 1 26111482Sandreas.sandberg@arm.com elif test.state == UnitResult.STATE_FAILURE: 26211482Sandreas.sandberg@arm.com failures += 1 26311482Sandreas.sandberg@arm.com 26411482Sandreas.sandberg@arm.com x_test = self.convert_unit(x_suite, test) 26511482Sandreas.sandberg@arm.com 26611482Sandreas.sandberg@arm.com x_suite.set("errors", str(errors)) 26711482Sandreas.sandberg@arm.com x_suite.set("failures", str(failures)) 26811482Sandreas.sandberg@arm.com x_suite.set("skipped", str(skipped)) 26911482Sandreas.sandberg@arm.com x_suite.set("tests", str(len(suite.results))) 27011482Sandreas.sandberg@arm.com 27111482Sandreas.sandberg@arm.com return x_suite 27211482Sandreas.sandberg@arm.com 27311482Sandreas.sandberg@arm.com def convert_suites(self, suites): 27411482Sandreas.sandberg@arm.com x_root = ET.Element("testsuites") 27511482Sandreas.sandberg@arm.com 27611482Sandreas.sandberg@arm.com for suite in suites: 27711482Sandreas.sandberg@arm.com self.convert_suite(x_root, suite) 27811482Sandreas.sandberg@arm.com 27911482Sandreas.sandberg@arm.com return x_root 28011482Sandreas.sandberg@arm.com 28111482Sandreas.sandberg@arm.com def dump_suites(self, suites): 28211482Sandreas.sandberg@arm.com et = ET.ElementTree(self.convert_suites(suites)) 28311482Sandreas.sandberg@arm.com et.write(self.fout, encoding="UTF-8") 284