results.py revision 11482
111482Sandreas.sandberg@arm.com#!/usr/bin/env python 211482Sandreas.sandberg@arm.com# 311482Sandreas.sandberg@arm.com# Copyright (c) 2016 ARM Limited 411482Sandreas.sandberg@arm.com# All rights reserved 511482Sandreas.sandberg@arm.com# 611482Sandreas.sandberg@arm.com# The license below extends only to copyright in the software and shall 711482Sandreas.sandberg@arm.com# not be construed as granting a license to any other intellectual 811482Sandreas.sandberg@arm.com# property including but not limited to intellectual property relating 911482Sandreas.sandberg@arm.com# to a hardware implementation of the functionality of the software 1011482Sandreas.sandberg@arm.com# licensed hereunder. You may use the software subject to the license 1111482Sandreas.sandberg@arm.com# terms below provided that you ensure that this notice is replicated 1211482Sandreas.sandberg@arm.com# unmodified and in its entirety in all distributions of the software, 1311482Sandreas.sandberg@arm.com# modified or unmodified, in source code or in binary form. 1411482Sandreas.sandberg@arm.com# 1511482Sandreas.sandberg@arm.com# Redistribution and use in source and binary forms, with or without 1611482Sandreas.sandberg@arm.com# modification, are permitted provided that the following conditions are 1711482Sandreas.sandberg@arm.com# met: redistributions of source code must retain the above copyright 1811482Sandreas.sandberg@arm.com# notice, this list of conditions and the following disclaimer; 1911482Sandreas.sandberg@arm.com# redistributions in binary form must reproduce the above copyright 2011482Sandreas.sandberg@arm.com# notice, this list of conditions and the following disclaimer in the 2111482Sandreas.sandberg@arm.com# documentation and/or other materials provided with the distribution; 2211482Sandreas.sandberg@arm.com# neither the name of the copyright holders nor the names of its 2311482Sandreas.sandberg@arm.com# contributors may be used to endorse or promote products derived from 2411482Sandreas.sandberg@arm.com# this software without specific prior written permission. 2511482Sandreas.sandberg@arm.com# 2611482Sandreas.sandberg@arm.com# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS 2711482Sandreas.sandberg@arm.com# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT 2811482Sandreas.sandberg@arm.com# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR 2911482Sandreas.sandberg@arm.com# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT 3011482Sandreas.sandberg@arm.com# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, 3111482Sandreas.sandberg@arm.com# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT 3211482Sandreas.sandberg@arm.com# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, 3311482Sandreas.sandberg@arm.com# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY 3411482Sandreas.sandberg@arm.com# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT 3511482Sandreas.sandberg@arm.com# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE 3611482Sandreas.sandberg@arm.com# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 3711482Sandreas.sandberg@arm.com# 3811482Sandreas.sandberg@arm.com# Authors: Andreas Sandberg 3911482Sandreas.sandberg@arm.com 4011482Sandreas.sandberg@arm.comfrom abc import ABCMeta, abstractmethod 4111482Sandreas.sandberg@arm.comimport inspect 4211482Sandreas.sandberg@arm.comimport pickle 4311482Sandreas.sandberg@arm.comimport string 4411482Sandreas.sandberg@arm.comimport sys 4511482Sandreas.sandberg@arm.com 4611482Sandreas.sandberg@arm.comimport xml.etree.cElementTree as ET 4711482Sandreas.sandberg@arm.com 4811482Sandreas.sandberg@arm.comclass UnitResult(object): 4911482Sandreas.sandberg@arm.com """Results of a single test unit. 5011482Sandreas.sandberg@arm.com 5111482Sandreas.sandberg@arm.com A test result can be one of: 5211482Sandreas.sandberg@arm.com - STATE_OK: Test ran successfully. 5311482Sandreas.sandberg@arm.com - STATE_SKIPPED: The test was skipped. 5411482Sandreas.sandberg@arm.com - STATE_ERROR: The test failed to run. 5511482Sandreas.sandberg@arm.com - STATE_FAILED: Test ran, but failed. 5611482Sandreas.sandberg@arm.com 5711482Sandreas.sandberg@arm.com The difference between STATE_ERROR and STATE_FAILED is very 5811482Sandreas.sandberg@arm.com subtle. In a gem5 context, STATE_ERROR would mean that gem5 failed 5911482Sandreas.sandberg@arm.com to start or crashed, while STATE_FAILED would mean that a test 6011482Sandreas.sandberg@arm.com failed (e.g., statistics mismatch). 6111482Sandreas.sandberg@arm.com 6211482Sandreas.sandberg@arm.com """ 6311482Sandreas.sandberg@arm.com 6411482Sandreas.sandberg@arm.com STATE_OK = 0 6511482Sandreas.sandberg@arm.com STATE_SKIPPED = 1 6611482Sandreas.sandberg@arm.com STATE_ERROR = 2 6711482Sandreas.sandberg@arm.com STATE_FAILURE = 3 6811482Sandreas.sandberg@arm.com 6911482Sandreas.sandberg@arm.com state_names = { 7011482Sandreas.sandberg@arm.com STATE_OK : "OK", 7111482Sandreas.sandberg@arm.com STATE_SKIPPED : "SKIPPED", 7211482Sandreas.sandberg@arm.com STATE_ERROR : "ERROR", 7311482Sandreas.sandberg@arm.com STATE_FAILURE : "FAILURE", 7411482Sandreas.sandberg@arm.com } 7511482Sandreas.sandberg@arm.com 7611482Sandreas.sandberg@arm.com def __init__(self, name, state, message="", stderr="", stdout="", 7711482Sandreas.sandberg@arm.com runtime=0.0): 7811482Sandreas.sandberg@arm.com self.name = name 7911482Sandreas.sandberg@arm.com self.state = state 8011482Sandreas.sandberg@arm.com self.message = message 8111482Sandreas.sandberg@arm.com self.stdout = stdout 8211482Sandreas.sandberg@arm.com self.stderr = stderr 8311482Sandreas.sandberg@arm.com self.runtime = runtime 8411482Sandreas.sandberg@arm.com 8511482Sandreas.sandberg@arm.com def skipped(self): 8611482Sandreas.sandberg@arm.com return self.state == UnitResult.STATE_SKIPPED 8711482Sandreas.sandberg@arm.com 8811482Sandreas.sandberg@arm.com def success(self): 8911482Sandreas.sandberg@arm.com return self.state == UnitResult.STATE_OK 9011482Sandreas.sandberg@arm.com 9111482Sandreas.sandberg@arm.com def state_name(self): 9211482Sandreas.sandberg@arm.com return UnitResult.state_names[self.state] 9311482Sandreas.sandberg@arm.com 9411482Sandreas.sandberg@arm.com def __nonzero__(self): 9511482Sandreas.sandberg@arm.com return self.success() or self.skipped() 9611482Sandreas.sandberg@arm.com 9711482Sandreas.sandberg@arm.com def __str__(self): 9811482Sandreas.sandberg@arm.com state_name = self.state_name() 9911482Sandreas.sandberg@arm.com 10011482Sandreas.sandberg@arm.com status = "%s: %s" % (state_name, self.message) if self.message else \ 10111482Sandreas.sandberg@arm.com state_name 10211482Sandreas.sandberg@arm.com 10311482Sandreas.sandberg@arm.com return "%s: %s" % (self.name, status) 10411482Sandreas.sandberg@arm.com 10511482Sandreas.sandberg@arm.comclass TestResult(object): 10611482Sandreas.sandberg@arm.com """Results for from a single test consisting of one or more units.""" 10711482Sandreas.sandberg@arm.com 10811482Sandreas.sandberg@arm.com def __init__(self, name, results=[]): 10911482Sandreas.sandberg@arm.com self.name = name 11011482Sandreas.sandberg@arm.com self.results = results 11111482Sandreas.sandberg@arm.com 11211482Sandreas.sandberg@arm.com def success(self): 11311482Sandreas.sandberg@arm.com return all([ r.success() for r in self.results]) 11411482Sandreas.sandberg@arm.com 11511482Sandreas.sandberg@arm.com def skipped(self): 11611482Sandreas.sandberg@arm.com return all([ r.skipped() for r in self.results]) 11711482Sandreas.sandberg@arm.com 11811482Sandreas.sandberg@arm.com def failed(self): 11911482Sandreas.sandberg@arm.com return any([ not r for r in self.results]) 12011482Sandreas.sandberg@arm.com 12111482Sandreas.sandberg@arm.com def runtime(self): 12211482Sandreas.sandberg@arm.com return sum([ r.runtime for r in self.results ]) 12311482Sandreas.sandberg@arm.com 12411482Sandreas.sandberg@arm.com def __nonzero__(self): 12511482Sandreas.sandberg@arm.com return all([r for r in self.results]) 12611482Sandreas.sandberg@arm.com 12711482Sandreas.sandberg@arm.comclass ResultFormatter(object): 12811482Sandreas.sandberg@arm.com __metaclass__ = ABCMeta 12911482Sandreas.sandberg@arm.com 13011482Sandreas.sandberg@arm.com def __init__(self, fout=sys.stdout, verbose=False): 13111482Sandreas.sandberg@arm.com self.verbose = verbose 13211482Sandreas.sandberg@arm.com self.fout = fout 13311482Sandreas.sandberg@arm.com 13411482Sandreas.sandberg@arm.com @abstractmethod 13511482Sandreas.sandberg@arm.com def dump_suites(self, suites): 13611482Sandreas.sandberg@arm.com pass 13711482Sandreas.sandberg@arm.com 13811482Sandreas.sandberg@arm.comclass Pickle(ResultFormatter): 13911482Sandreas.sandberg@arm.com """Save test results as a binary using Python's pickle 14011482Sandreas.sandberg@arm.com functionality. 14111482Sandreas.sandberg@arm.com 14211482Sandreas.sandberg@arm.com """ 14311482Sandreas.sandberg@arm.com 14411482Sandreas.sandberg@arm.com def __init__(self, **kwargs): 14511482Sandreas.sandberg@arm.com super(Pickle, self).__init__(**kwargs) 14611482Sandreas.sandberg@arm.com 14711482Sandreas.sandberg@arm.com def dump_suites(self, suites): 14811482Sandreas.sandberg@arm.com pickle.dump(suites, self.fout, pickle.HIGHEST_PROTOCOL) 14911482Sandreas.sandberg@arm.com 15011482Sandreas.sandberg@arm.comclass Text(ResultFormatter): 15111482Sandreas.sandberg@arm.com """Output test results as text.""" 15211482Sandreas.sandberg@arm.com 15311482Sandreas.sandberg@arm.com def __init__(self, **kwargs): 15411482Sandreas.sandberg@arm.com super(Text, self).__init__(**kwargs) 15511482Sandreas.sandberg@arm.com 15611482Sandreas.sandberg@arm.com def dump_suites(self, suites): 15711482Sandreas.sandberg@arm.com fout = self.fout 15811482Sandreas.sandberg@arm.com for suite in suites: 15911482Sandreas.sandberg@arm.com print >> fout, "--- %s ---" % suite.name 16011482Sandreas.sandberg@arm.com 16111482Sandreas.sandberg@arm.com for t in suite.results: 16211482Sandreas.sandberg@arm.com print >> fout, "*** %s" % t 16311482Sandreas.sandberg@arm.com 16411482Sandreas.sandberg@arm.com if t and not self.verbose: 16511482Sandreas.sandberg@arm.com continue 16611482Sandreas.sandberg@arm.com 16711482Sandreas.sandberg@arm.com if t.message: 16811482Sandreas.sandberg@arm.com print >> fout, t.message 16911482Sandreas.sandberg@arm.com 17011482Sandreas.sandberg@arm.com if t.stderr: 17111482Sandreas.sandberg@arm.com print >> fout, t.stderr 17211482Sandreas.sandberg@arm.com if t.stdout: 17311482Sandreas.sandberg@arm.com print >> fout, t.stdout 17411482Sandreas.sandberg@arm.com 17511482Sandreas.sandberg@arm.comclass TextSummary(ResultFormatter): 17611482Sandreas.sandberg@arm.com """Output test results as a text summary""" 17711482Sandreas.sandberg@arm.com 17811482Sandreas.sandberg@arm.com def __init__(self, **kwargs): 17911482Sandreas.sandberg@arm.com super(TextSummary, self).__init__(**kwargs) 18011482Sandreas.sandberg@arm.com 18111482Sandreas.sandberg@arm.com def dump_suites(self, suites): 18211482Sandreas.sandberg@arm.com fout = self.fout 18311482Sandreas.sandberg@arm.com for suite in suites: 18411482Sandreas.sandberg@arm.com status = "SKIPPED" if suite.skipped() else \ 18511482Sandreas.sandberg@arm.com ("OK" if suite else "FAILED") 18611482Sandreas.sandberg@arm.com print >> fout, "%s: %s" % (suite.name, status) 18711482Sandreas.sandberg@arm.com 18811482Sandreas.sandberg@arm.comclass JUnit(ResultFormatter): 18911482Sandreas.sandberg@arm.com """Output test results as JUnit XML""" 19011482Sandreas.sandberg@arm.com 19111482Sandreas.sandberg@arm.com def __init__(self, translate_names=True, **kwargs): 19211482Sandreas.sandberg@arm.com super(JUnit, self).__init__(**kwargs) 19311482Sandreas.sandberg@arm.com 19411482Sandreas.sandberg@arm.com if translate_names: 19511482Sandreas.sandberg@arm.com self.name_table = string.maketrans( 19611482Sandreas.sandberg@arm.com "/.", 19711482Sandreas.sandberg@arm.com ".-", 19811482Sandreas.sandberg@arm.com ) 19911482Sandreas.sandberg@arm.com else: 20011482Sandreas.sandberg@arm.com self.name_table = string.maketrans("", "") 20111482Sandreas.sandberg@arm.com 20211482Sandreas.sandberg@arm.com def convert_unit(self, x_suite, test): 20311482Sandreas.sandberg@arm.com x_test = ET.SubElement(x_suite, "testcase", 20411482Sandreas.sandberg@arm.com name=test.name, 20511482Sandreas.sandberg@arm.com time="%f" % test.runtime) 20611482Sandreas.sandberg@arm.com 20711482Sandreas.sandberg@arm.com x_state = None 20811482Sandreas.sandberg@arm.com if test.state == UnitResult.STATE_OK: 20911482Sandreas.sandberg@arm.com pass 21011482Sandreas.sandberg@arm.com elif test.state == UnitResult.STATE_SKIPPED: 21111482Sandreas.sandberg@arm.com x_state = ET.SubElement(x_test, "skipped") 21211482Sandreas.sandberg@arm.com elif test.state == UnitResult.STATE_FAILURE: 21311482Sandreas.sandberg@arm.com x_state = ET.SubElement(x_test, "failure") 21411482Sandreas.sandberg@arm.com elif test.state == UnitResult.STATE_ERROR: 21511482Sandreas.sandberg@arm.com x_state = ET.SubElement(x_test, "error") 21611482Sandreas.sandberg@arm.com else: 21711482Sandreas.sandberg@arm.com assert False, "Unknown test state" 21811482Sandreas.sandberg@arm.com 21911482Sandreas.sandberg@arm.com if x_state is not None: 22011482Sandreas.sandberg@arm.com if test.message: 22111482Sandreas.sandberg@arm.com x_state.set("message", test.message) 22211482Sandreas.sandberg@arm.com 22311482Sandreas.sandberg@arm.com msg = [] 22411482Sandreas.sandberg@arm.com if test.stderr: 22511482Sandreas.sandberg@arm.com msg.append("*** Standard Errror: ***") 22611482Sandreas.sandberg@arm.com msg.append(test.stderr) 22711482Sandreas.sandberg@arm.com if test.stdout: 22811482Sandreas.sandberg@arm.com msg.append("*** Standard Out: ***") 22911482Sandreas.sandberg@arm.com msg.append(test.stdout) 23011482Sandreas.sandberg@arm.com 23111482Sandreas.sandberg@arm.com x_state.text = "\n".join(msg) 23211482Sandreas.sandberg@arm.com 23311482Sandreas.sandberg@arm.com return x_test 23411482Sandreas.sandberg@arm.com 23511482Sandreas.sandberg@arm.com def convert_suite(self, x_suites, suite): 23611482Sandreas.sandberg@arm.com x_suite = ET.SubElement(x_suites, "testsuite", 23711482Sandreas.sandberg@arm.com name=suite.name.translate(self.name_table), 23811482Sandreas.sandberg@arm.com time="%f" % suite.runtime()) 23911482Sandreas.sandberg@arm.com errors = 0 24011482Sandreas.sandberg@arm.com failures = 0 24111482Sandreas.sandberg@arm.com skipped = 0 24211482Sandreas.sandberg@arm.com 24311482Sandreas.sandberg@arm.com for test in suite.results: 24411482Sandreas.sandberg@arm.com if test.state != UnitResult.STATE_OK: 24511482Sandreas.sandberg@arm.com if test.state == UnitResult.STATE_SKIPPED: 24611482Sandreas.sandberg@arm.com skipped += 1 24711482Sandreas.sandberg@arm.com elif test.state == UnitResult.STATE_ERROR: 24811482Sandreas.sandberg@arm.com errors += 1 24911482Sandreas.sandberg@arm.com elif test.state == UnitResult.STATE_FAILURE: 25011482Sandreas.sandberg@arm.com failures += 1 25111482Sandreas.sandberg@arm.com 25211482Sandreas.sandberg@arm.com x_test = self.convert_unit(x_suite, test) 25311482Sandreas.sandberg@arm.com 25411482Sandreas.sandberg@arm.com x_suite.set("errors", str(errors)) 25511482Sandreas.sandberg@arm.com x_suite.set("failures", str(failures)) 25611482Sandreas.sandberg@arm.com x_suite.set("skipped", str(skipped)) 25711482Sandreas.sandberg@arm.com x_suite.set("tests", str(len(suite.results))) 25811482Sandreas.sandberg@arm.com 25911482Sandreas.sandberg@arm.com return x_suite 26011482Sandreas.sandberg@arm.com 26111482Sandreas.sandberg@arm.com def convert_suites(self, suites): 26211482Sandreas.sandberg@arm.com x_root = ET.Element("testsuites") 26311482Sandreas.sandberg@arm.com 26411482Sandreas.sandberg@arm.com for suite in suites: 26511482Sandreas.sandberg@arm.com self.convert_suite(x_root, suite) 26611482Sandreas.sandberg@arm.com 26711482Sandreas.sandberg@arm.com return x_root 26811482Sandreas.sandberg@arm.com 26911482Sandreas.sandberg@arm.com def dump_suites(self, suites): 27011482Sandreas.sandberg@arm.com et = ET.ElementTree(self.convert_suites(suites)) 27111482Sandreas.sandberg@arm.com et.write(self.fout, encoding="UTF-8") 272