results.py revision 13540
113540Sandrea.mondelli@ucf.edu#!/usr/bin/env python2.7 211482Sandreas.sandberg@arm.com# 311482Sandreas.sandberg@arm.com# Copyright (c) 2016 ARM Limited 411482Sandreas.sandberg@arm.com# All rights reserved 511482Sandreas.sandberg@arm.com# 611482Sandreas.sandberg@arm.com# The license below extends only to copyright in the software and shall 711482Sandreas.sandberg@arm.com# not be construed as granting a license to any other intellectual 811482Sandreas.sandberg@arm.com# property including but not limited to intellectual property relating 911482Sandreas.sandberg@arm.com# to a hardware implementation of the functionality of the software 1011482Sandreas.sandberg@arm.com# licensed hereunder. You may use the software subject to the license 1111482Sandreas.sandberg@arm.com# terms below provided that you ensure that this notice is replicated 1211482Sandreas.sandberg@arm.com# unmodified and in its entirety in all distributions of the software, 1311482Sandreas.sandberg@arm.com# modified or unmodified, in source code or in binary form. 1411482Sandreas.sandberg@arm.com# 1511482Sandreas.sandberg@arm.com# Redistribution and use in source and binary forms, with or without 1611482Sandreas.sandberg@arm.com# modification, are permitted provided that the following conditions are 1711482Sandreas.sandberg@arm.com# met: redistributions of source code must retain the above copyright 1811482Sandreas.sandberg@arm.com# notice, this list of conditions and the following disclaimer; 1911482Sandreas.sandberg@arm.com# redistributions in binary form must reproduce the above copyright 2011482Sandreas.sandberg@arm.com# notice, this list of conditions and the following disclaimer in the 2111482Sandreas.sandberg@arm.com# documentation and/or other materials provided with the distribution; 2211482Sandreas.sandberg@arm.com# neither the name of the copyright holders nor the names of its 2311482Sandreas.sandberg@arm.com# contributors may be used to endorse or promote products derived from 2411482Sandreas.sandberg@arm.com# this software without specific prior written permission. 2511482Sandreas.sandberg@arm.com# 2611482Sandreas.sandberg@arm.com# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS 2711482Sandreas.sandberg@arm.com# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT 2811482Sandreas.sandberg@arm.com# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR 2911482Sandreas.sandberg@arm.com# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT 3011482Sandreas.sandberg@arm.com# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, 3111482Sandreas.sandberg@arm.com# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT 3211482Sandreas.sandberg@arm.com# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, 3311482Sandreas.sandberg@arm.com# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY 3411482Sandreas.sandberg@arm.com# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT 3511482Sandreas.sandberg@arm.com# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE 3611482Sandreas.sandberg@arm.com# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 3711482Sandreas.sandberg@arm.com# 3811482Sandreas.sandberg@arm.com# Authors: Andreas Sandberg 3911482Sandreas.sandberg@arm.com 4012581Sgiacomo.travaglini@arm.comfrom __future__ import print_function 4112581Sgiacomo.travaglini@arm.com 4211482Sandreas.sandberg@arm.comfrom abc import ABCMeta, abstractmethod 4311482Sandreas.sandberg@arm.comimport inspect 4411482Sandreas.sandberg@arm.comimport pickle 4511482Sandreas.sandberg@arm.comimport string 4611482Sandreas.sandberg@arm.comimport sys 4711482Sandreas.sandberg@arm.com 4811482Sandreas.sandberg@arm.comimport xml.etree.cElementTree as ET 4911482Sandreas.sandberg@arm.com 5011482Sandreas.sandberg@arm.comclass UnitResult(object): 5111482Sandreas.sandberg@arm.com """Results of a single test unit. 5211482Sandreas.sandberg@arm.com 5311482Sandreas.sandberg@arm.com A test result can be one of: 5411482Sandreas.sandberg@arm.com - STATE_OK: Test ran successfully. 5511482Sandreas.sandberg@arm.com - STATE_SKIPPED: The test was skipped. 5611482Sandreas.sandberg@arm.com - STATE_ERROR: The test failed to run. 5711482Sandreas.sandberg@arm.com - STATE_FAILED: Test ran, but failed. 5811482Sandreas.sandberg@arm.com 5911482Sandreas.sandberg@arm.com The difference between STATE_ERROR and STATE_FAILED is very 6011482Sandreas.sandberg@arm.com subtle. In a gem5 context, STATE_ERROR would mean that gem5 failed 6111482Sandreas.sandberg@arm.com to start or crashed, while STATE_FAILED would mean that a test 6211482Sandreas.sandberg@arm.com failed (e.g., statistics mismatch). 6311482Sandreas.sandberg@arm.com 6411482Sandreas.sandberg@arm.com """ 6511482Sandreas.sandberg@arm.com 6611482Sandreas.sandberg@arm.com STATE_OK = 0 6711482Sandreas.sandberg@arm.com STATE_SKIPPED = 1 6811482Sandreas.sandberg@arm.com STATE_ERROR = 2 6911482Sandreas.sandberg@arm.com STATE_FAILURE = 3 7011482Sandreas.sandberg@arm.com 7111482Sandreas.sandberg@arm.com state_names = { 7211482Sandreas.sandberg@arm.com STATE_OK : "OK", 7311482Sandreas.sandberg@arm.com STATE_SKIPPED : "SKIPPED", 7411482Sandreas.sandberg@arm.com STATE_ERROR : "ERROR", 7511482Sandreas.sandberg@arm.com STATE_FAILURE : "FAILURE", 7611482Sandreas.sandberg@arm.com } 7711482Sandreas.sandberg@arm.com 7811482Sandreas.sandberg@arm.com def __init__(self, name, state, message="", stderr="", stdout="", 7911482Sandreas.sandberg@arm.com runtime=0.0): 8011482Sandreas.sandberg@arm.com self.name = name 8111482Sandreas.sandberg@arm.com self.state = state 8211482Sandreas.sandberg@arm.com self.message = message 8311482Sandreas.sandberg@arm.com self.stdout = stdout 8411482Sandreas.sandberg@arm.com self.stderr = stderr 8511482Sandreas.sandberg@arm.com self.runtime = runtime 8611482Sandreas.sandberg@arm.com 8711482Sandreas.sandberg@arm.com def skipped(self): 8811482Sandreas.sandberg@arm.com return self.state == UnitResult.STATE_SKIPPED 8911482Sandreas.sandberg@arm.com 9011482Sandreas.sandberg@arm.com def success(self): 9111482Sandreas.sandberg@arm.com return self.state == UnitResult.STATE_OK 9211482Sandreas.sandberg@arm.com 9311482Sandreas.sandberg@arm.com def state_name(self): 9411482Sandreas.sandberg@arm.com return UnitResult.state_names[self.state] 9511482Sandreas.sandberg@arm.com 9611482Sandreas.sandberg@arm.com def __nonzero__(self): 9711482Sandreas.sandberg@arm.com return self.success() or self.skipped() 9811482Sandreas.sandberg@arm.com 9911482Sandreas.sandberg@arm.com def __str__(self): 10011482Sandreas.sandberg@arm.com state_name = self.state_name() 10111482Sandreas.sandberg@arm.com 10211482Sandreas.sandberg@arm.com status = "%s: %s" % (state_name, self.message) if self.message else \ 10311482Sandreas.sandberg@arm.com state_name 10411482Sandreas.sandberg@arm.com 10511482Sandreas.sandberg@arm.com return "%s: %s" % (self.name, status) 10611482Sandreas.sandberg@arm.com 10711482Sandreas.sandberg@arm.comclass TestResult(object): 10811482Sandreas.sandberg@arm.com """Results for from a single test consisting of one or more units.""" 10911482Sandreas.sandberg@arm.com 11011542Sandreas.sandberg@arm.com def __init__(self, name, run_results=[], verify_results=[]): 11111482Sandreas.sandberg@arm.com self.name = name 11211542Sandreas.sandberg@arm.com self.results = run_results + verify_results 11311542Sandreas.sandberg@arm.com self.run_results = run_results 11411542Sandreas.sandberg@arm.com self.verify_results = verify_results 11511482Sandreas.sandberg@arm.com 11611482Sandreas.sandberg@arm.com def success(self): 11711542Sandreas.sandberg@arm.com return self.success_run() and self.success_verify() 11811542Sandreas.sandberg@arm.com 11911542Sandreas.sandberg@arm.com def success_run(self): 12011542Sandreas.sandberg@arm.com return all([ r.success() for r in self.run_results ]) 12111542Sandreas.sandberg@arm.com 12211542Sandreas.sandberg@arm.com def success_verify(self): 12311542Sandreas.sandberg@arm.com return all([ r.success() for r in self.verify_results ]) 12411542Sandreas.sandberg@arm.com 12511542Sandreas.sandberg@arm.com def failed(self): 12611542Sandreas.sandberg@arm.com return self.failed_run() or self.failed_verify() 12711542Sandreas.sandberg@arm.com 12811542Sandreas.sandberg@arm.com def failed_run(self): 12911542Sandreas.sandberg@arm.com return any([ not r for r in self.run_results ]) 13011542Sandreas.sandberg@arm.com 13111542Sandreas.sandberg@arm.com def failed_verify(self): 13211542Sandreas.sandberg@arm.com return any([ not r for r in self.verify_results ]) 13311482Sandreas.sandberg@arm.com 13411482Sandreas.sandberg@arm.com def skipped(self): 13511542Sandreas.sandberg@arm.com return all([ r.skipped() for r in self.run_results ]) 13611482Sandreas.sandberg@arm.com 13711512SCurtis.Dunham@arm.com def changed(self): 13811542Sandreas.sandberg@arm.com return self.success_run() and self.failed_verify() 13911482Sandreas.sandberg@arm.com 14011482Sandreas.sandberg@arm.com def runtime(self): 14111482Sandreas.sandberg@arm.com return sum([ r.runtime for r in self.results ]) 14211482Sandreas.sandberg@arm.com 14311482Sandreas.sandberg@arm.com def __nonzero__(self): 14411542Sandreas.sandberg@arm.com return all([ r for r in self.results ]) 14511482Sandreas.sandberg@arm.com 14611482Sandreas.sandberg@arm.comclass ResultFormatter(object): 14711482Sandreas.sandberg@arm.com __metaclass__ = ABCMeta 14811482Sandreas.sandberg@arm.com 14911482Sandreas.sandberg@arm.com def __init__(self, fout=sys.stdout, verbose=False): 15011482Sandreas.sandberg@arm.com self.verbose = verbose 15111482Sandreas.sandberg@arm.com self.fout = fout 15211482Sandreas.sandberg@arm.com 15311482Sandreas.sandberg@arm.com @abstractmethod 15411482Sandreas.sandberg@arm.com def dump_suites(self, suites): 15511482Sandreas.sandberg@arm.com pass 15611482Sandreas.sandberg@arm.com 15711482Sandreas.sandberg@arm.comclass Pickle(ResultFormatter): 15811482Sandreas.sandberg@arm.com """Save test results as a binary using Python's pickle 15911482Sandreas.sandberg@arm.com functionality. 16011482Sandreas.sandberg@arm.com 16111482Sandreas.sandberg@arm.com """ 16211482Sandreas.sandberg@arm.com 16311482Sandreas.sandberg@arm.com def __init__(self, **kwargs): 16411482Sandreas.sandberg@arm.com super(Pickle, self).__init__(**kwargs) 16511482Sandreas.sandberg@arm.com 16611482Sandreas.sandberg@arm.com def dump_suites(self, suites): 16711482Sandreas.sandberg@arm.com pickle.dump(suites, self.fout, pickle.HIGHEST_PROTOCOL) 16811482Sandreas.sandberg@arm.com 16911482Sandreas.sandberg@arm.comclass Text(ResultFormatter): 17011482Sandreas.sandberg@arm.com """Output test results as text.""" 17111482Sandreas.sandberg@arm.com 17211482Sandreas.sandberg@arm.com def __init__(self, **kwargs): 17311482Sandreas.sandberg@arm.com super(Text, self).__init__(**kwargs) 17411482Sandreas.sandberg@arm.com 17511482Sandreas.sandberg@arm.com def dump_suites(self, suites): 17611482Sandreas.sandberg@arm.com fout = self.fout 17711482Sandreas.sandberg@arm.com for suite in suites: 17812581Sgiacomo.travaglini@arm.com print("--- %s ---" % suite.name, file=fout) 17911482Sandreas.sandberg@arm.com 18011482Sandreas.sandberg@arm.com for t in suite.results: 18112581Sgiacomo.travaglini@arm.com print("*** %s" % t, file=fout) 18211482Sandreas.sandberg@arm.com 18311482Sandreas.sandberg@arm.com if t and not self.verbose: 18411482Sandreas.sandberg@arm.com continue 18511482Sandreas.sandberg@arm.com 18611482Sandreas.sandberg@arm.com if t.message: 18712581Sgiacomo.travaglini@arm.com print(t.message, file=fout) 18811482Sandreas.sandberg@arm.com 18911482Sandreas.sandberg@arm.com if t.stderr: 19012581Sgiacomo.travaglini@arm.com print(t.stderr, file=fout) 19111482Sandreas.sandberg@arm.com if t.stdout: 19212581Sgiacomo.travaglini@arm.com print(t.stdout, file=fout) 19311482Sandreas.sandberg@arm.com 19411482Sandreas.sandberg@arm.comclass TextSummary(ResultFormatter): 19511482Sandreas.sandberg@arm.com """Output test results as a text summary""" 19611482Sandreas.sandberg@arm.com 19711482Sandreas.sandberg@arm.com def __init__(self, **kwargs): 19811482Sandreas.sandberg@arm.com super(TextSummary, self).__init__(**kwargs) 19911482Sandreas.sandberg@arm.com 20011512SCurtis.Dunham@arm.com def test_status(self, suite): 20111512SCurtis.Dunham@arm.com if suite.skipped(): 20211512SCurtis.Dunham@arm.com return "SKIPPED" 20311512SCurtis.Dunham@arm.com elif suite.changed(): 20411512SCurtis.Dunham@arm.com return "CHANGED" 20511512SCurtis.Dunham@arm.com elif suite: 20611512SCurtis.Dunham@arm.com return "OK" 20711512SCurtis.Dunham@arm.com else: 20811512SCurtis.Dunham@arm.com return "FAILED" 20911512SCurtis.Dunham@arm.com 21011482Sandreas.sandberg@arm.com def dump_suites(self, suites): 21111482Sandreas.sandberg@arm.com fout = self.fout 21211482Sandreas.sandberg@arm.com for suite in suites: 21311512SCurtis.Dunham@arm.com status = self.test_status(suite) 21412581Sgiacomo.travaglini@arm.com print("%s: %s" % (suite.name, status), file=fout) 21511482Sandreas.sandberg@arm.com 21611482Sandreas.sandberg@arm.comclass JUnit(ResultFormatter): 21711482Sandreas.sandberg@arm.com """Output test results as JUnit XML""" 21811482Sandreas.sandberg@arm.com 21911482Sandreas.sandberg@arm.com def __init__(self, translate_names=True, **kwargs): 22011482Sandreas.sandberg@arm.com super(JUnit, self).__init__(**kwargs) 22111482Sandreas.sandberg@arm.com 22211482Sandreas.sandberg@arm.com if translate_names: 22311482Sandreas.sandberg@arm.com self.name_table = string.maketrans( 22411482Sandreas.sandberg@arm.com "/.", 22511482Sandreas.sandberg@arm.com ".-", 22611482Sandreas.sandberg@arm.com ) 22711482Sandreas.sandberg@arm.com else: 22811482Sandreas.sandberg@arm.com self.name_table = string.maketrans("", "") 22911482Sandreas.sandberg@arm.com 23011482Sandreas.sandberg@arm.com def convert_unit(self, x_suite, test): 23111482Sandreas.sandberg@arm.com x_test = ET.SubElement(x_suite, "testcase", 23211482Sandreas.sandberg@arm.com name=test.name, 23311482Sandreas.sandberg@arm.com time="%f" % test.runtime) 23411482Sandreas.sandberg@arm.com 23511482Sandreas.sandberg@arm.com x_state = None 23611482Sandreas.sandberg@arm.com if test.state == UnitResult.STATE_OK: 23711482Sandreas.sandberg@arm.com pass 23811482Sandreas.sandberg@arm.com elif test.state == UnitResult.STATE_SKIPPED: 23911482Sandreas.sandberg@arm.com x_state = ET.SubElement(x_test, "skipped") 24011482Sandreas.sandberg@arm.com elif test.state == UnitResult.STATE_FAILURE: 24111482Sandreas.sandberg@arm.com x_state = ET.SubElement(x_test, "failure") 24211482Sandreas.sandberg@arm.com elif test.state == UnitResult.STATE_ERROR: 24311482Sandreas.sandberg@arm.com x_state = ET.SubElement(x_test, "error") 24411482Sandreas.sandberg@arm.com else: 24511482Sandreas.sandberg@arm.com assert False, "Unknown test state" 24611482Sandreas.sandberg@arm.com 24711482Sandreas.sandberg@arm.com if x_state is not None: 24811482Sandreas.sandberg@arm.com if test.message: 24911482Sandreas.sandberg@arm.com x_state.set("message", test.message) 25011482Sandreas.sandberg@arm.com 25111482Sandreas.sandberg@arm.com msg = [] 25211482Sandreas.sandberg@arm.com if test.stderr: 25311482Sandreas.sandberg@arm.com msg.append("*** Standard Errror: ***") 25411482Sandreas.sandberg@arm.com msg.append(test.stderr) 25511482Sandreas.sandberg@arm.com if test.stdout: 25611482Sandreas.sandberg@arm.com msg.append("*** Standard Out: ***") 25711482Sandreas.sandberg@arm.com msg.append(test.stdout) 25811482Sandreas.sandberg@arm.com 25911482Sandreas.sandberg@arm.com x_state.text = "\n".join(msg) 26011482Sandreas.sandberg@arm.com 26111482Sandreas.sandberg@arm.com return x_test 26211482Sandreas.sandberg@arm.com 26311482Sandreas.sandberg@arm.com def convert_suite(self, x_suites, suite): 26411482Sandreas.sandberg@arm.com x_suite = ET.SubElement(x_suites, "testsuite", 26511482Sandreas.sandberg@arm.com name=suite.name.translate(self.name_table), 26611482Sandreas.sandberg@arm.com time="%f" % suite.runtime()) 26711482Sandreas.sandberg@arm.com errors = 0 26811482Sandreas.sandberg@arm.com failures = 0 26911482Sandreas.sandberg@arm.com skipped = 0 27011482Sandreas.sandberg@arm.com 27111482Sandreas.sandberg@arm.com for test in suite.results: 27211482Sandreas.sandberg@arm.com if test.state != UnitResult.STATE_OK: 27311482Sandreas.sandberg@arm.com if test.state == UnitResult.STATE_SKIPPED: 27411482Sandreas.sandberg@arm.com skipped += 1 27511482Sandreas.sandberg@arm.com elif test.state == UnitResult.STATE_ERROR: 27611482Sandreas.sandberg@arm.com errors += 1 27711482Sandreas.sandberg@arm.com elif test.state == UnitResult.STATE_FAILURE: 27811482Sandreas.sandberg@arm.com failures += 1 27911482Sandreas.sandberg@arm.com 28011482Sandreas.sandberg@arm.com x_test = self.convert_unit(x_suite, test) 28111482Sandreas.sandberg@arm.com 28211482Sandreas.sandberg@arm.com x_suite.set("errors", str(errors)) 28311482Sandreas.sandberg@arm.com x_suite.set("failures", str(failures)) 28411482Sandreas.sandberg@arm.com x_suite.set("skipped", str(skipped)) 28511482Sandreas.sandberg@arm.com x_suite.set("tests", str(len(suite.results))) 28611482Sandreas.sandberg@arm.com 28711482Sandreas.sandberg@arm.com return x_suite 28811482Sandreas.sandberg@arm.com 28911482Sandreas.sandberg@arm.com def convert_suites(self, suites): 29011482Sandreas.sandberg@arm.com x_root = ET.Element("testsuites") 29111482Sandreas.sandberg@arm.com 29211482Sandreas.sandberg@arm.com for suite in suites: 29311482Sandreas.sandberg@arm.com self.convert_suite(x_root, suite) 29411482Sandreas.sandberg@arm.com 29511482Sandreas.sandberg@arm.com return x_root 29611482Sandreas.sandberg@arm.com 29711482Sandreas.sandberg@arm.com def dump_suites(self, suites): 29811482Sandreas.sandberg@arm.com et = ET.ElementTree(self.convert_suites(suites)) 29911482Sandreas.sandberg@arm.com et.write(self.fout, encoding="UTF-8") 300