results.py revision 11482
11689SN/A#!/usr/bin/env python 29444SAndreas.Sandberg@ARM.com# 39913Ssteve.reinhardt@amd.com# Copyright (c) 2016 ARM Limited 47854SAli.Saidi@ARM.com# All rights reserved 57854SAli.Saidi@ARM.com# 67854SAli.Saidi@ARM.com# The license below extends only to copyright in the software and shall 77854SAli.Saidi@ARM.com# not be construed as granting a license to any other intellectual 87854SAli.Saidi@ARM.com# property including but not limited to intellectual property relating 97854SAli.Saidi@ARM.com# to a hardware implementation of the functionality of the software 107854SAli.Saidi@ARM.com# licensed hereunder. You may use the software subject to the license 117854SAli.Saidi@ARM.com# terms below provided that you ensure that this notice is replicated 127854SAli.Saidi@ARM.com# unmodified and in its entirety in all distributions of the software, 137854SAli.Saidi@ARM.com# modified or unmodified, in source code or in binary form. 147854SAli.Saidi@ARM.com# 152329SN/A# Redistribution and use in source and binary forms, with or without 161689SN/A# modification, are permitted provided that the following conditions are 171689SN/A# met: redistributions of source code must retain the above copyright 181689SN/A# notice, this list of conditions and the following disclaimer; 191689SN/A# redistributions in binary form must reproduce the above copyright 201689SN/A# notice, this list of conditions and the following disclaimer in the 211689SN/A# documentation and/or other materials provided with the distribution; 221689SN/A# neither the name of the copyright holders nor the names of its 231689SN/A# contributors may be used to endorse or promote products derived from 241689SN/A# this software without specific prior written permission. 251689SN/A# 261689SN/A# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS 271689SN/A# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT 281689SN/A# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR 291689SN/A# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT 301689SN/A# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, 311689SN/A# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT 321689SN/A# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, 331689SN/A# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY 341689SN/A# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT 351689SN/A# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE 361689SN/A# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 371689SN/A# 381689SN/A# Authors: Andreas Sandberg 391689SN/A 402665Ssaidi@eecs.umich.edufrom abc import ABCMeta, abstractmethod 412665Ssaidi@eecs.umich.eduimport inspect 422935Sksewell@umich.eduimport pickle 431689SN/Aimport string 441689SN/Aimport sys 459944Smatt.horsnell@ARM.com 469944Smatt.horsnell@ARM.comimport xml.etree.cElementTree as ET 479944Smatt.horsnell@ARM.com 481060SN/Aclass UnitResult(object): 491060SN/A """Results of a single test unit. 503773Sgblack@eecs.umich.edu 516329Sgblack@eecs.umich.edu A test result can be one of: 526658Snate@binkert.org - STATE_OK: Test ran successfully. 531717SN/A - STATE_SKIPPED: The test was skipped. 549913Ssteve.reinhardt@amd.com - STATE_ERROR: The test failed to run. 558232Snate@binkert.org - STATE_FAILED: Test ran, but failed. 568232Snate@binkert.org 579527SMatt.Horsnell@arm.com The difference between STATE_ERROR and STATE_FAILED is very 585529Snate@binkert.org subtle. In a gem5 context, STATE_ERROR would mean that gem5 failed 591060SN/A to start or crashed, while STATE_FAILED would mean that a test 606221Snate@binkert.org failed (e.g., statistics mismatch). 616221Snate@binkert.org 621061SN/A """ 635529Snate@binkert.org 644329Sktlim@umich.edu STATE_OK = 0 654329Sktlim@umich.edu STATE_SKIPPED = 1 662292SN/A STATE_ERROR = 2 672292SN/A STATE_FAILURE = 3 682292SN/A 692292SN/A state_names = { 705529Snate@binkert.org STATE_OK : "OK", 719920Syasuko.eckert@amd.com STATE_SKIPPED : "SKIPPED", 729920Syasuko.eckert@amd.com STATE_ERROR : "ERROR", 731060SN/A STATE_FAILURE : "FAILURE", 7410172Sdam.sunwoo@arm.com } 7510172Sdam.sunwoo@arm.com 7610172Sdam.sunwoo@arm.com def __init__(self, name, state, message="", stderr="", stdout="", 7710172Sdam.sunwoo@arm.com runtime=0.0): 7810172Sdam.sunwoo@arm.com self.name = name 792292SN/A self.state = state 808907Slukefahr@umich.edu self.message = message 812292SN/A self.stdout = stdout 822292SN/A self.stderr = stderr 832292SN/A self.runtime = runtime 842292SN/A 852292SN/A def skipped(self): 862292SN/A return self.state == UnitResult.STATE_SKIPPED 872292SN/A 881060SN/A def success(self): 891060SN/A return self.state == UnitResult.STATE_OK 901061SN/A 911060SN/A def state_name(self): 922292SN/A return UnitResult.state_names[self.state] 931062SN/A 941062SN/A def __nonzero__(self): 958240Snate@binkert.org return self.success() or self.skipped() 961062SN/A 971062SN/A def __str__(self): 981062SN/A state_name = self.state_name() 998240Snate@binkert.org 1001062SN/A status = "%s: %s" % (state_name, self.message) if self.message else \ 1011062SN/A state_name 1021062SN/A 1038240Snate@binkert.org return "%s: %s" % (self.name, status) 1041062SN/A 1051062SN/Aclass TestResult(object): 1062301SN/A """Results for from a single test consisting of one or more units.""" 1078240Snate@binkert.org 1082301SN/A def __init__(self, name, results=[]): 1092301SN/A self.name = name 1102292SN/A self.results = results 1118240Snate@binkert.org 1122292SN/A def success(self): 1132292SN/A return all([ r.success() for r in self.results]) 1141062SN/A 1158240Snate@binkert.org def skipped(self): 1161062SN/A return all([ r.skipped() for r in self.results]) 1171062SN/A 1181062SN/A def failed(self): 1198240Snate@binkert.org return any([ not r for r in self.results]) 1201062SN/A 1211062SN/A def runtime(self): 1221062SN/A return sum([ r.runtime for r in self.results ]) 1238240Snate@binkert.org 1241062SN/A def __nonzero__(self): 1251062SN/A return all([r for r in self.results]) 1261062SN/A 1278240Snate@binkert.orgclass ResultFormatter(object): 1282292SN/A __metaclass__ = ABCMeta 1291062SN/A 1301062SN/A def __init__(self, fout=sys.stdout, verbose=False): 1318240Snate@binkert.org self.verbose = verbose 1322292SN/A self.fout = fout 1331062SN/A 1342292SN/A @abstractmethod 1358240Snate@binkert.org def dump_suites(self, suites): 1362292SN/A pass 1372292SN/A 1381062SN/Aclass Pickle(ResultFormatter): 1398240Snate@binkert.org """Save test results as a binary using Python's pickle 1401062SN/A functionality. 1411062SN/A 1421062SN/A """ 1438240Snate@binkert.org 1441062SN/A def __init__(self, **kwargs): 1451062SN/A super(Pickle, self).__init__(**kwargs) 1461062SN/A 1478240Snate@binkert.org def dump_suites(self, suites): 1481062SN/A pickle.dump(suites, self.fout, pickle.HIGHEST_PROTOCOL) 1491062SN/A 1501062SN/Aclass Text(ResultFormatter): 1518240Snate@binkert.org """Output test results as text.""" 1521062SN/A 1531062SN/A def __init__(self, **kwargs): 1541062SN/A super(Text, self).__init__(**kwargs) 1558240Snate@binkert.org 1561062SN/A def dump_suites(self, suites): 1571062SN/A fout = self.fout 1582301SN/A for suite in suites: 1598240Snate@binkert.org print >> fout, "--- %s ---" % suite.name 1602301SN/A 1612301SN/A for t in suite.results: 1622301SN/A print >> fout, "*** %s" % t 1632301SN/A 1648240Snate@binkert.org if t and not self.verbose: 1652301SN/A continue 1662301SN/A 1672301SN/A if t.message: 1682307SN/A print >> fout, t.message 1698240Snate@binkert.org 1702307SN/A if t.stderr: 1712307SN/A print >> fout, t.stderr 1722307SN/A if t.stdout: 1737897Shestness@cs.utexas.edu print >> fout, t.stdout 1748240Snate@binkert.org 1757897Shestness@cs.utexas.educlass TextSummary(ResultFormatter): 1767897Shestness@cs.utexas.edu """Output test results as a text summary""" 1777897Shestness@cs.utexas.edu 1788240Snate@binkert.org def __init__(self, **kwargs): 1797897Shestness@cs.utexas.edu super(TextSummary, self).__init__(**kwargs) 1807897Shestness@cs.utexas.edu 1811062SN/A def dump_suites(self, suites): 1821062SN/A fout = self.fout 1831062SN/A for suite in suites: 1841062SN/A status = "SKIPPED" if suite.skipped() else \ 1852292SN/A ("OK" if suite else "FAILED") 1861060SN/A print >> fout, "%s: %s" % (suite.name, status) 1871060SN/A 1881060SN/Aclass JUnit(ResultFormatter): 1891060SN/A """Output test results as JUnit XML""" 1901060SN/A 1911060SN/A def __init__(self, translate_names=True, **kwargs): 1921060SN/A super(JUnit, self).__init__(**kwargs) 1931060SN/A 1941060SN/A if translate_names: 1951060SN/A self.name_table = string.maketrans( 1961060SN/A "/.", 1971060SN/A ".-", 1981060SN/A ) 1991061SN/A else: 2001060SN/A self.name_table = string.maketrans("", "") 2012292SN/A 2021060SN/A def convert_unit(self, x_suite, test): 2031060SN/A x_test = ET.SubElement(x_suite, "testcase", 2041060SN/A name=test.name, 2051060SN/A time="%f" % test.runtime) 2061060SN/A 2071060SN/A x_state = None 2081060SN/A if test.state == UnitResult.STATE_OK: 2091061SN/A pass 2101060SN/A elif test.state == UnitResult.STATE_SKIPPED: 2112292SN/A x_state = ET.SubElement(x_test, "skipped") 2121060SN/A elif test.state == UnitResult.STATE_FAILURE: 2131060SN/A x_state = ET.SubElement(x_test, "failure") 2141060SN/A elif test.state == UnitResult.STATE_ERROR: 2151060SN/A x_state = ET.SubElement(x_test, "error") 2161060SN/A else: 2171060SN/A assert False, "Unknown test state" 2181060SN/A 2191061SN/A if x_state is not None: 2201060SN/A if test.message: 2219427SAndreas.Sandberg@ARM.com x_state.set("message", test.message) 2221060SN/A 2239444SAndreas.Sandberg@ARM.com msg = [] 2249444SAndreas.Sandberg@ARM.com if test.stderr: 2259444SAndreas.Sandberg@ARM.com msg.append("*** Standard Errror: ***") 2269444SAndreas.Sandberg@ARM.com msg.append(test.stderr) 2279444SAndreas.Sandberg@ARM.com if test.stdout: 2289444SAndreas.Sandberg@ARM.com msg.append("*** Standard Out: ***") 2299444SAndreas.Sandberg@ARM.com msg.append(test.stdout) 2309444SAndreas.Sandberg@ARM.com 2319444SAndreas.Sandberg@ARM.com x_state.text = "\n".join(msg) 2329444SAndreas.Sandberg@ARM.com 2339444SAndreas.Sandberg@ARM.com return x_test 2349444SAndreas.Sandberg@ARM.com 2352329SN/A def convert_suite(self, x_suites, suite): 2366221Snate@binkert.org x_suite = ET.SubElement(x_suites, "testsuite", 2379444SAndreas.Sandberg@ARM.com name=suite.name.translate(self.name_table), 2389444SAndreas.Sandberg@ARM.com time="%f" % suite.runtime()) 2392292SN/A errors = 0 2402292SN/A failures = 0 2412292SN/A skipped = 0 2422292SN/A 2439444SAndreas.Sandberg@ARM.com for test in suite.results: 2449444SAndreas.Sandberg@ARM.com if test.state != UnitResult.STATE_OK: 2459444SAndreas.Sandberg@ARM.com if test.state == UnitResult.STATE_SKIPPED: 2469444SAndreas.Sandberg@ARM.com skipped += 1 2479444SAndreas.Sandberg@ARM.com elif test.state == UnitResult.STATE_ERROR: 2489444SAndreas.Sandberg@ARM.com errors += 1 2499444SAndreas.Sandberg@ARM.com elif test.state == UnitResult.STATE_FAILURE: 2509444SAndreas.Sandberg@ARM.com failures += 1 2512292SN/A 2521060SN/A x_test = self.convert_unit(x_suite, test) 2531060SN/A 2542292SN/A x_suite.set("errors", str(errors)) 2552292SN/A x_suite.set("failures", str(failures)) 2566221Snate@binkert.org x_suite.set("skipped", str(skipped)) 2572292SN/A x_suite.set("tests", str(len(suite.results))) 2582292SN/A 2592292SN/A return x_suite 2602292SN/A 2612292SN/A def convert_suites(self, suites): 2621061SN/A x_root = ET.Element("testsuites") 2631060SN/A 2642292SN/A for suite in suites: 2651060SN/A self.convert_suite(x_root, suite) 2666221Snate@binkert.org 2676221Snate@binkert.org return x_root 2681060SN/A 2691060SN/A def dump_suites(self, suites): 2701061SN/A et = ET.ElementTree(self.convert_suites(suites)) 2711060SN/A et.write(self.fout, encoding="UTF-8") 2722292SN/A