results.py revision 11512:060fc1591151 
1#!/usr/bin/env python
2#
3# Copyright (c) 2016 ARM Limited
4# All rights reserved
5#
6# The license below extends only to copyright in the software and shall
7# not be construed as granting a license to any other intellectual
8# property including but not limited to intellectual property relating
9# to a hardware implementation of the functionality of the software
10# licensed hereunder.  You may use the software subject to the license
11# terms below provided that you ensure that this notice is replicated
12# unmodified and in its entirety in all distributions of the software,
13# modified or unmodified, in source code or in binary form.
14#
15# Redistribution and use in source and binary forms, with or without
16# modification, are permitted provided that the following conditions are
17# met: redistributions of source code must retain the above copyright
18# notice, this list of conditions and the following disclaimer;
19# redistributions in binary form must reproduce the above copyright
20# notice, this list of conditions and the following disclaimer in the
21# documentation and/or other materials provided with the distribution;
22# neither the name of the copyright holders nor the names of its
23# contributors may be used to endorse or promote products derived from
24# this software without specific prior written permission.
25#
26# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
27# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
28# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
29# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
30# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
31# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
32# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
33# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
34# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
35# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
36# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
37#
38# Authors: Andreas Sandberg
39
40from abc import ABCMeta, abstractmethod
41import inspect
42import pickle
43import string
44import sys
45
46import xml.etree.cElementTree as ET
47
48class UnitResult(object):
49    """Results of a single test unit.
50
51    A test result can be one of:
52        - STATE_OK: Test ran successfully.
53        - STATE_SKIPPED: The test was skipped.
54        - STATE_ERROR: The test failed to run.
55        - STATE_FAILED: Test ran, but failed.
56
57    The difference between STATE_ERROR and STATE_FAILED is very
58    subtle. In a gem5 context, STATE_ERROR would mean that gem5 failed
59    to start or crashed, while STATE_FAILED would mean that a test
60    failed (e.g., statistics mismatch).
61
62    """
63
64    STATE_OK = 0
65    STATE_SKIPPED = 1
66    STATE_ERROR = 2
67    STATE_FAILURE = 3
68
69    state_names = {
70        STATE_OK : "OK",
71        STATE_SKIPPED : "SKIPPED",
72        STATE_ERROR : "ERROR",
73        STATE_FAILURE : "FAILURE",
74    }
75
76    def __init__(self, name, state, message="", stderr="", stdout="",
77                 runtime=0.0):
78        self.name = name
79        self.state = state
80        self.message = message
81        self.stdout = stdout
82        self.stderr = stderr
83        self.runtime = runtime
84
85    def skipped(self):
86        return self.state == UnitResult.STATE_SKIPPED
87
88    def success(self):
89        return self.state == UnitResult.STATE_OK
90
91    def state_name(self):
92        return UnitResult.state_names[self.state]
93
94    def __nonzero__(self):
95        return self.success() or self.skipped()
96
97    def __str__(self):
98        state_name = self.state_name()
99
100        status = "%s: %s" % (state_name, self.message) if self.message else \
101                 state_name
102
103        return "%s: %s" % (self.name, status)
104
105class TestResult(object):
106    """Results for from a single test consisting of one or more units."""
107
108    def __init__(self, name, results=[]):
109        self.name = name
110        self.results = results
111
112    def success(self):
113        return all([ r.success() for r in self.results])
114
115    def skipped(self):
116        return all([ r.skipped() for r in self.results])
117
118    def changed(self):
119        return self.results[0].success() and self.failed()
120
121    def failed(self):
122        return any([ not r for r in self.results])
123
124    def runtime(self):
125        return sum([ r.runtime for r in self.results ])
126
127    def __nonzero__(self):
128        return all([r for r in self.results])
129
130class ResultFormatter(object):
131    __metaclass__ = ABCMeta
132
133    def __init__(self, fout=sys.stdout, verbose=False):
134        self.verbose = verbose
135        self.fout = fout
136
137    @abstractmethod
138    def dump_suites(self, suites):
139        pass
140
141class Pickle(ResultFormatter):
142    """Save test results as a binary using Python's pickle
143    functionality.
144
145    """
146
147    def __init__(self, **kwargs):
148        super(Pickle, self).__init__(**kwargs)
149
150    def dump_suites(self, suites):
151        pickle.dump(suites, self.fout, pickle.HIGHEST_PROTOCOL)
152
153class Text(ResultFormatter):
154    """Output test results as text."""
155
156    def __init__(self, **kwargs):
157        super(Text, self).__init__(**kwargs)
158
159    def dump_suites(self, suites):
160        fout = self.fout
161        for suite in suites:
162            print >> fout, "--- %s ---" % suite.name
163
164            for t in suite.results:
165                print >> fout, "*** %s" % t
166
167                if t and not self.verbose:
168                    continue
169
170                if t.message:
171                    print >> fout, t.message
172
173                if t.stderr:
174                    print >> fout, t.stderr
175                if t.stdout:
176                    print >> fout, t.stdout
177
178class TextSummary(ResultFormatter):
179    """Output test results as a text summary"""
180
181    def __init__(self, **kwargs):
182        super(TextSummary, self).__init__(**kwargs)
183
184    def test_status(self, suite):
185        if suite.skipped():
186            return "SKIPPED"
187        elif suite.changed():
188            return "CHANGED"
189        elif suite:
190            return "OK"
191        else:
192            return "FAILED"
193
194    def dump_suites(self, suites):
195        fout = self.fout
196        for suite in suites:
197            status = self.test_status(suite)
198            print >> fout, "%s: %s" % (suite.name, status)
199
200class JUnit(ResultFormatter):
201    """Output test results as JUnit XML"""
202
203    def __init__(self, translate_names=True, **kwargs):
204        super(JUnit, self).__init__(**kwargs)
205
206        if translate_names:
207            self.name_table = string.maketrans(
208                "/.",
209                ".-",
210            )
211        else:
212            self.name_table = string.maketrans("", "")
213
214    def convert_unit(self, x_suite, test):
215        x_test = ET.SubElement(x_suite, "testcase",
216                               name=test.name,
217                               time="%f" % test.runtime)
218
219        x_state = None
220        if test.state == UnitResult.STATE_OK:
221            pass
222        elif test.state == UnitResult.STATE_SKIPPED:
223            x_state = ET.SubElement(x_test, "skipped")
224        elif test.state == UnitResult.STATE_FAILURE:
225            x_state = ET.SubElement(x_test, "failure")
226        elif test.state == UnitResult.STATE_ERROR:
227            x_state = ET.SubElement(x_test, "error")
228        else:
229            assert False, "Unknown test state"
230
231        if x_state is not None:
232            if test.message:
233                x_state.set("message", test.message)
234
235            msg = []
236            if test.stderr:
237                msg.append("*** Standard Errror: ***")
238                msg.append(test.stderr)
239            if test.stdout:
240                msg.append("*** Standard Out: ***")
241                msg.append(test.stdout)
242
243            x_state.text = "\n".join(msg)
244
245        return x_test
246
247    def convert_suite(self, x_suites, suite):
248        x_suite = ET.SubElement(x_suites, "testsuite",
249                                name=suite.name.translate(self.name_table),
250                                time="%f" % suite.runtime())
251        errors = 0
252        failures = 0
253        skipped = 0
254
255        for test in suite.results:
256            if test.state != UnitResult.STATE_OK:
257                if test.state == UnitResult.STATE_SKIPPED:
258                    skipped += 1
259                elif test.state == UnitResult.STATE_ERROR:
260                    errors += 1
261                elif test.state == UnitResult.STATE_FAILURE:
262                    failures += 1
263
264            x_test = self.convert_unit(x_suite, test)
265
266        x_suite.set("errors", str(errors))
267        x_suite.set("failures", str(failures))
268        x_suite.set("skipped", str(skipped))
269        x_suite.set("tests", str(len(suite.results)))
270
271        return x_suite
272
273    def convert_suites(self, suites):
274        x_root = ET.Element("testsuites")
275
276        for suite in suites:
277            self.convert_suite(x_root, suite)
278
279        return x_root
280
281    def dump_suites(self, suites):
282        et = ET.ElementTree(self.convert_suites(suites))
283        et.write(self.fout, encoding="UTF-8")
284