1#!/usr/bin/env python2.7
2#
3# Copyright (c) 2016 ARM Limited
4# All rights reserved
5#
6# The license below extends only to copyright in the software and shall
7# not be construed as granting a license to any other intellectual
8# property including but not limited to intellectual property relating
9# to a hardware implementation of the functionality of the software
10# licensed hereunder.  You may use the software subject to the license
11# terms below provided that you ensure that this notice is replicated
12# unmodified and in its entirety in all distributions of the software,
13# modified or unmodified, in source code or in binary form.
14#
15# Redistribution and use in source and binary forms, with or without
16# modification, are permitted provided that the following conditions are
17# met: redistributions of source code must retain the above copyright
18# notice, this list of conditions and the following disclaimer;
19# redistributions in binary form must reproduce the above copyright
20# notice, this list of conditions and the following disclaimer in the
21# documentation and/or other materials provided with the distribution;
22# neither the name of the copyright holders nor the names of its
23# contributors may be used to endorse or promote products derived from
24# this software without specific prior written permission.
25#
26# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
27# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
28# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
29# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
30# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
31# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
32# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
33# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
34# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
35# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
36# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
37#
38# Authors: Andreas Sandberg
39
40from __future__ import print_function
41
42from abc import ABCMeta, abstractmethod
43import inspect
44import pickle
45import string
46import sys
47
48import xml.etree.cElementTree as ET
49
50class UnitResult(object):
51    """Results of a single test unit.
52
53    A test result can be one of:
54        - STATE_OK: Test ran successfully.
55        - STATE_SKIPPED: The test was skipped.
56        - STATE_ERROR: The test failed to run.
57        - STATE_FAILED: Test ran, but failed.
58
59    The difference between STATE_ERROR and STATE_FAILED is very
60    subtle. In a gem5 context, STATE_ERROR would mean that gem5 failed
61    to start or crashed, while STATE_FAILED would mean that a test
62    failed (e.g., statistics mismatch).
63
64    """
65
66    STATE_OK = 0
67    STATE_SKIPPED = 1
68    STATE_ERROR = 2
69    STATE_FAILURE = 3
70
71    state_names = {
72        STATE_OK : "OK",
73        STATE_SKIPPED : "SKIPPED",
74        STATE_ERROR : "ERROR",
75        STATE_FAILURE : "FAILURE",
76    }
77
78    def __init__(self, name, state, message="", stderr="", stdout="",
79                 runtime=0.0):
80        self.name = name
81        self.state = state
82        self.message = message
83        self.stdout = stdout
84        self.stderr = stderr
85        self.runtime = runtime
86
87    def skipped(self):
88        return self.state == UnitResult.STATE_SKIPPED
89
90    def success(self):
91        return self.state == UnitResult.STATE_OK
92
93    def state_name(self):
94        return UnitResult.state_names[self.state]
95
96    def __nonzero__(self):
97        return self.success() or self.skipped()
98
99    def __str__(self):
100        state_name = self.state_name()
101
102        status = "%s: %s" % (state_name, self.message) if self.message else \
103                 state_name
104
105        return "%s: %s" % (self.name, status)
106
107class TestResult(object):
108    """Results for from a single test consisting of one or more units."""
109
110    def __init__(self, name, run_results=[], verify_results=[]):
111        self.name = name
112        self.results = run_results + verify_results
113        self.run_results = run_results
114        self.verify_results = verify_results
115
116    def success(self):
117        return self.success_run() and self.success_verify()
118
119    def success_run(self):
120        return all([ r.success() for r in self.run_results ])
121
122    def success_verify(self):
123        return all([ r.success() for r in self.verify_results ])
124
125    def failed(self):
126        return self.failed_run() or self.failed_verify()
127
128    def failed_run(self):
129        return any([ not r for r in self.run_results ])
130
131    def failed_verify(self):
132        return any([ not r for r in self.verify_results ])
133
134    def skipped(self):
135        return all([ r.skipped() for r in self.run_results ])
136
137    def changed(self):
138        return self.success_run() and self.failed_verify()
139
140    def runtime(self):
141        return sum([ r.runtime for r in self.results ])
142
143    def __nonzero__(self):
144        return all([ r for r in self.results ])
145
146class ResultFormatter(object):
147    __metaclass__ = ABCMeta
148
149    def __init__(self, fout=sys.stdout, verbose=False):
150        self.verbose = verbose
151        self.fout = fout
152
153    @abstractmethod
154    def dump_suites(self, suites):
155        pass
156
157class Pickle(ResultFormatter):
158    """Save test results as a binary using Python's pickle
159    functionality.
160
161    """
162
163    def __init__(self, **kwargs):
164        super(Pickle, self).__init__(**kwargs)
165
166    def dump_suites(self, suites):
167        pickle.dump(suites, self.fout, pickle.HIGHEST_PROTOCOL)
168
169class Text(ResultFormatter):
170    """Output test results as text."""
171
172    def __init__(self, **kwargs):
173        super(Text, self).__init__(**kwargs)
174
175    def dump_suites(self, suites):
176        fout = self.fout
177        for suite in suites:
178            print("--- %s ---" % suite.name, file=fout)
179
180            for t in suite.results:
181                print("*** %s" % t, file=fout)
182
183                if t and not self.verbose:
184                    continue
185
186                if t.message:
187                    print(t.message, file=fout)
188
189                if t.stderr:
190                    print(t.stderr, file=fout)
191                if t.stdout:
192                    print(t.stdout, file=fout)
193
194class TextSummary(ResultFormatter):
195    """Output test results as a text summary"""
196
197    def __init__(self, **kwargs):
198        super(TextSummary, self).__init__(**kwargs)
199
200    def test_status(self, suite):
201        if suite.skipped():
202            return "SKIPPED"
203        elif suite.changed():
204            return "CHANGED"
205        elif suite:
206            return "OK"
207        else:
208            return "FAILED"
209
210    def dump_suites(self, suites):
211        fout = self.fout
212        for suite in suites:
213            status = self.test_status(suite)
214            print("%s: %s" % (suite.name, status), file=fout)
215
216class JUnit(ResultFormatter):
217    """Output test results as JUnit XML"""
218
219    def __init__(self, translate_names=True, **kwargs):
220        super(JUnit, self).__init__(**kwargs)
221
222        if translate_names:
223            self.name_table = string.maketrans(
224                "/.",
225                ".-",
226            )
227        else:
228            self.name_table = string.maketrans("", "")
229
230    def convert_unit(self, x_suite, test):
231        x_test = ET.SubElement(x_suite, "testcase",
232                               name=test.name,
233                               time="%f" % test.runtime)
234
235        x_state = None
236        if test.state == UnitResult.STATE_OK:
237            pass
238        elif test.state == UnitResult.STATE_SKIPPED:
239            x_state = ET.SubElement(x_test, "skipped")
240        elif test.state == UnitResult.STATE_FAILURE:
241            x_state = ET.SubElement(x_test, "failure")
242        elif test.state == UnitResult.STATE_ERROR:
243            x_state = ET.SubElement(x_test, "error")
244        else:
245            assert False, "Unknown test state"
246
247        if x_state is not None:
248            if test.message:
249                x_state.set("message", test.message)
250
251            msg = []
252            if test.stderr:
253                msg.append("*** Standard Errror: ***")
254                msg.append(test.stderr)
255            if test.stdout:
256                msg.append("*** Standard Out: ***")
257                msg.append(test.stdout)
258
259            x_state.text = "\n".join(msg)
260
261        return x_test
262
263    def convert_suite(self, x_suites, suite):
264        x_suite = ET.SubElement(x_suites, "testsuite",
265                                name=suite.name.translate(self.name_table),
266                                time="%f" % suite.runtime())
267        errors = 0
268        failures = 0
269        skipped = 0
270
271        for test in suite.results:
272            if test.state != UnitResult.STATE_OK:
273                if test.state == UnitResult.STATE_SKIPPED:
274                    skipped += 1
275                elif test.state == UnitResult.STATE_ERROR:
276                    errors += 1
277                elif test.state == UnitResult.STATE_FAILURE:
278                    failures += 1
279
280            x_test = self.convert_unit(x_suite, test)
281
282        x_suite.set("errors", str(errors))
283        x_suite.set("failures", str(failures))
284        x_suite.set("skipped", str(skipped))
285        x_suite.set("tests", str(len(suite.results)))
286
287        return x_suite
288
289    def convert_suites(self, suites):
290        x_root = ET.Element("testsuites")
291
292        for suite in suites:
293            self.convert_suite(x_root, suite)
294
295        return x_root
296
297    def dump_suites(self, suites):
298        et = ET.ElementTree(self.convert_suites(suites))
299        et.write(self.fout, encoding="UTF-8")
300