1#!/usr/bin/env python2
2#
3# Copyright (c) 2016 ARM Limited
4# All rights reserved
5#
6# The license below extends only to copyright in the software and shall
7# not be construed as granting a license to any other intellectual
8# property including but not limited to intellectual property relating
9# to a hardware implementation of the functionality of the software
10# licensed hereunder. You may use the software subject to the license
11# terms below provided that you ensure that this notice is replicated
12# unmodified and in its entirety in all distributions of the software,
13# modified or unmodified, in source code or in binary form.
14#
15# Redistribution and use in source and binary forms, with or without
16# modification, are permitted provided that the following conditions are
17# met: redistributions of source code must retain the above copyright
18# notice, this list of conditions and the following disclaimer;
19# redistributions in binary form must reproduce the above copyright
20# notice, this list of conditions and the following disclaimer in the
21# documentation and/or other materials provided with the distribution;
22# neither the name of the copyright holders nor the names of its
23# contributors may be used to endorse or promote products derived from
24# this software without specific prior written permission.
25#
26# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
27# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
28# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
29# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
30# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
31# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
32# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
33# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
34# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
35# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
36# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
37#
38# Authors: Andreas Sandberg
39
| 1#!/usr/bin/env python2
2#
3# Copyright (c) 2016 ARM Limited
4# All rights reserved
5#
6# The license below extends only to copyright in the software and shall
7# not be construed as granting a license to any other intellectual
8# property including but not limited to intellectual property relating
9# to a hardware implementation of the functionality of the software
10# licensed hereunder. You may use the software subject to the license
11# terms below provided that you ensure that this notice is replicated
12# unmodified and in its entirety in all distributions of the software,
13# modified or unmodified, in source code or in binary form.
14#
15# Redistribution and use in source and binary forms, with or without
16# modification, are permitted provided that the following conditions are
17# met: redistributions of source code must retain the above copyright
18# notice, this list of conditions and the following disclaimer;
19# redistributions in binary form must reproduce the above copyright
20# notice, this list of conditions and the following disclaimer in the
21# documentation and/or other materials provided with the distribution;
22# neither the name of the copyright holders nor the names of its
23# contributors may be used to endorse or promote products derived from
24# this software without specific prior written permission.
25#
26# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
27# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
28# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
29# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
30# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
31# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
32# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
33# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
34# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
35# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
36# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
37#
38# Authors: Andreas Sandberg
39
|
| 40from __future__ import print_function
41
|
40from abc import ABCMeta, abstractmethod
41import inspect
42import pickle
43import string
44import sys
45
46import xml.etree.cElementTree as ET
47
48class UnitResult(object):
49 """Results of a single test unit.
50
51 A test result can be one of:
52 - STATE_OK: Test ran successfully.
53 - STATE_SKIPPED: The test was skipped.
54 - STATE_ERROR: The test failed to run.
55 - STATE_FAILED: Test ran, but failed.
56
57 The difference between STATE_ERROR and STATE_FAILED is very
58 subtle. In a gem5 context, STATE_ERROR would mean that gem5 failed
59 to start or crashed, while STATE_FAILED would mean that a test
60 failed (e.g., statistics mismatch).
61
62 """
63
64 STATE_OK = 0
65 STATE_SKIPPED = 1
66 STATE_ERROR = 2
67 STATE_FAILURE = 3
68
69 state_names = {
70 STATE_OK : "OK",
71 STATE_SKIPPED : "SKIPPED",
72 STATE_ERROR : "ERROR",
73 STATE_FAILURE : "FAILURE",
74 }
75
76 def __init__(self, name, state, message="", stderr="", stdout="",
77 runtime=0.0):
78 self.name = name
79 self.state = state
80 self.message = message
81 self.stdout = stdout
82 self.stderr = stderr
83 self.runtime = runtime
84
85 def skipped(self):
86 return self.state == UnitResult.STATE_SKIPPED
87
88 def success(self):
89 return self.state == UnitResult.STATE_OK
90
91 def state_name(self):
92 return UnitResult.state_names[self.state]
93
94 def __nonzero__(self):
95 return self.success() or self.skipped()
96
97 def __str__(self):
98 state_name = self.state_name()
99
100 status = "%s: %s" % (state_name, self.message) if self.message else \
101 state_name
102
103 return "%s: %s" % (self.name, status)
104
105class TestResult(object):
106 """Results for from a single test consisting of one or more units."""
107
108 def __init__(self, name, run_results=[], verify_results=[]):
109 self.name = name
110 self.results = run_results + verify_results
111 self.run_results = run_results
112 self.verify_results = verify_results
113
114 def success(self):
115 return self.success_run() and self.success_verify()
116
117 def success_run(self):
118 return all([ r.success() for r in self.run_results ])
119
120 def success_verify(self):
121 return all([ r.success() for r in self.verify_results ])
122
123 def failed(self):
124 return self.failed_run() or self.failed_verify()
125
126 def failed_run(self):
127 return any([ not r for r in self.run_results ])
128
129 def failed_verify(self):
130 return any([ not r for r in self.verify_results ])
131
132 def skipped(self):
133 return all([ r.skipped() for r in self.run_results ])
134
135 def changed(self):
136 return self.success_run() and self.failed_verify()
137
138 def runtime(self):
139 return sum([ r.runtime for r in self.results ])
140
141 def __nonzero__(self):
142 return all([ r for r in self.results ])
143
144class ResultFormatter(object):
145 __metaclass__ = ABCMeta
146
147 def __init__(self, fout=sys.stdout, verbose=False):
148 self.verbose = verbose
149 self.fout = fout
150
151 @abstractmethod
152 def dump_suites(self, suites):
153 pass
154
155class Pickle(ResultFormatter):
156 """Save test results as a binary using Python's pickle
157 functionality.
158
159 """
160
161 def __init__(self, **kwargs):
162 super(Pickle, self).__init__(**kwargs)
163
164 def dump_suites(self, suites):
165 pickle.dump(suites, self.fout, pickle.HIGHEST_PROTOCOL)
166
167class Text(ResultFormatter):
168 """Output test results as text."""
169
170 def __init__(self, **kwargs):
171 super(Text, self).__init__(**kwargs)
172
173 def dump_suites(self, suites):
174 fout = self.fout
175 for suite in suites:
| 42from abc import ABCMeta, abstractmethod
43import inspect
44import pickle
45import string
46import sys
47
48import xml.etree.cElementTree as ET
49
50class UnitResult(object):
51 """Results of a single test unit.
52
53 A test result can be one of:
54 - STATE_OK: Test ran successfully.
55 - STATE_SKIPPED: The test was skipped.
56 - STATE_ERROR: The test failed to run.
57 - STATE_FAILED: Test ran, but failed.
58
59 The difference between STATE_ERROR and STATE_FAILED is very
60 subtle. In a gem5 context, STATE_ERROR would mean that gem5 failed
61 to start or crashed, while STATE_FAILED would mean that a test
62 failed (e.g., statistics mismatch).
63
64 """
65
66 STATE_OK = 0
67 STATE_SKIPPED = 1
68 STATE_ERROR = 2
69 STATE_FAILURE = 3
70
71 state_names = {
72 STATE_OK : "OK",
73 STATE_SKIPPED : "SKIPPED",
74 STATE_ERROR : "ERROR",
75 STATE_FAILURE : "FAILURE",
76 }
77
78 def __init__(self, name, state, message="", stderr="", stdout="",
79 runtime=0.0):
80 self.name = name
81 self.state = state
82 self.message = message
83 self.stdout = stdout
84 self.stderr = stderr
85 self.runtime = runtime
86
87 def skipped(self):
88 return self.state == UnitResult.STATE_SKIPPED
89
90 def success(self):
91 return self.state == UnitResult.STATE_OK
92
93 def state_name(self):
94 return UnitResult.state_names[self.state]
95
96 def __nonzero__(self):
97 return self.success() or self.skipped()
98
99 def __str__(self):
100 state_name = self.state_name()
101
102 status = "%s: %s" % (state_name, self.message) if self.message else \
103 state_name
104
105 return "%s: %s" % (self.name, status)
106
107class TestResult(object):
108 """Results for from a single test consisting of one or more units."""
109
110 def __init__(self, name, run_results=[], verify_results=[]):
111 self.name = name
112 self.results = run_results + verify_results
113 self.run_results = run_results
114 self.verify_results = verify_results
115
116 def success(self):
117 return self.success_run() and self.success_verify()
118
119 def success_run(self):
120 return all([ r.success() for r in self.run_results ])
121
122 def success_verify(self):
123 return all([ r.success() for r in self.verify_results ])
124
125 def failed(self):
126 return self.failed_run() or self.failed_verify()
127
128 def failed_run(self):
129 return any([ not r for r in self.run_results ])
130
131 def failed_verify(self):
132 return any([ not r for r in self.verify_results ])
133
134 def skipped(self):
135 return all([ r.skipped() for r in self.run_results ])
136
137 def changed(self):
138 return self.success_run() and self.failed_verify()
139
140 def runtime(self):
141 return sum([ r.runtime for r in self.results ])
142
143 def __nonzero__(self):
144 return all([ r for r in self.results ])
145
146class ResultFormatter(object):
147 __metaclass__ = ABCMeta
148
149 def __init__(self, fout=sys.stdout, verbose=False):
150 self.verbose = verbose
151 self.fout = fout
152
153 @abstractmethod
154 def dump_suites(self, suites):
155 pass
156
157class Pickle(ResultFormatter):
158 """Save test results as a binary using Python's pickle
159 functionality.
160
161 """
162
163 def __init__(self, **kwargs):
164 super(Pickle, self).__init__(**kwargs)
165
166 def dump_suites(self, suites):
167 pickle.dump(suites, self.fout, pickle.HIGHEST_PROTOCOL)
168
169class Text(ResultFormatter):
170 """Output test results as text."""
171
172 def __init__(self, **kwargs):
173 super(Text, self).__init__(**kwargs)
174
175 def dump_suites(self, suites):
176 fout = self.fout
177 for suite in suites:
|
176 print >> fout, "--- %s ---" % suite.name
| 178 print("--- %s ---" % suite.name, file=fout)
|
177
178 for t in suite.results:
| 179
180 for t in suite.results:
|
179 print >> fout, "*** %s" % t
| 181 print("*** %s" % t, file=fout)
|
180
181 if t and not self.verbose:
182 continue
183
184 if t.message:
| 182
183 if t and not self.verbose:
184 continue
185
186 if t.message:
|
185 print >> fout, t.message
| 187 print(t.message, file=fout)
|
186
187 if t.stderr:
| 188
189 if t.stderr:
|
188 print >> fout, t.stderr
| 190 print(t.stderr, file=fout)
|
189 if t.stdout:
| 191 if t.stdout:
|
190 print >> fout, t.stdout
| 192 print(t.stdout, file=fout)
|
191
192class TextSummary(ResultFormatter):
193 """Output test results as a text summary"""
194
195 def __init__(self, **kwargs):
196 super(TextSummary, self).__init__(**kwargs)
197
198 def test_status(self, suite):
199 if suite.skipped():
200 return "SKIPPED"
201 elif suite.changed():
202 return "CHANGED"
203 elif suite:
204 return "OK"
205 else:
206 return "FAILED"
207
208 def dump_suites(self, suites):
209 fout = self.fout
210 for suite in suites:
211 status = self.test_status(suite)
| 193
194class TextSummary(ResultFormatter):
195 """Output test results as a text summary"""
196
197 def __init__(self, **kwargs):
198 super(TextSummary, self).__init__(**kwargs)
199
200 def test_status(self, suite):
201 if suite.skipped():
202 return "SKIPPED"
203 elif suite.changed():
204 return "CHANGED"
205 elif suite:
206 return "OK"
207 else:
208 return "FAILED"
209
210 def dump_suites(self, suites):
211 fout = self.fout
212 for suite in suites:
213 status = self.test_status(suite)
|
212 print >> fout, "%s: %s" % (suite.name, status)
| 214 print("%s: %s" % (suite.name, status), file=fout)
|
213
214class JUnit(ResultFormatter):
215 """Output test results as JUnit XML"""
216
217 def __init__(self, translate_names=True, **kwargs):
218 super(JUnit, self).__init__(**kwargs)
219
220 if translate_names:
221 self.name_table = string.maketrans(
222 "/.",
223 ".-",
224 )
225 else:
226 self.name_table = string.maketrans("", "")
227
228 def convert_unit(self, x_suite, test):
229 x_test = ET.SubElement(x_suite, "testcase",
230 name=test.name,
231 time="%f" % test.runtime)
232
233 x_state = None
234 if test.state == UnitResult.STATE_OK:
235 pass
236 elif test.state == UnitResult.STATE_SKIPPED:
237 x_state = ET.SubElement(x_test, "skipped")
238 elif test.state == UnitResult.STATE_FAILURE:
239 x_state = ET.SubElement(x_test, "failure")
240 elif test.state == UnitResult.STATE_ERROR:
241 x_state = ET.SubElement(x_test, "error")
242 else:
243 assert False, "Unknown test state"
244
245 if x_state is not None:
246 if test.message:
247 x_state.set("message", test.message)
248
249 msg = []
250 if test.stderr:
251 msg.append("*** Standard Errror: ***")
252 msg.append(test.stderr)
253 if test.stdout:
254 msg.append("*** Standard Out: ***")
255 msg.append(test.stdout)
256
257 x_state.text = "\n".join(msg)
258
259 return x_test
260
261 def convert_suite(self, x_suites, suite):
262 x_suite = ET.SubElement(x_suites, "testsuite",
263 name=suite.name.translate(self.name_table),
264 time="%f" % suite.runtime())
265 errors = 0
266 failures = 0
267 skipped = 0
268
269 for test in suite.results:
270 if test.state != UnitResult.STATE_OK:
271 if test.state == UnitResult.STATE_SKIPPED:
272 skipped += 1
273 elif test.state == UnitResult.STATE_ERROR:
274 errors += 1
275 elif test.state == UnitResult.STATE_FAILURE:
276 failures += 1
277
278 x_test = self.convert_unit(x_suite, test)
279
280 x_suite.set("errors", str(errors))
281 x_suite.set("failures", str(failures))
282 x_suite.set("skipped", str(skipped))
283 x_suite.set("tests", str(len(suite.results)))
284
285 return x_suite
286
287 def convert_suites(self, suites):
288 x_root = ET.Element("testsuites")
289
290 for suite in suites:
291 self.convert_suite(x_root, suite)
292
293 return x_root
294
295 def dump_suites(self, suites):
296 et = ET.ElementTree(self.convert_suites(suites))
297 et.write(self.fout, encoding="UTF-8")
| 215
216class JUnit(ResultFormatter):
217 """Output test results as JUnit XML"""
218
219 def __init__(self, translate_names=True, **kwargs):
220 super(JUnit, self).__init__(**kwargs)
221
222 if translate_names:
223 self.name_table = string.maketrans(
224 "/.",
225 ".-",
226 )
227 else:
228 self.name_table = string.maketrans("", "")
229
230 def convert_unit(self, x_suite, test):
231 x_test = ET.SubElement(x_suite, "testcase",
232 name=test.name,
233 time="%f" % test.runtime)
234
235 x_state = None
236 if test.state == UnitResult.STATE_OK:
237 pass
238 elif test.state == UnitResult.STATE_SKIPPED:
239 x_state = ET.SubElement(x_test, "skipped")
240 elif test.state == UnitResult.STATE_FAILURE:
241 x_state = ET.SubElement(x_test, "failure")
242 elif test.state == UnitResult.STATE_ERROR:
243 x_state = ET.SubElement(x_test, "error")
244 else:
245 assert False, "Unknown test state"
246
247 if x_state is not None:
248 if test.message:
249 x_state.set("message", test.message)
250
251 msg = []
252 if test.stderr:
253 msg.append("*** Standard Errror: ***")
254 msg.append(test.stderr)
255 if test.stdout:
256 msg.append("*** Standard Out: ***")
257 msg.append(test.stdout)
258
259 x_state.text = "\n".join(msg)
260
261 return x_test
262
263 def convert_suite(self, x_suites, suite):
264 x_suite = ET.SubElement(x_suites, "testsuite",
265 name=suite.name.translate(self.name_table),
266 time="%f" % suite.runtime())
267 errors = 0
268 failures = 0
269 skipped = 0
270
271 for test in suite.results:
272 if test.state != UnitResult.STATE_OK:
273 if test.state == UnitResult.STATE_SKIPPED:
274 skipped += 1
275 elif test.state == UnitResult.STATE_ERROR:
276 errors += 1
277 elif test.state == UnitResult.STATE_FAILURE:
278 failures += 1
279
280 x_test = self.convert_unit(x_suite, test)
281
282 x_suite.set("errors", str(errors))
283 x_suite.set("failures", str(failures))
284 x_suite.set("skipped", str(skipped))
285 x_suite.set("tests", str(len(suite.results)))
286
287 return x_suite
288
289 def convert_suites(self, suites):
290 x_root = ET.Element("testsuites")
291
292 for suite in suites:
293 self.convert_suite(x_root, suite)
294
295 return x_root
296
297 def dump_suites(self, suites):
298 et = ET.ElementTree(self.convert_suites(suites))
299 et.write(self.fout, encoding="UTF-8")
|