1#!/usr/bin/env python2
2#
3# Copyright (c) 2016 ARM Limited
4# All rights reserved
5#
6# The license below extends only to copyright in the software and shall
7# not be construed as granting a license to any other intellectual
8# property including but not limited to intellectual property relating
9# to a hardware implementation of the functionality of the software
10# licensed hereunder. You may use the software subject to the license
11# terms below provided that you ensure that this notice is replicated
12# unmodified and in its entirety in all distributions of the software,
13# modified or unmodified, in source code or in binary form.
14#
15# Redistribution and use in source and binary forms, with or without
16# modification, are permitted provided that the following conditions are
17# met: redistributions of source code must retain the above copyright
18# notice, this list of conditions and the following disclaimer;
19# redistributions in binary form must reproduce the above copyright
20# notice, this list of conditions and the following disclaimer in the
21# documentation and/or other materials provided with the distribution;
22# neither the name of the copyright holders nor the names of its
23# contributors may be used to endorse or promote products derived from
24# this software without specific prior written permission.
25#
26# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
27# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
28# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
29# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
30# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
31# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
32# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
33# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
34# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
35# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
36# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
37#
38# Authors: Andreas Sandberg
39
40from abc import ABCMeta, abstractmethod
41from datetime import datetime
42import difflib
43import functools
44import os
45import re
46import subprocess
47import sys
48import traceback
49
50from results import UnitResult
51from helpers import *
52
53_test_base = os.path.join(os.path.dirname(__file__), "..")
54
55class TestUnit(object):
56 """Base class for all test units.
57
58 A test unit is a part of a larger test case. Test cases usually
59 contain two types of units, run units (run gem5) and verify units
60 (diff output files). All unit implementations inherit from this
61 class.
62
63 A unit implementation overrides the _run() method. The test runner
64 calls the run() method, which wraps _run() to protect against
65 exceptions.
66
67 """
68
69 __metaclass__ = ABCMeta
70
71 def __init__(self, name, ref_dir, test_dir, skip=False):
72 self.name = name
73 self.ref_dir = ref_dir
74 self.test_dir = test_dir
75 self.force_skip = skip
76 self.start_time = None
77 self.stop_time = None
78
79 def result(self, state, **kwargs):
80 if self.start_time is not None and "runtime" not in kwargs:
81 self.stop_time = datetime.utcnow()
82 delta = self.stop_time - self.start_time
83 kwargs["runtime"] = delta.total_seconds()
84
85 return UnitResult(self.name, state, **kwargs)
86
87 def ok(self, **kwargs):
88 return self.result(UnitResult.STATE_OK, **kwargs)
89
90 def skip(self, **kwargs):
91 return self.result(UnitResult.STATE_SKIPPED, **kwargs)
92
93 def error(self, message, **kwargs):
94 return self.result(UnitResult.STATE_ERROR, message=message, **kwargs)
95
96 def failure(self, message, **kwargs):
97 return self.result(UnitResult.STATE_FAILURE, message=message, **kwargs)
98
99 def ref_file(self, fname):
100 return os.path.join(self.ref_dir, fname)
101
102 def out_file(self, fname):
103 return os.path.join(self.test_dir, fname)
104
105 def _read_output(self, fname, default=""):
106 try:
107 with open(self.out_file(fname), "r") as f:
108 return f.read()
109 except IOError:
110 return default
111
112 def run(self):
113 self.start_time = datetime.utcnow()
114 try:
115 if self.force_skip:
116 return self.skip()
117 else:
118 return self._run()
119 except:
120 return self.error("Python exception:\n%s" % traceback.format_exc())
121
122 @abstractmethod
123 def _run(self):
124 pass
125
126class RunGem5(TestUnit):
127 """Test unit representing a gem5 run.
128
129 Possible failure modes:
130 - gem5 failed to run -> STATE_ERROR
131 - timeout -> STATE_ERROR
132 - non-zero exit code -> STATE_ERROR
133
134 Possible non-failure results:
135 - exit code == 0 -> STATE_OK
136 - exit code == 2 -> STATE_SKIPPED
137 """
138
139 def __init__(self, gem5, gem5_args, timeout=0, **kwargs):
140 super(RunGem5, self).__init__("gem5", **kwargs)
141 self.gem5 = gem5
142 self.args = gem5_args
143 self.timeout = timeout
144
145 def _run(self):
146 gem5_cmd = [
147 self.gem5,
148 "-d", self.test_dir,
| 1#!/usr/bin/env python2
2#
3# Copyright (c) 2016 ARM Limited
4# All rights reserved
5#
6# The license below extends only to copyright in the software and shall
7# not be construed as granting a license to any other intellectual
8# property including but not limited to intellectual property relating
9# to a hardware implementation of the functionality of the software
10# licensed hereunder. You may use the software subject to the license
11# terms below provided that you ensure that this notice is replicated
12# unmodified and in its entirety in all distributions of the software,
13# modified or unmodified, in source code or in binary form.
14#
15# Redistribution and use in source and binary forms, with or without
16# modification, are permitted provided that the following conditions are
17# met: redistributions of source code must retain the above copyright
18# notice, this list of conditions and the following disclaimer;
19# redistributions in binary form must reproduce the above copyright
20# notice, this list of conditions and the following disclaimer in the
21# documentation and/or other materials provided with the distribution;
22# neither the name of the copyright holders nor the names of its
23# contributors may be used to endorse or promote products derived from
24# this software without specific prior written permission.
25#
26# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
27# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
28# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
29# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
30# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
31# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
32# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
33# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
34# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
35# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
36# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
37#
38# Authors: Andreas Sandberg
39
40from abc import ABCMeta, abstractmethod
41from datetime import datetime
42import difflib
43import functools
44import os
45import re
46import subprocess
47import sys
48import traceback
49
50from results import UnitResult
51from helpers import *
52
53_test_base = os.path.join(os.path.dirname(__file__), "..")
54
55class TestUnit(object):
56 """Base class for all test units.
57
58 A test unit is a part of a larger test case. Test cases usually
59 contain two types of units, run units (run gem5) and verify units
60 (diff output files). All unit implementations inherit from this
61 class.
62
63 A unit implementation overrides the _run() method. The test runner
64 calls the run() method, which wraps _run() to protect against
65 exceptions.
66
67 """
68
69 __metaclass__ = ABCMeta
70
71 def __init__(self, name, ref_dir, test_dir, skip=False):
72 self.name = name
73 self.ref_dir = ref_dir
74 self.test_dir = test_dir
75 self.force_skip = skip
76 self.start_time = None
77 self.stop_time = None
78
79 def result(self, state, **kwargs):
80 if self.start_time is not None and "runtime" not in kwargs:
81 self.stop_time = datetime.utcnow()
82 delta = self.stop_time - self.start_time
83 kwargs["runtime"] = delta.total_seconds()
84
85 return UnitResult(self.name, state, **kwargs)
86
87 def ok(self, **kwargs):
88 return self.result(UnitResult.STATE_OK, **kwargs)
89
90 def skip(self, **kwargs):
91 return self.result(UnitResult.STATE_SKIPPED, **kwargs)
92
93 def error(self, message, **kwargs):
94 return self.result(UnitResult.STATE_ERROR, message=message, **kwargs)
95
96 def failure(self, message, **kwargs):
97 return self.result(UnitResult.STATE_FAILURE, message=message, **kwargs)
98
99 def ref_file(self, fname):
100 return os.path.join(self.ref_dir, fname)
101
102 def out_file(self, fname):
103 return os.path.join(self.test_dir, fname)
104
105 def _read_output(self, fname, default=""):
106 try:
107 with open(self.out_file(fname), "r") as f:
108 return f.read()
109 except IOError:
110 return default
111
112 def run(self):
113 self.start_time = datetime.utcnow()
114 try:
115 if self.force_skip:
116 return self.skip()
117 else:
118 return self._run()
119 except:
120 return self.error("Python exception:\n%s" % traceback.format_exc())
121
122 @abstractmethod
123 def _run(self):
124 pass
125
126class RunGem5(TestUnit):
127 """Test unit representing a gem5 run.
128
129 Possible failure modes:
130 - gem5 failed to run -> STATE_ERROR
131 - timeout -> STATE_ERROR
132 - non-zero exit code -> STATE_ERROR
133
134 Possible non-failure results:
135 - exit code == 0 -> STATE_OK
136 - exit code == 2 -> STATE_SKIPPED
137 """
138
139 def __init__(self, gem5, gem5_args, timeout=0, **kwargs):
140 super(RunGem5, self).__init__("gem5", **kwargs)
141 self.gem5 = gem5
142 self.args = gem5_args
143 self.timeout = timeout
144
145 def _run(self):
146 gem5_cmd = [
147 self.gem5,
148 "-d", self.test_dir,
|
149 "-re",
150 ] + self.args
151
152 try:
153 with ProcessHelper(gem5_cmd, stdout=subprocess.PIPE,
154 stderr=subprocess.PIPE) as p:
155 status, gem5_stdout, gem5_stderr = p.call(timeout=self.timeout)
156 except CallTimeoutException as te:
157 return self.error("Timeout", stdout=te.stdout, stderr=te.stderr)
158 except OSError as ose:
159 return self.error("Failed to launch gem5: %s" % ose)
160
161 stderr = "\n".join([
162 "*** gem5 stderr ***",
163 gem5_stderr,
164 "",
165 "*** m5out/simerr ***",
166 self._read_output("simerr"),
167 ])
168
169 stdout = "\n".join([
170 "*** gem5 stdout ***",
171 gem5_stdout,
172 "",
173 "*** m5out/simout ***",
174 self._read_output("simout"),
175 ])
176
177 # Signal
178 if status < 0:
179 return self.error("gem5 terminated by signal %i" % (-status, ),
180 stdout=stdout, stderr=stderr)
181 elif status == 2:
182 return self.skip(stdout=stdout, stderr=stderr)
183 elif status > 0:
184 return self.error("gem5 exited with non-zero status: %i" % status,
185 stdout=stdout, stderr=stderr)
186 else:
187 return self.ok(stdout=stdout, stderr=stderr)
188
189class DiffOutFile(TestUnit):
190 """Test unit comparing and output file and a reference file."""
191
192 # regular expressions of lines to ignore when diffing outputs
193 diff_ignore_regexes = {
194 "simout" : [
195 re.compile('^Redirecting (stdout|stderr) to'),
196 re.compile('^gem5 compiled '),
197 re.compile('^gem5 started '),
198 re.compile('^gem5 executing on '),
199 re.compile('^command line:'),
200 re.compile("^Couldn't import dot_parser,"),
201 re.compile("^info: kernel located at:"),
202 re.compile("^Couldn't unlink "),
203 re.compile("^Using GPU kernel code file\(s\) "),
204 ],
205 "simerr" : [
206 #re.compile('^Simulation complete at'),
207 ],
208 "config.ini" : [
209 re.compile("^(executable|readfile|kernel|image_file)="),
210 re.compile("^(cwd|input|codefile)="),
211 ],
212 "config.json" : [
213 re.compile(r'''^\s*"(executable|readfile|kernel|image_file)":'''),
214 re.compile(r'''^\s*"(cwd|input|codefile)":'''),
215 ],
216 }
217
218 def __init__(self, fname, **kwargs):
219 super(DiffOutFile, self).__init__("diff[%s]" % fname,
220 **kwargs)
221
222 self.fname = fname
223 self.line_filters = DiffOutFile.diff_ignore_regexes.get(fname, tuple())
224
225 def _filter_file(self, fname):
226 def match_line(l):
227 for r in self.line_filters:
228 if r.match(l):
229 return True
230 return False
231
232 with open(fname, "r") as f:
233 for l in f:
234 if not match_line(l):
235 yield l
236
237
238 def _run(self):
239 fname = self.fname
240 ref = self.ref_file(fname)
241 out = self.out_file(fname)
242
243 if not os.path.exists(ref):
244 return self.error("%s doesn't exist in reference directory" \
245 % fname)
246
247 if not os.path.exists(out):
248 return self.error("%s doesn't exist in output directory" % fname)
249
250 diff = difflib.unified_diff(
251 tuple(self._filter_file(ref)),
252 tuple(self._filter_file(out)),
253 fromfile="ref/%s" % fname, tofile="out/%s" % fname)
254
255 diff = list(diff)
256 if diff:
257 return self.error("ref/%s and out/%s differ" % (fname, fname),
258 stderr="".join(diff))
259 else:
260 return self.ok(stdout="-- ref/%s and out/%s are identical --" \
261 % (fname, fname))
262
263class DiffStatFile(TestUnit):
264 """Test unit comparing two gem5 stat files."""
265
266 def __init__(self, **kwargs):
267 super(DiffStatFile, self).__init__("stat_diff", **kwargs)
268
269 self.stat_diff = os.path.join(_test_base, "diff-out")
270
271 def _run(self):
272 stats = "stats.txt"
273
274 cmd = [
275 self.stat_diff,
276 self.ref_file(stats), self.out_file(stats),
277 ]
278 with ProcessHelper(cmd,
279 stdout=subprocess.PIPE,
280 stderr=subprocess.PIPE) as p:
281 status, stdout, stderr = p.call()
282
283 if status == 0:
284 return self.ok(stdout=stdout, stderr=stderr)
285 if status == 1:
286 return self.failure("Statistics mismatch",
287 stdout=stdout, stderr=stderr)
288 else:
289 return self.error("diff-out returned an error: %i" % status,
290 stdout=stdout, stderr=stderr)
| 150 "-re",
151 ] + self.args
152
153 try:
154 with ProcessHelper(gem5_cmd, stdout=subprocess.PIPE,
155 stderr=subprocess.PIPE) as p:
156 status, gem5_stdout, gem5_stderr = p.call(timeout=self.timeout)
157 except CallTimeoutException as te:
158 return self.error("Timeout", stdout=te.stdout, stderr=te.stderr)
159 except OSError as ose:
160 return self.error("Failed to launch gem5: %s" % ose)
161
162 stderr = "\n".join([
163 "*** gem5 stderr ***",
164 gem5_stderr,
165 "",
166 "*** m5out/simerr ***",
167 self._read_output("simerr"),
168 ])
169
170 stdout = "\n".join([
171 "*** gem5 stdout ***",
172 gem5_stdout,
173 "",
174 "*** m5out/simout ***",
175 self._read_output("simout"),
176 ])
177
178 # Signal
179 if status < 0:
180 return self.error("gem5 terminated by signal %i" % (-status, ),
181 stdout=stdout, stderr=stderr)
182 elif status == 2:
183 return self.skip(stdout=stdout, stderr=stderr)
184 elif status > 0:
185 return self.error("gem5 exited with non-zero status: %i" % status,
186 stdout=stdout, stderr=stderr)
187 else:
188 return self.ok(stdout=stdout, stderr=stderr)
189
190class DiffOutFile(TestUnit):
191 """Test unit comparing and output file and a reference file."""
192
193 # regular expressions of lines to ignore when diffing outputs
194 diff_ignore_regexes = {
195 "simout" : [
196 re.compile('^Redirecting (stdout|stderr) to'),
197 re.compile('^gem5 compiled '),
198 re.compile('^gem5 started '),
199 re.compile('^gem5 executing on '),
200 re.compile('^command line:'),
201 re.compile("^Couldn't import dot_parser,"),
202 re.compile("^info: kernel located at:"),
203 re.compile("^Couldn't unlink "),
204 re.compile("^Using GPU kernel code file\(s\) "),
205 ],
206 "simerr" : [
207 #re.compile('^Simulation complete at'),
208 ],
209 "config.ini" : [
210 re.compile("^(executable|readfile|kernel|image_file)="),
211 re.compile("^(cwd|input|codefile)="),
212 ],
213 "config.json" : [
214 re.compile(r'''^\s*"(executable|readfile|kernel|image_file)":'''),
215 re.compile(r'''^\s*"(cwd|input|codefile)":'''),
216 ],
217 }
218
219 def __init__(self, fname, **kwargs):
220 super(DiffOutFile, self).__init__("diff[%s]" % fname,
221 **kwargs)
222
223 self.fname = fname
224 self.line_filters = DiffOutFile.diff_ignore_regexes.get(fname, tuple())
225
226 def _filter_file(self, fname):
227 def match_line(l):
228 for r in self.line_filters:
229 if r.match(l):
230 return True
231 return False
232
233 with open(fname, "r") as f:
234 for l in f:
235 if not match_line(l):
236 yield l
237
238
239 def _run(self):
240 fname = self.fname
241 ref = self.ref_file(fname)
242 out = self.out_file(fname)
243
244 if not os.path.exists(ref):
245 return self.error("%s doesn't exist in reference directory" \
246 % fname)
247
248 if not os.path.exists(out):
249 return self.error("%s doesn't exist in output directory" % fname)
250
251 diff = difflib.unified_diff(
252 tuple(self._filter_file(ref)),
253 tuple(self._filter_file(out)),
254 fromfile="ref/%s" % fname, tofile="out/%s" % fname)
255
256 diff = list(diff)
257 if diff:
258 return self.error("ref/%s and out/%s differ" % (fname, fname),
259 stderr="".join(diff))
260 else:
261 return self.ok(stdout="-- ref/%s and out/%s are identical --" \
262 % (fname, fname))
263
264class DiffStatFile(TestUnit):
265 """Test unit comparing two gem5 stat files."""
266
267 def __init__(self, **kwargs):
268 super(DiffStatFile, self).__init__("stat_diff", **kwargs)
269
270 self.stat_diff = os.path.join(_test_base, "diff-out")
271
272 def _run(self):
273 stats = "stats.txt"
274
275 cmd = [
276 self.stat_diff,
277 self.ref_file(stats), self.out_file(stats),
278 ]
279 with ProcessHelper(cmd,
280 stdout=subprocess.PIPE,
281 stderr=subprocess.PIPE) as p:
282 status, stdout, stderr = p.call()
283
284 if status == 0:
285 return self.ok(stdout=stdout, stderr=stderr)
286 if status == 1:
287 return self.failure("Statistics mismatch",
288 stdout=stdout, stderr=stderr)
289 else:
290 return self.error("diff-out returned an error: %i" % status,
291 stdout=stdout, stderr=stderr)
|