1#!/usr/bin/env python2
2#
3# Copyright (c) 2016 ARM Limited
4# All rights reserved
5#
6# The license below extends only to copyright in the software and shall
7# not be construed as granting a license to any other intellectual
8# property including but not limited to intellectual property relating
9# to a hardware implementation of the functionality of the software
10# licensed hereunder. You may use the software subject to the license
11# terms below provided that you ensure that this notice is replicated
12# unmodified and in its entirety in all distributions of the software,
13# modified or unmodified, in source code or in binary form.
14#
15# Redistribution and use in source and binary forms, with or without
16# modification, are permitted provided that the following conditions are
17# met: redistributions of source code must retain the above copyright
18# notice, this list of conditions and the following disclaimer;
19# redistributions in binary form must reproduce the above copyright
20# notice, this list of conditions and the following disclaimer in the
21# documentation and/or other materials provided with the distribution;
22# neither the name of the copyright holders nor the names of its
23# contributors may be used to endorse or promote products derived from
24# this software without specific prior written permission.
25#
26# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
27# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
28# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
29# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
30# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
31# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
32# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
33# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
34# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
35# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
36# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
37#
38# Authors: Andreas Sandberg
39
40from abc import ABCMeta, abstractmethod
41from datetime import datetime
42import difflib
43import functools
44import os
45import re
46import subprocess
47import sys
48import traceback
49
50from results import UnitResult
51from helpers import *
52
53_test_base = os.path.join(os.path.dirname(__file__), "..")
54
55class TestUnit(object):
56 """Base class for all test units.
57
58 A test unit is a part of a larger test case. Test cases usually
59 contain two types of units, run units (run gem5) and verify units
60 (diff output files). All unit implementations inherit from this
61 class.
62
63 A unit implementation overrides the _run() method. The test runner
64 calls the run() method, which wraps _run() to protect against
65 exceptions.
66
67 """
68
69 __metaclass__ = ABCMeta
70
71 def __init__(self, name, ref_dir, test_dir, skip=False):
72 self.name = name
73 self.ref_dir = ref_dir
74 self.test_dir = test_dir
75 self.force_skip = skip
76 self.start_time = None
77 self.stop_time = None
78
79 def result(self, state, **kwargs):
80 if self.start_time is not None and "runtime" not in kwargs:
81 self.stop_time = datetime.utcnow()
82 delta = self.stop_time - self.start_time
83 kwargs["runtime"] = delta.total_seconds()
84
85 return UnitResult(self.name, state, **kwargs)
86
87 def ok(self, **kwargs):
88 return self.result(UnitResult.STATE_OK, **kwargs)
89
90 def skip(self, **kwargs):
91 return self.result(UnitResult.STATE_SKIPPED, **kwargs)
92
93 def error(self, message, **kwargs):
94 return self.result(UnitResult.STATE_ERROR, message=message, **kwargs)
95
96 def failure(self, message, **kwargs):
97 return self.result(UnitResult.STATE_FAILURE, message=message, **kwargs)
98
99 def ref_file(self, fname):
100 return os.path.join(self.ref_dir, fname)
101
102 def out_file(self, fname):
103 return os.path.join(self.test_dir, fname)
104
105 def _read_output(self, fname, default=""):
106 try:
107 with open(self.out_file(fname), "r") as f:
108 return f.read()
109 except IOError:
110 return default
111
112 def run(self):
113 self.start_time = datetime.utcnow()
114 try:
115 if self.force_skip:
116 return self.skip()
117 else:
118 return self._run()
119 except:
120 return self.error("Python exception:\n%s" % traceback.format_exc())
121
122 @abstractmethod
123 def _run(self):
124 pass
125
126class RunGem5(TestUnit):
127 """Test unit representing a gem5 run.
128
129 Possible failure modes:
130 - gem5 failed to run -> STATE_ERROR
131 - timeout -> STATE_ERROR
132 - non-zero exit code -> STATE_ERROR
133
134 Possible non-failure results:
135 - exit code == 0 -> STATE_OK
136 - exit code == 2 -> STATE_SKIPPED
137 """
138
139 def __init__(self, gem5, gem5_args, timeout=0, **kwargs):
140 super(RunGem5, self).__init__("gem5", **kwargs)
141 self.gem5 = gem5
142 self.args = gem5_args
143 self.timeout = timeout
144
145 def _run(self):
146 gem5_cmd = [
147 self.gem5,
148 "-d", self.test_dir,
149 "--stats-file", "text://stats.txt?desc=False",
150 "-re",
151 ] + self.args
152
153 try:
154 with ProcessHelper(gem5_cmd, stdout=subprocess.PIPE,
155 stderr=subprocess.PIPE) as p:
156 status, gem5_stdout, gem5_stderr = p.call(timeout=self.timeout)
157 except CallTimeoutException as te:
158 return self.error("Timeout", stdout=te.stdout, stderr=te.stderr)
159 except OSError as ose:
160 return self.error("Failed to launch gem5: %s" % ose)
161
162 stderr = "\n".join([
163 "*** gem5 stderr ***",
164 gem5_stderr,
165 "",
166 "*** m5out/simerr ***",
167 self._read_output("simerr"),
168 ])
169
170 stdout = "\n".join([
171 "*** gem5 stdout ***",
172 gem5_stdout,
173 "",
174 "*** m5out/simout ***",
175 self._read_output("simout"),
176 ])
177
178 # Signal
179 if status < 0:
180 return self.error("gem5 terminated by signal %i" % (-status, ),
181 stdout=stdout, stderr=stderr)
182 elif status == 2:
183 return self.skip(stdout=stdout, stderr=stderr)
184 elif status > 0:
185 return self.error("gem5 exited with non-zero status: %i" % status,
186 stdout=stdout, stderr=stderr)
187 else:
188 return self.ok(stdout=stdout, stderr=stderr)
189
190class DiffOutFile(TestUnit):
191 """Test unit comparing and output file and a reference file."""
192
193 # regular expressions of lines to ignore when diffing outputs
194 diff_ignore_regexes = {
195 "simout" : [
196 re.compile('^Redirecting (stdout|stderr) to'),
197 re.compile('^gem5 compiled '),
198 re.compile('^gem5 started '),
199 re.compile('^gem5 executing on '),
200 re.compile('^command line:'),
201 re.compile("^Couldn't import dot_parser,"),
202 re.compile("^info: kernel located at:"),
203 re.compile("^Couldn't unlink "),
204 re.compile("^Using GPU kernel code file\(s\) "),
205 ],
206 "simerr" : [
207 #re.compile('^Simulation complete at'),
208 ],
209 "config.ini" : [
210 re.compile("^(executable|readfile|kernel|image_file)="),
211 re.compile("^(cwd|input|codefile)="),
212 ],
213 "config.json" : [
214 re.compile(r'''^\s*"(executable|readfile|kernel|image_file)":'''),
215 re.compile(r'''^\s*"(cwd|input|codefile)":'''),
216 ],
217 }
218
219 def __init__(self, fname, **kwargs):
220 super(DiffOutFile, self).__init__("diff[%s]" % fname,
221 **kwargs)
222
223 self.fname = fname
224 self.line_filters = DiffOutFile.diff_ignore_regexes.get(fname, tuple())
225
226 def _filter_file(self, fname):
227 def match_line(l):
228 for r in self.line_filters:
229 if r.match(l):
230 return True
231 return False
232
233 with open(fname, "r") as f:
234 for l in f:
235 if not match_line(l):
236 yield l
237
238
239 def _run(self):
240 fname = self.fname
241 ref = self.ref_file(fname)
242 out = self.out_file(fname)
243
244 if not os.path.exists(ref):
245 return self.error("%s doesn't exist in reference directory" \
246 % fname)
247
248 if not os.path.exists(out):
249 return self.error("%s doesn't exist in output directory" % fname)
250
251 diff = difflib.unified_diff(
252 tuple(self._filter_file(ref)),
253 tuple(self._filter_file(out)),
254 fromfile="ref/%s" % fname, tofile="out/%s" % fname)
255
256 diff = list(diff)
257 if diff:
258 return self.error("ref/%s and out/%s differ" % (fname, fname),
259 stderr="".join(diff))
260 else:
261 return self.ok(stdout="-- ref/%s and out/%s are identical --" \
262 % (fname, fname))
263
264class DiffStatFile(TestUnit):
265 """Test unit comparing two gem5 stat files."""
266
267 def __init__(self, **kwargs):
268 super(DiffStatFile, self).__init__("stat_diff", **kwargs)
269
270 self.stat_diff = os.path.join(_test_base, "diff-out")
271
272 def _run(self):
| 1#!/usr/bin/env python2
2#
3# Copyright (c) 2016 ARM Limited
4# All rights reserved
5#
6# The license below extends only to copyright in the software and shall
7# not be construed as granting a license to any other intellectual
8# property including but not limited to intellectual property relating
9# to a hardware implementation of the functionality of the software
10# licensed hereunder. You may use the software subject to the license
11# terms below provided that you ensure that this notice is replicated
12# unmodified and in its entirety in all distributions of the software,
13# modified or unmodified, in source code or in binary form.
14#
15# Redistribution and use in source and binary forms, with or without
16# modification, are permitted provided that the following conditions are
17# met: redistributions of source code must retain the above copyright
18# notice, this list of conditions and the following disclaimer;
19# redistributions in binary form must reproduce the above copyright
20# notice, this list of conditions and the following disclaimer in the
21# documentation and/or other materials provided with the distribution;
22# neither the name of the copyright holders nor the names of its
23# contributors may be used to endorse or promote products derived from
24# this software without specific prior written permission.
25#
26# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
27# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
28# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
29# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
30# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
31# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
32# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
33# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
34# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
35# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
36# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
37#
38# Authors: Andreas Sandberg
39
40from abc import ABCMeta, abstractmethod
41from datetime import datetime
42import difflib
43import functools
44import os
45import re
46import subprocess
47import sys
48import traceback
49
50from results import UnitResult
51from helpers import *
52
53_test_base = os.path.join(os.path.dirname(__file__), "..")
54
55class TestUnit(object):
56 """Base class for all test units.
57
58 A test unit is a part of a larger test case. Test cases usually
59 contain two types of units, run units (run gem5) and verify units
60 (diff output files). All unit implementations inherit from this
61 class.
62
63 A unit implementation overrides the _run() method. The test runner
64 calls the run() method, which wraps _run() to protect against
65 exceptions.
66
67 """
68
69 __metaclass__ = ABCMeta
70
71 def __init__(self, name, ref_dir, test_dir, skip=False):
72 self.name = name
73 self.ref_dir = ref_dir
74 self.test_dir = test_dir
75 self.force_skip = skip
76 self.start_time = None
77 self.stop_time = None
78
79 def result(self, state, **kwargs):
80 if self.start_time is not None and "runtime" not in kwargs:
81 self.stop_time = datetime.utcnow()
82 delta = self.stop_time - self.start_time
83 kwargs["runtime"] = delta.total_seconds()
84
85 return UnitResult(self.name, state, **kwargs)
86
87 def ok(self, **kwargs):
88 return self.result(UnitResult.STATE_OK, **kwargs)
89
90 def skip(self, **kwargs):
91 return self.result(UnitResult.STATE_SKIPPED, **kwargs)
92
93 def error(self, message, **kwargs):
94 return self.result(UnitResult.STATE_ERROR, message=message, **kwargs)
95
96 def failure(self, message, **kwargs):
97 return self.result(UnitResult.STATE_FAILURE, message=message, **kwargs)
98
99 def ref_file(self, fname):
100 return os.path.join(self.ref_dir, fname)
101
102 def out_file(self, fname):
103 return os.path.join(self.test_dir, fname)
104
105 def _read_output(self, fname, default=""):
106 try:
107 with open(self.out_file(fname), "r") as f:
108 return f.read()
109 except IOError:
110 return default
111
112 def run(self):
113 self.start_time = datetime.utcnow()
114 try:
115 if self.force_skip:
116 return self.skip()
117 else:
118 return self._run()
119 except:
120 return self.error("Python exception:\n%s" % traceback.format_exc())
121
122 @abstractmethod
123 def _run(self):
124 pass
125
126class RunGem5(TestUnit):
127 """Test unit representing a gem5 run.
128
129 Possible failure modes:
130 - gem5 failed to run -> STATE_ERROR
131 - timeout -> STATE_ERROR
132 - non-zero exit code -> STATE_ERROR
133
134 Possible non-failure results:
135 - exit code == 0 -> STATE_OK
136 - exit code == 2 -> STATE_SKIPPED
137 """
138
139 def __init__(self, gem5, gem5_args, timeout=0, **kwargs):
140 super(RunGem5, self).__init__("gem5", **kwargs)
141 self.gem5 = gem5
142 self.args = gem5_args
143 self.timeout = timeout
144
145 def _run(self):
146 gem5_cmd = [
147 self.gem5,
148 "-d", self.test_dir,
149 "--stats-file", "text://stats.txt?desc=False",
150 "-re",
151 ] + self.args
152
153 try:
154 with ProcessHelper(gem5_cmd, stdout=subprocess.PIPE,
155 stderr=subprocess.PIPE) as p:
156 status, gem5_stdout, gem5_stderr = p.call(timeout=self.timeout)
157 except CallTimeoutException as te:
158 return self.error("Timeout", stdout=te.stdout, stderr=te.stderr)
159 except OSError as ose:
160 return self.error("Failed to launch gem5: %s" % ose)
161
162 stderr = "\n".join([
163 "*** gem5 stderr ***",
164 gem5_stderr,
165 "",
166 "*** m5out/simerr ***",
167 self._read_output("simerr"),
168 ])
169
170 stdout = "\n".join([
171 "*** gem5 stdout ***",
172 gem5_stdout,
173 "",
174 "*** m5out/simout ***",
175 self._read_output("simout"),
176 ])
177
178 # Signal
179 if status < 0:
180 return self.error("gem5 terminated by signal %i" % (-status, ),
181 stdout=stdout, stderr=stderr)
182 elif status == 2:
183 return self.skip(stdout=stdout, stderr=stderr)
184 elif status > 0:
185 return self.error("gem5 exited with non-zero status: %i" % status,
186 stdout=stdout, stderr=stderr)
187 else:
188 return self.ok(stdout=stdout, stderr=stderr)
189
190class DiffOutFile(TestUnit):
191 """Test unit comparing and output file and a reference file."""
192
193 # regular expressions of lines to ignore when diffing outputs
194 diff_ignore_regexes = {
195 "simout" : [
196 re.compile('^Redirecting (stdout|stderr) to'),
197 re.compile('^gem5 compiled '),
198 re.compile('^gem5 started '),
199 re.compile('^gem5 executing on '),
200 re.compile('^command line:'),
201 re.compile("^Couldn't import dot_parser,"),
202 re.compile("^info: kernel located at:"),
203 re.compile("^Couldn't unlink "),
204 re.compile("^Using GPU kernel code file\(s\) "),
205 ],
206 "simerr" : [
207 #re.compile('^Simulation complete at'),
208 ],
209 "config.ini" : [
210 re.compile("^(executable|readfile|kernel|image_file)="),
211 re.compile("^(cwd|input|codefile)="),
212 ],
213 "config.json" : [
214 re.compile(r'''^\s*"(executable|readfile|kernel|image_file)":'''),
215 re.compile(r'''^\s*"(cwd|input|codefile)":'''),
216 ],
217 }
218
219 def __init__(self, fname, **kwargs):
220 super(DiffOutFile, self).__init__("diff[%s]" % fname,
221 **kwargs)
222
223 self.fname = fname
224 self.line_filters = DiffOutFile.diff_ignore_regexes.get(fname, tuple())
225
226 def _filter_file(self, fname):
227 def match_line(l):
228 for r in self.line_filters:
229 if r.match(l):
230 return True
231 return False
232
233 with open(fname, "r") as f:
234 for l in f:
235 if not match_line(l):
236 yield l
237
238
239 def _run(self):
240 fname = self.fname
241 ref = self.ref_file(fname)
242 out = self.out_file(fname)
243
244 if not os.path.exists(ref):
245 return self.error("%s doesn't exist in reference directory" \
246 % fname)
247
248 if not os.path.exists(out):
249 return self.error("%s doesn't exist in output directory" % fname)
250
251 diff = difflib.unified_diff(
252 tuple(self._filter_file(ref)),
253 tuple(self._filter_file(out)),
254 fromfile="ref/%s" % fname, tofile="out/%s" % fname)
255
256 diff = list(diff)
257 if diff:
258 return self.error("ref/%s and out/%s differ" % (fname, fname),
259 stderr="".join(diff))
260 else:
261 return self.ok(stdout="-- ref/%s and out/%s are identical --" \
262 % (fname, fname))
263
264class DiffStatFile(TestUnit):
265 """Test unit comparing two gem5 stat files."""
266
267 def __init__(self, **kwargs):
268 super(DiffStatFile, self).__init__("stat_diff", **kwargs)
269
270 self.stat_diff = os.path.join(_test_base, "diff-out")
271
272 def _run(self):
|