4# All rights reserved
5#
6# The license below extends only to copyright in the software and shall
7# not be construed as granting a license to any other intellectual
8# property including but not limited to intellectual property relating
9# to a hardware implementation of the functionality of the software
10# licensed hereunder. You may use the software subject to the license
11# terms below provided that you ensure that this notice is replicated
12# unmodified and in its entirety in all distributions of the software,
13# modified or unmodified, in source code or in binary form.
14#
15# Redistribution and use in source and binary forms, with or without
16# modification, are permitted provided that the following conditions are
17# met: redistributions of source code must retain the above copyright
18# notice, this list of conditions and the following disclaimer;
19# redistributions in binary form must reproduce the above copyright
20# notice, this list of conditions and the following disclaimer in the
21# documentation and/or other materials provided with the distribution;
22# neither the name of the copyright holders nor the names of its
23# contributors may be used to endorse or promote products derived from
24# this software without specific prior written permission.
25#
26# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
27# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
28# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
29# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
30# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
31# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
32# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
33# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
34# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
35# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
36# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
37#
38# Authors: Andreas Sandberg
39
40from abc import ABCMeta, abstractmethod
41import os
42from collections import namedtuple
43from units import *
44from results import TestResult
45import shutil
46
47_test_base = os.path.join(os.path.dirname(__file__), "..")
48
49ClassicConfig = namedtuple("ClassicConfig", (
50 "category",
51 "mode",
52 "workload",
53 "isa",
54 "os",
55 "config",
56))
57
58# There are currently two "classes" of test
59# configurations. Architecture-specific ones and generic ones
60# (typically SE mode tests). In both cases, the configuration name
61# matches a file in tests/configs/ that will be picked up by the test
62# runner (run.py).
63#
64# Architecture specific configurations are listed in the arch_configs
65# dictionary. This is indexed by a (cpu architecture, gpu
66# architecture) tuple. GPU architecture is optional and may be None.
67#
68# Generic configurations are listed in the generic_configs tuple.
69#
70# When discovering available test cases, this script look uses the
71# test list as a list of /candidate/ configurations. A configuration
72# is only used if a test has a reference output for that
73# configuration. In addition to the base configurations from
74# arch_configs and generic_configs, a Ruby configuration may be
75# appended to the base name (this is probed /in addition/ to the
76# original name. See get_tests() for details.
77#
78arch_configs = {
79 ("alpha", None) : (
80 'tsunami-simple-atomic',
81 'tsunami-simple-timing',
82 'tsunami-simple-atomic-dual',
83 'tsunami-simple-timing-dual',
84 'twosys-tsunami-simple-atomic',
85 'tsunami-o3', 'tsunami-o3-dual',
86 'tsunami-minor', 'tsunami-minor-dual',
87 'tsunami-switcheroo-full',
88 ),
89
90 ("arm", None) : (
91 'simple-atomic-dummychecker',
92 'o3-timing-checker',
93 'realview-simple-atomic',
94 'realview-simple-atomic-dual',
95 'realview-simple-atomic-checkpoint',
96 'realview-simple-timing',
97 'realview-simple-timing-dual',
98 'realview-o3',
99 'realview-o3-checker',
100 'realview-o3-dual',
101 'realview-minor',
102 'realview-minor-dual',
103 'realview-switcheroo-atomic',
104 'realview-switcheroo-timing',
105 'realview-switcheroo-o3',
106 'realview-switcheroo-full',
107 'realview64-simple-atomic',
108 'realview64-simple-atomic-checkpoint',
109 'realview64-simple-atomic-dual',
110 'realview64-simple-timing',
111 'realview64-simple-timing-dual',
112 'realview64-o3',
113 'realview64-o3-checker',
114 'realview64-o3-dual',
115 'realview64-minor',
116 'realview64-minor-dual',
117 'realview64-switcheroo-atomic',
118 'realview64-switcheroo-timing',
119 'realview64-switcheroo-o3',
120 'realview64-switcheroo-full',
121 ),
122
123 ("sparc", None) : (
124 't1000-simple-atomic',
125 't1000-simple-x86',
126 ),
127
128 ("timing", None) : (
129 'pc-simple-atomic',
130 'pc-simple-timing',
131 'pc-o3-timing',
132 'pc-switcheroo-full',
133 ),
134
135 ("x86", "hsail") : (
136 'gpu',
137 ),
138}
139
140generic_configs = (
141 'simple-atomic',
142 'simple-atomic-mp',
143 'simple-timing',
144 'simple-timing-mp',
145
146 'minor-timing',
147 'minor-timing-mp',
148
149 'o3-timing',
150 'o3-timing-mt',
151 'o3-timing-mp',
152
153 'rubytest',
154 'memcheck',
155 'memtest',
156 'memtest-filter',
157 'tgen-simple-mem',
158 'tgen-dram-ctrl',
159
160 'learning-gem5-p1-simple',
161 'learning-gem5-p1-two-level',
162)
163
| 4# All rights reserved
5#
6# The license below extends only to copyright in the software and shall
7# not be construed as granting a license to any other intellectual
8# property including but not limited to intellectual property relating
9# to a hardware implementation of the functionality of the software
10# licensed hereunder. You may use the software subject to the license
11# terms below provided that you ensure that this notice is replicated
12# unmodified and in its entirety in all distributions of the software,
13# modified or unmodified, in source code or in binary form.
14#
15# Redistribution and use in source and binary forms, with or without
16# modification, are permitted provided that the following conditions are
17# met: redistributions of source code must retain the above copyright
18# notice, this list of conditions and the following disclaimer;
19# redistributions in binary form must reproduce the above copyright
20# notice, this list of conditions and the following disclaimer in the
21# documentation and/or other materials provided with the distribution;
22# neither the name of the copyright holders nor the names of its
23# contributors may be used to endorse or promote products derived from
24# this software without specific prior written permission.
25#
26# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
27# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
28# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
29# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
30# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
31# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
32# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
33# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
34# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
35# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
36# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
37#
38# Authors: Andreas Sandberg
39
40from abc import ABCMeta, abstractmethod
41import os
42from collections import namedtuple
43from units import *
44from results import TestResult
45import shutil
46
47_test_base = os.path.join(os.path.dirname(__file__), "..")
48
49ClassicConfig = namedtuple("ClassicConfig", (
50 "category",
51 "mode",
52 "workload",
53 "isa",
54 "os",
55 "config",
56))
57
58# There are currently two "classes" of test
59# configurations. Architecture-specific ones and generic ones
60# (typically SE mode tests). In both cases, the configuration name
61# matches a file in tests/configs/ that will be picked up by the test
62# runner (run.py).
63#
64# Architecture specific configurations are listed in the arch_configs
65# dictionary. This is indexed by a (cpu architecture, gpu
66# architecture) tuple. GPU architecture is optional and may be None.
67#
68# Generic configurations are listed in the generic_configs tuple.
69#
70# When discovering available test cases, this script look uses the
71# test list as a list of /candidate/ configurations. A configuration
72# is only used if a test has a reference output for that
73# configuration. In addition to the base configurations from
74# arch_configs and generic_configs, a Ruby configuration may be
75# appended to the base name (this is probed /in addition/ to the
76# original name. See get_tests() for details.
77#
78arch_configs = {
79 ("alpha", None) : (
80 'tsunami-simple-atomic',
81 'tsunami-simple-timing',
82 'tsunami-simple-atomic-dual',
83 'tsunami-simple-timing-dual',
84 'twosys-tsunami-simple-atomic',
85 'tsunami-o3', 'tsunami-o3-dual',
86 'tsunami-minor', 'tsunami-minor-dual',
87 'tsunami-switcheroo-full',
88 ),
89
90 ("arm", None) : (
91 'simple-atomic-dummychecker',
92 'o3-timing-checker',
93 'realview-simple-atomic',
94 'realview-simple-atomic-dual',
95 'realview-simple-atomic-checkpoint',
96 'realview-simple-timing',
97 'realview-simple-timing-dual',
98 'realview-o3',
99 'realview-o3-checker',
100 'realview-o3-dual',
101 'realview-minor',
102 'realview-minor-dual',
103 'realview-switcheroo-atomic',
104 'realview-switcheroo-timing',
105 'realview-switcheroo-o3',
106 'realview-switcheroo-full',
107 'realview64-simple-atomic',
108 'realview64-simple-atomic-checkpoint',
109 'realview64-simple-atomic-dual',
110 'realview64-simple-timing',
111 'realview64-simple-timing-dual',
112 'realview64-o3',
113 'realview64-o3-checker',
114 'realview64-o3-dual',
115 'realview64-minor',
116 'realview64-minor-dual',
117 'realview64-switcheroo-atomic',
118 'realview64-switcheroo-timing',
119 'realview64-switcheroo-o3',
120 'realview64-switcheroo-full',
121 ),
122
123 ("sparc", None) : (
124 't1000-simple-atomic',
125 't1000-simple-x86',
126 ),
127
128 ("timing", None) : (
129 'pc-simple-atomic',
130 'pc-simple-timing',
131 'pc-o3-timing',
132 'pc-switcheroo-full',
133 ),
134
135 ("x86", "hsail") : (
136 'gpu',
137 ),
138}
139
140generic_configs = (
141 'simple-atomic',
142 'simple-atomic-mp',
143 'simple-timing',
144 'simple-timing-mp',
145
146 'minor-timing',
147 'minor-timing-mp',
148
149 'o3-timing',
150 'o3-timing-mt',
151 'o3-timing-mp',
152
153 'rubytest',
154 'memcheck',
155 'memtest',
156 'memtest-filter',
157 'tgen-simple-mem',
158 'tgen-dram-ctrl',
159
160 'learning-gem5-p1-simple',
161 'learning-gem5-p1-two-level',
162)
163
|
164all_categories = ("quick", "long")
165all_modes = ("fs", "se")
166
167class Test(object):
168 """Test case base class.
169
170 Test cases consists of one or more test units that are run in two
171 phases. A run phase (units produced by run_units() and a verify
172 phase (units from verify_units()). The verify phase is skipped if
173 the run phase fails.
174
175 """
176
177 __metaclass__ = ABCMeta
178
179 def __init__(self, name):
180 self.test_name = name
181
182 @abstractmethod
183 def ref_files(self):
184 """Get a list of reference files used by this test case"""
185 pass
186
187 @abstractmethod
188 def run_units(self):
189 """Units (typically RunGem5 instances) that describe the run phase of
190 this test.
191
192 """
193 pass
194
195 @abstractmethod
196 def verify_units(self):
197 """Verify the output from the run phase (see run_units())."""
198 pass
199
200 @abstractmethod
201 def update_ref(self):
202 """Update reference files with files from a test run"""
203 pass
204
205 def run(self):
206 """Run this test case and return a list of results"""
207
208 run_results = [ u.run() for u in self.run_units() ]
209 run_ok = all([not r.skipped() and r for r in run_results ])
210
211 verify_results = [
212 u.run() if run_ok else u.skip()
213 for u in self.verify_units()
214 ]
215
216 return TestResult(self.test_name,
217 run_results=run_results,
218 verify_results=verify_results)
219
220 def __str__(self):
221 return self.test_name
222
223class ClassicTest(Test):
224 # The diff ignore list contains all files that shouldn't be diffed
225 # using DiffOutFile. These files typically use special-purpose
226 # diff tools (e.g., DiffStatFile).
227 diff_ignore_files = FileIgnoreList(
228 names=(
229 # Stat files use a special stat differ
230 "stats.txt",
231 ), rex=(
232 ))
233
234 # These files should never be included in the list of
235 # reference files. This list should include temporary files
236 # and other files that we don't care about.
237 ref_ignore_files = FileIgnoreList(
238 names=(
239 "EMPTY",
240 ), rex=(
241 # Mercurial sometimes leaves backups when applying MQ patches
242 r"\.orig$",
243 r"\.rej$",
244 ))
245
246 def __init__(self, gem5, output_dir, config_tuple,
247 timeout=None,
248 skip=False, skip_diff_out=False, skip_diff_stat=False):
249
250 super(ClassicTest, self).__init__("/".join(config_tuple))
251
252 ct = config_tuple
253
254 self.gem5 = os.path.abspath(gem5)
255 self.script = os.path.join(_test_base, "run.py")
256 self.config_tuple = ct
257 self.timeout = timeout
258
259 self.output_dir = output_dir
260 self.ref_dir = os.path.join(_test_base,
261 ct.category, ct.mode, ct.workload,
262 "ref", ct.isa, ct.os, ct.config)
263 self.skip_run = skip
264 self.skip_diff_out = skip or skip_diff_out
265 self.skip_diff_stat = skip or skip_diff_stat
266
267 def ref_files(self):
268 ref_dir = os.path.abspath(self.ref_dir)
269 for root, dirs, files in os.walk(ref_dir, topdown=False):
270 for f in files:
271 fpath = os.path.join(root[len(ref_dir) + 1:], f)
272 if fpath not in ClassicTest.ref_ignore_files:
273 yield fpath
274
275 def run_units(self):
276 args = [
277 self.script,
278 "/".join(self.config_tuple),
279 ]
280
281 return [
282 RunGem5(self.gem5, args,
283 ref_dir=self.ref_dir, test_dir=self.output_dir,
284 skip=self.skip_run),
285 ]
286
287 def verify_units(self):
288 ref_files = set(self.ref_files())
289 units = []
290 if "stats.txt" in ref_files:
291 units.append(
292 DiffStatFile(ref_dir=self.ref_dir, test_dir=self.output_dir,
293 skip=self.skip_diff_stat))
294 units += [
295 DiffOutFile(f,
296 ref_dir=self.ref_dir, test_dir=self.output_dir,
297 skip=self.skip_diff_out)
298 for f in ref_files if f not in ClassicTest.diff_ignore_files
299 ]
300
301 return units
302
303 def update_ref(self):
304 for fname in self.ref_files():
305 shutil.copy(
306 os.path.join(self.output_dir, fname),
307 os.path.join(self.ref_dir, fname))
308
309def parse_test_filter(test_filter):
310 wildcards = ("", "*")
311
312 _filter = list(test_filter.split("/"))
313 if len(_filter) > 3:
314 raise RuntimeError("Illegal test filter string")
315 _filter += [ "", ] * (3 - len(_filter))
316
317 isa, cat, mode = _filter
318
319 if isa in wildcards:
320 raise RuntimeError("No ISA specified")
321
322 cat = all_categories if cat in wildcards else (cat, )
323 mode = all_modes if mode in wildcards else (mode, )
324
325 return isa, cat, mode
326
327def get_tests(isa,
328 categories=all_categories, modes=all_modes,
329 ruby_protocol=None, gpu_isa=None):
330
331 # Generate a list of candidate configs
332 configs = list(arch_configs.get((isa, gpu_isa), []))
333
334 if (isa, gpu_isa) == ("x86", "hsail"):
335 if ruby_protocol == "GPU_RfO":
336 configs += ['gpu-randomtest']
337 else:
338 configs += generic_configs
339
| 174all_categories = ("quick", "long")
175all_modes = ("fs", "se")
176
177class Test(object):
178 """Test case base class.
179
180 Test cases consists of one or more test units that are run in two
181 phases. A run phase (units produced by run_units() and a verify
182 phase (units from verify_units()). The verify phase is skipped if
183 the run phase fails.
184
185 """
186
187 __metaclass__ = ABCMeta
188
189 def __init__(self, name):
190 self.test_name = name
191
192 @abstractmethod
193 def ref_files(self):
194 """Get a list of reference files used by this test case"""
195 pass
196
197 @abstractmethod
198 def run_units(self):
199 """Units (typically RunGem5 instances) that describe the run phase of
200 this test.
201
202 """
203 pass
204
205 @abstractmethod
206 def verify_units(self):
207 """Verify the output from the run phase (see run_units())."""
208 pass
209
210 @abstractmethod
211 def update_ref(self):
212 """Update reference files with files from a test run"""
213 pass
214
215 def run(self):
216 """Run this test case and return a list of results"""
217
218 run_results = [ u.run() for u in self.run_units() ]
219 run_ok = all([not r.skipped() and r for r in run_results ])
220
221 verify_results = [
222 u.run() if run_ok else u.skip()
223 for u in self.verify_units()
224 ]
225
226 return TestResult(self.test_name,
227 run_results=run_results,
228 verify_results=verify_results)
229
230 def __str__(self):
231 return self.test_name
232
233class ClassicTest(Test):
234 # The diff ignore list contains all files that shouldn't be diffed
235 # using DiffOutFile. These files typically use special-purpose
236 # diff tools (e.g., DiffStatFile).
237 diff_ignore_files = FileIgnoreList(
238 names=(
239 # Stat files use a special stat differ
240 "stats.txt",
241 ), rex=(
242 ))
243
244 # These files should never be included in the list of
245 # reference files. This list should include temporary files
246 # and other files that we don't care about.
247 ref_ignore_files = FileIgnoreList(
248 names=(
249 "EMPTY",
250 ), rex=(
251 # Mercurial sometimes leaves backups when applying MQ patches
252 r"\.orig$",
253 r"\.rej$",
254 ))
255
256 def __init__(self, gem5, output_dir, config_tuple,
257 timeout=None,
258 skip=False, skip_diff_out=False, skip_diff_stat=False):
259
260 super(ClassicTest, self).__init__("/".join(config_tuple))
261
262 ct = config_tuple
263
264 self.gem5 = os.path.abspath(gem5)
265 self.script = os.path.join(_test_base, "run.py")
266 self.config_tuple = ct
267 self.timeout = timeout
268
269 self.output_dir = output_dir
270 self.ref_dir = os.path.join(_test_base,
271 ct.category, ct.mode, ct.workload,
272 "ref", ct.isa, ct.os, ct.config)
273 self.skip_run = skip
274 self.skip_diff_out = skip or skip_diff_out
275 self.skip_diff_stat = skip or skip_diff_stat
276
277 def ref_files(self):
278 ref_dir = os.path.abspath(self.ref_dir)
279 for root, dirs, files in os.walk(ref_dir, topdown=False):
280 for f in files:
281 fpath = os.path.join(root[len(ref_dir) + 1:], f)
282 if fpath not in ClassicTest.ref_ignore_files:
283 yield fpath
284
285 def run_units(self):
286 args = [
287 self.script,
288 "/".join(self.config_tuple),
289 ]
290
291 return [
292 RunGem5(self.gem5, args,
293 ref_dir=self.ref_dir, test_dir=self.output_dir,
294 skip=self.skip_run),
295 ]
296
297 def verify_units(self):
298 ref_files = set(self.ref_files())
299 units = []
300 if "stats.txt" in ref_files:
301 units.append(
302 DiffStatFile(ref_dir=self.ref_dir, test_dir=self.output_dir,
303 skip=self.skip_diff_stat))
304 units += [
305 DiffOutFile(f,
306 ref_dir=self.ref_dir, test_dir=self.output_dir,
307 skip=self.skip_diff_out)
308 for f in ref_files if f not in ClassicTest.diff_ignore_files
309 ]
310
311 return units
312
313 def update_ref(self):
314 for fname in self.ref_files():
315 shutil.copy(
316 os.path.join(self.output_dir, fname),
317 os.path.join(self.ref_dir, fname))
318
319def parse_test_filter(test_filter):
320 wildcards = ("", "*")
321
322 _filter = list(test_filter.split("/"))
323 if len(_filter) > 3:
324 raise RuntimeError("Illegal test filter string")
325 _filter += [ "", ] * (3 - len(_filter))
326
327 isa, cat, mode = _filter
328
329 if isa in wildcards:
330 raise RuntimeError("No ISA specified")
331
332 cat = all_categories if cat in wildcards else (cat, )
333 mode = all_modes if mode in wildcards else (mode, )
334
335 return isa, cat, mode
336
337def get_tests(isa,
338 categories=all_categories, modes=all_modes,
339 ruby_protocol=None, gpu_isa=None):
340
341 # Generate a list of candidate configs
342 configs = list(arch_configs.get((isa, gpu_isa), []))
343
344 if (isa, gpu_isa) == ("x86", "hsail"):
345 if ruby_protocol == "GPU_RfO":
346 configs += ['gpu-randomtest']
347 else:
348 configs += generic_configs
349
|