1# -*- mode:python -*-
2#
3# Copyright (c) 2016 ARM Limited
4# All rights reserved
5#
6# The license below extends only to copyright in the software and shall
7# not be construed as granting a license to any other intellectual
8# property including but not limited to intellectual property relating
9# to a hardware implementation of the functionality of the software
10# licensed hereunder. You may use the software subject to the license
11# terms below provided that you ensure that this notice is replicated
12# unmodified and in its entirety in all distributions of the software,
13# modified or unmodified, in source code or in binary form.
14#
15# Copyright (c) 2004-2006 The Regents of The University of Michigan
16# All rights reserved.
17#
18# Redistribution and use in source and binary forms, with or without
19# modification, are permitted provided that the following conditions are
20# met: redistributions of source code must retain the above copyright
21# notice, this list of conditions and the following disclaimer;
22# redistributions in binary form must reproduce the above copyright
23# notice, this list of conditions and the following disclaimer in the
24# documentation and/or other materials provided with the distribution;
25# neither the name of the copyright holders nor the names of its
26# contributors may be used to endorse or promote products derived from
27# this software without specific prior written permission.
28#
29# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
30# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
31# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
32# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
33# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
34# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
35# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
36# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
37# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
38# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
39# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
40#
41# Authors: Steve Reinhardt
42# Kevin Lim
43# Andreas Sandberg
44
45from SCons.Script.SConscript import SConsEnvironment
46import os
47import pickle
48import sys
49
50sys.path.insert(0, Dir(".").srcnode().abspath)
51import testing.tests as tests
52import testing.results as results
53
54Import('env')
55
56# get the termcap from the environment
57termcap = env['TERMCAP']
58
59# Dict that accumulates lists of tests by category (quick, medium, long)
60env.Tests = {}
61gpu_isa = env['TARGET_GPU_ISA'] if env['BUILD_GPU'] else None
62for cat in tests.all_categories:
63 env.Tests[cat] = tuple(
64 tests.get_tests(env["TARGET_ISA"],
65 categories=(cat, ),
66 ruby_protocol=env["PROTOCOL"],
67 gpu_isa=gpu_isa))
68
69def color_message(color, msg):
70 return color + msg + termcap.Normal
71
72def run_test(target, source, env):
73 """Run a test and produce results as a pickle file.
74
75 Targets are as follows:
76 target[0] : Pickle file
77
78 Sources are:
79 source[0] : gem5 binary
80 source[1] : tests/run.py script
81 source[2:] : reference files
82
83 """
84 tgt_dir = os.path.dirname(str(target[0]))
85 config = tests.ClassicConfig(*tgt_dir.split('/')[-6:])
86 test = tests.ClassicTest(source[0].abspath, tgt_dir, config,
87 timeout=5*60*60,
88 skip_diff_out=True)
89
90 for ref in test.ref_files():
91 out_file = os.path.join(tgt_dir, ref)
92 if os.path.exists(out_file):
93 env.Execute(Delete(out_file))
94
95 with open(target[0].abspath, "wb") as fout:
96 formatter = results.Pickle(fout=fout)
97 formatter.dump_suites([ test.run() ])
98
99 return 0
100
101def run_test_string(target, source, env):
102 return env.subst("Running test in ${TARGETS[0].dir}.",
103 target=target, source=source)
104
105testAction = env.Action(run_test, run_test_string)
106
107def print_test(target, source, env):
108 """Run a test and produce results as a pickle file.
109
110 Targets are as follows:
111 target[*] : Dummy targets
112
113 Sources are:
114 source[0] : Pickle file
115
116 """
117 with open(source[0].abspath, "rb") as fin:
118 result = pickle.load(fin)
119
120 assert len(result) == 1
121 result = result[0]
122
| 1# -*- mode:python -*-
2#
3# Copyright (c) 2016 ARM Limited
4# All rights reserved
5#
6# The license below extends only to copyright in the software and shall
7# not be construed as granting a license to any other intellectual
8# property including but not limited to intellectual property relating
9# to a hardware implementation of the functionality of the software
10# licensed hereunder. You may use the software subject to the license
11# terms below provided that you ensure that this notice is replicated
12# unmodified and in its entirety in all distributions of the software,
13# modified or unmodified, in source code or in binary form.
14#
15# Copyright (c) 2004-2006 The Regents of The University of Michigan
16# All rights reserved.
17#
18# Redistribution and use in source and binary forms, with or without
19# modification, are permitted provided that the following conditions are
20# met: redistributions of source code must retain the above copyright
21# notice, this list of conditions and the following disclaimer;
22# redistributions in binary form must reproduce the above copyright
23# notice, this list of conditions and the following disclaimer in the
24# documentation and/or other materials provided with the distribution;
25# neither the name of the copyright holders nor the names of its
26# contributors may be used to endorse or promote products derived from
27# this software without specific prior written permission.
28#
29# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
30# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
31# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
32# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
33# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
34# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
35# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
36# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
37# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
38# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
39# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
40#
41# Authors: Steve Reinhardt
42# Kevin Lim
43# Andreas Sandberg
44
45from SCons.Script.SConscript import SConsEnvironment
46import os
47import pickle
48import sys
49
50sys.path.insert(0, Dir(".").srcnode().abspath)
51import testing.tests as tests
52import testing.results as results
53
54Import('env')
55
56# get the termcap from the environment
57termcap = env['TERMCAP']
58
59# Dict that accumulates lists of tests by category (quick, medium, long)
60env.Tests = {}
61gpu_isa = env['TARGET_GPU_ISA'] if env['BUILD_GPU'] else None
62for cat in tests.all_categories:
63 env.Tests[cat] = tuple(
64 tests.get_tests(env["TARGET_ISA"],
65 categories=(cat, ),
66 ruby_protocol=env["PROTOCOL"],
67 gpu_isa=gpu_isa))
68
69def color_message(color, msg):
70 return color + msg + termcap.Normal
71
72def run_test(target, source, env):
73 """Run a test and produce results as a pickle file.
74
75 Targets are as follows:
76 target[0] : Pickle file
77
78 Sources are:
79 source[0] : gem5 binary
80 source[1] : tests/run.py script
81 source[2:] : reference files
82
83 """
84 tgt_dir = os.path.dirname(str(target[0]))
85 config = tests.ClassicConfig(*tgt_dir.split('/')[-6:])
86 test = tests.ClassicTest(source[0].abspath, tgt_dir, config,
87 timeout=5*60*60,
88 skip_diff_out=True)
89
90 for ref in test.ref_files():
91 out_file = os.path.join(tgt_dir, ref)
92 if os.path.exists(out_file):
93 env.Execute(Delete(out_file))
94
95 with open(target[0].abspath, "wb") as fout:
96 formatter = results.Pickle(fout=fout)
97 formatter.dump_suites([ test.run() ])
98
99 return 0
100
101def run_test_string(target, source, env):
102 return env.subst("Running test in ${TARGETS[0].dir}.",
103 target=target, source=source)
104
105testAction = env.Action(run_test, run_test_string)
106
107def print_test(target, source, env):
108 """Run a test and produce results as a pickle file.
109
110 Targets are as follows:
111 target[*] : Dummy targets
112
113 Sources are:
114 source[0] : Pickle file
115
116 """
117 with open(source[0].abspath, "rb") as fin:
118 result = pickle.load(fin)
119
120 assert len(result) == 1
121 result = result[0]
122
|
123 run = result.results[0]
124 assert run.name == "gem5"
125
| |
126 formatter = None
| 123 formatter = None
|
127 if not run:
128 status = color_message(termcap.Red, "FAILED!")
129 formatter = results.Text()
130 elif run.skipped():
| 124 if result.skipped():
|
131 status = color_message(termcap.Cyan, "skipped.")
| 125 status = color_message(termcap.Cyan, "skipped.")
|
| 126 elif result.changed():
127 status = color_message(termcap.Yellow, "CHANGED!")
128 formatter = results.Text()
|
132 elif result:
133 status = color_message(termcap.Green, "passed.")
134 else:
| 129 elif result:
130 status = color_message(termcap.Green, "passed.")
131 else:
|
135 status = color_message(termcap.Yellow, "CHANGED!")
| 132 status = color_message(termcap.Red, "FAILED!")
|
136 formatter = results.Text()
137
138 if formatter:
139 formatter.dump_suites([result])
140
141 print "***** %s: %s" % (source[0].dir, status)
142 return 0
143
144printAction = env.Action(print_test, strfunction=None)
145
146def update_test(target, source, env):
147 """Update test reference data
148
149 Targets are as follows:
150 target[0] : Dummy file
151
152 Sources are:
153 source[0] : Pickle file
154 """
155
156 src_dir = os.path.dirname(str(source[0]))
157 config = tests.ClassicConfig(*src_dir.split('/')[-6:])
158 test = tests.ClassicTest(source[0].abspath, src_dir, config)
159 ref_dir = test.ref_dir
160
161 with open(source[0].abspath, "rb") as fin:
162 result = pickle.load(fin)
163
164 assert len(result) == 1
165 result = result[0]
166
| 133 formatter = results.Text()
134
135 if formatter:
136 formatter.dump_suites([result])
137
138 print "***** %s: %s" % (source[0].dir, status)
139 return 0
140
141printAction = env.Action(print_test, strfunction=None)
142
143def update_test(target, source, env):
144 """Update test reference data
145
146 Targets are as follows:
147 target[0] : Dummy file
148
149 Sources are:
150 source[0] : Pickle file
151 """
152
153 src_dir = os.path.dirname(str(source[0]))
154 config = tests.ClassicConfig(*src_dir.split('/')[-6:])
155 test = tests.ClassicTest(source[0].abspath, src_dir, config)
156 ref_dir = test.ref_dir
157
158 with open(source[0].abspath, "rb") as fin:
159 result = pickle.load(fin)
160
161 assert len(result) == 1
162 result = result[0]
163
|
167 run = result.results[0]
168 assert run.name == "gem5"
169
170 if run.skipped():
| 164 if result.skipped():
|
171 print "*** %s: %s: Test skipped, not updating." % (
172 source[0].dir, color_message(termcap.Yellow, "WARNING"), )
173 return 0
174 elif result:
175 print "*** %s: %s: Test successful, not updating." % (
176 source[0].dir, color_message(termcap.Green, "skipped"), )
177 return 0
| 165 print "*** %s: %s: Test skipped, not updating." % (
166 source[0].dir, color_message(termcap.Yellow, "WARNING"), )
167 return 0
168 elif result:
169 print "*** %s: %s: Test successful, not updating." % (
170 source[0].dir, color_message(termcap.Green, "skipped"), )
171 return 0
|
178 elif not run.success():
| 172 elif result.failed_run():
|
179 print "*** %s: %s: Test failed, not updating." % (
180 source[0].dir, color_message(termcap.Red, "ERROR"), )
181 return 1
182
183 print "** Updating %s" % (test, )
184 test.update_ref()
185
186 return 0
187
188def update_test_string(target, source, env):
189 return env.subst("Updating ${SOURCES[0].dir}",
190 target=target, source=source)
191
192updateAction = env.Action(update_test, update_test_string)
193
194def test_builder(test_tuple):
195 """Define a test."""
196
197 out_dir = "/".join(test_tuple)
198 binary = env.M5Binary.abspath
199 test = tests.ClassicTest(binary, out_dir, test_tuple)
200
201 def tgt(name):
202 return os.path.join(out_dir, name)
203
204 def ref(name):
205 return os.path.join(test.ref_dir, name)
206
207 pickle_file = tgt("status.pickle")
208 targets = [
209 pickle_file,
210 ]
211
212 sources = [
213 env.M5Binary,
214 "run.py",
215 ] + [ ref(f) for f in test.ref_files() ]
216
217 env.Command(targets, sources, testAction)
218
219 # phony target to echo status
220 if GetOption('update_ref'):
221 p = env.Command(tgt("_update"), [pickle_file], updateAction)
222 else:
223 p = env.Command(tgt("_print"), [pickle_file], printAction)
224
225 env.AlwaysBuild(p)
226
227def list_tests(target, source, env):
228 """Create a list of tests
229
230 Targets are as follows:
231 target[0] : List file (e.g., tests/opt/all.list, tests/opt/quick.list)
232
233 Sources are: -
234
235 """
236
237 tgt_name = os.path.basename(str(target[0]))
238 base, ext = os.path.splitext(tgt_name)
239 categories = tests.all_categories if base == "all" else (base, )
240
241 with open(target[0].abspath, "w") as fout:
242 for cat in categories:
243 for test in env.Tests[cat]:
244 print >> fout,"/".join(test)
245
246 return 0
247
248testListAction = env.Action(list_tests, strfunction=None)
249
250env.Command("all.list", tuple(), testListAction)
251for cat, test_list in env.Tests.items():
252 env.Command("%s.list" % cat, tuple(), testListAction)
253 for test in test_list:
254 test_builder(test)
| 173 print "*** %s: %s: Test failed, not updating." % (
174 source[0].dir, color_message(termcap.Red, "ERROR"), )
175 return 1
176
177 print "** Updating %s" % (test, )
178 test.update_ref()
179
180 return 0
181
182def update_test_string(target, source, env):
183 return env.subst("Updating ${SOURCES[0].dir}",
184 target=target, source=source)
185
186updateAction = env.Action(update_test, update_test_string)
187
188def test_builder(test_tuple):
189 """Define a test."""
190
191 out_dir = "/".join(test_tuple)
192 binary = env.M5Binary.abspath
193 test = tests.ClassicTest(binary, out_dir, test_tuple)
194
195 def tgt(name):
196 return os.path.join(out_dir, name)
197
198 def ref(name):
199 return os.path.join(test.ref_dir, name)
200
201 pickle_file = tgt("status.pickle")
202 targets = [
203 pickle_file,
204 ]
205
206 sources = [
207 env.M5Binary,
208 "run.py",
209 ] + [ ref(f) for f in test.ref_files() ]
210
211 env.Command(targets, sources, testAction)
212
213 # phony target to echo status
214 if GetOption('update_ref'):
215 p = env.Command(tgt("_update"), [pickle_file], updateAction)
216 else:
217 p = env.Command(tgt("_print"), [pickle_file], printAction)
218
219 env.AlwaysBuild(p)
220
221def list_tests(target, source, env):
222 """Create a list of tests
223
224 Targets are as follows:
225 target[0] : List file (e.g., tests/opt/all.list, tests/opt/quick.list)
226
227 Sources are: -
228
229 """
230
231 tgt_name = os.path.basename(str(target[0]))
232 base, ext = os.path.splitext(tgt_name)
233 categories = tests.all_categories if base == "all" else (base, )
234
235 with open(target[0].abspath, "w") as fout:
236 for cat in categories:
237 for test in env.Tests[cat]:
238 print >> fout,"/".join(test)
239
240 return 0
241
242testListAction = env.Action(list_tests, strfunction=None)
243
244env.Command("all.list", tuple(), testListAction)
245for cat, test_list in env.Tests.items():
246 env.Command("%s.list" % cat, tuple(), testListAction)
247 for test in test_list:
248 test_builder(test)
|