1# Copyright (c) 2017 Mark D. Hill and David A. Wood 2# All rights reserved. 3# 4# Redistribution and use in source and binary forms, with or without 5# modification, are permitted provided that the following conditions are 6# met: redistributions of source code must retain the above copyright 7# notice, this list of conditions and the following disclaimer; 8# redistributions in binary form must reproduce the above copyright 9# notice, this list of conditions and the following disclaimer in the 10# documentation and/or other materials provided with the distribution; 11# neither the name of the copyright holders nor the names of its 12# contributors may be used to endorse or promote products derived from 13# this software without specific prior written permission. 14# 15# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS 16# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT 17# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR 18# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT 19# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, 20# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT 21# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, 22# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY 23# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT 24# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE 25# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 26# 27# Authors: Sean Wilson 28 29import os 30import tempfile 31import shutil
| 13# Copyright (c) 2017 Mark D. Hill and David A. Wood 14# All rights reserved. 15# 16# Redistribution and use in source and binary forms, with or without 17# modification, are permitted provided that the following conditions are 18# met: redistributions of source code must retain the above copyright 19# notice, this list of conditions and the following disclaimer; 20# redistributions in binary form must reproduce the above copyright 21# notice, this list of conditions and the following disclaimer in the 22# documentation and/or other materials provided with the distribution; 23# neither the name of the copyright holders nor the names of its 24# contributors may be used to endorse or promote products derived from 25# this software without specific prior written permission. 26# 27# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS 28# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT 29# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR 30# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT 31# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, 32# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT 33# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, 34# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY 35# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT 36# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE 37# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 38# 39# Authors: Sean Wilson 40 41import os 42import tempfile 43import shutil
|
63class SConsFixture(Fixture): 64 ''' 65 Fixture will wait until all SCons targets are collected and tests are 66 about to be ran, then will invocate a single instance of SCons for all 67 targets. 68 69 :param directory: The directory which scons will -C (cd) into before 70 executing. If None is provided, will choose the config base_dir. 71 ''' 72 def __init__(self, directory=None, target_class=None, options=[]): 73 self.directory = directory if directory else config.base_dir 74 self.target_class = target_class if target_class else SConsTarget 75 self.threads = config.threads 76 self.targets = set() 77 self.options = options 78 super(SConsFixture, self).__init__() 79 80 def setup(self, testitem): 81 if config.skip_build: 82 return 83 84 command = [ 85 'scons', '-C', self.directory, 86 '-j', str(self.threads), 87 '--ignore-style' 88 ] 89 90 if not self.targets: 91 log.test_log.warn( 92 'No SCons targets specified, this will' 93 ' build the default all target.\n' 94 'This is likely unintended, and you' 95 ' may wish to kill testlib and reconfigure.') 96 else: 97 log.test_log.message( 98 'Building the following targets.' 99 ' This may take a while.') 100 log.test_log.message('%s' % (', '.join(self.targets))) 101 log.test_log.message( 102 "You may want to run with only a single ISA" 103 "(--isa=), use --skip-build, or use 'rerun'.") 104 105 command.extend(self.targets) 106 if self.options: 107 command.extend(self.options) 108 log_call(log.test_log, command) 109 110 111class SConsTarget(Fixture): 112 # The singleton scons fixture we'll use for all targets. 113 default_scons_invocation = None 114 115 def __init__(self, target, build_dir=None, invocation=None): 116 ''' 117 Represents a target to be built by an 'invocation' of scons. 118 119 :param target: The target known to scons. 120 121 :param build_dir: The 'build' directory path which will be prepended 122 to the target name. 123 124 :param invocation: Represents an invocation of scons which we will 125 automatically attach this target to. If None provided, uses the 126 main 'scons' invocation. 127 ''' 128 129 if build_dir is None: 130 build_dir = config.build_dir 131 self.target = os.path.join(build_dir, target) 132 super(SConsTarget, self).__init__(name=target) 133 134 if invocation is None: 135 if self.default_scons_invocation is None: 136 SConsTarget.default_scons_invocation = SConsFixture() 137 globalfixture(SConsTarget.default_scons_invocation) 138 139 invocation = self.default_scons_invocation 140 self.invocation = invocation 141 142 def schedule_finalized(self, schedule): 143 self.invocation.targets.add(self.target) 144 return Fixture.schedule_finalized(self, schedule) 145 146class Gem5Fixture(SConsTarget): 147 other_invocations = {} # stores scons invocations other than the default 148 149 def __init__(self, isa, variant, protocol=None): 150 if protocol: 151 # When specifying an non-default protocol, we have to make a 152 # separate scons invocation with specific parameters. However, if 153 # more than one tests needs the same target, we need to make sure 154 # that we don't call scons too many times. 155 target_dir = isa.upper()+'-'+protocol 156 target = joinpath(target_dir, 'gem5.%s' % variant) 157 if target_dir in self.other_invocations.keys(): 158 invocation = self.other_invocations[target_dir] 159 else: 160 options = ['PROTOCOL='+protocol, '--default='+isa.upper()] 161 invocation = SConsFixture(options=options) 162 globalfixture(invocation) 163 Gem5Fixture.other_invocations[target_dir] = invocation 164 else: 165 target = joinpath(isa.upper(), 'gem5.%s' % variant) 166 invocation = None # use default 167 super(Gem5Fixture, self).__init__(target, invocation=invocation) 168 169 self.name = constants.gem5_binary_fixture_name 170 self.path = self.target 171 self.isa = isa 172 self.variant = variant 173 174 175class MakeFixture(Fixture): 176 def __init__(self, directory, *args, **kwargs): 177 name = 'make -C %s' % directory 178 super(MakeFixture, self).__init__(build_once=True, lazy_init=False, 179 name=name, 180 *args, **kwargs) 181 self.targets = [] 182 self.directory = directory 183 184 def setup(self): 185 super(MakeFixture, self).setup() 186 targets = set(self.required_by) 187 command = ['make', '-C', self.directory] 188 command.extend([target.target for target in targets]) 189 log_call(command) 190 191 192class MakeTarget(Fixture): 193 def __init__(self, target, make_fixture=None, *args, **kwargs): 194 ''' 195 :param make_fixture: The make invocation we will be attached to. 196 Since we don't have a single global instance of make in gem5 like we do 197 scons we need to know what invocation to attach to. If none given, 198 creates its own. 199 ''' 200 super(MakeTarget, self).__init__(name=target, *args, **kwargs) 201 self.target = self.name 202 203 if make_fixture is None: 204 make_fixture = MakeFixture( 205 absdirpath(target), 206 lazy_init=True, 207 build_once=False) 208 209 self.make_fixture = make_fixture 210 211 # Add our self to the required targets of the main MakeFixture 212 self.require(self.make_fixture) 213 214 def setup(self, testitem): 215 super(MakeTarget, self).setup() 216 self.make_fixture.setup() 217 return self 218 219class TestProgram(MakeTarget): 220 def __init__(self, program, isa, os, recompile=False): 221 make_dir = joinpath('test-progs', program) 222 make_fixture = MakeFixture(make_dir) 223 target = joinpath('bin', isa, os, program) 224 super(TestProgram, self).__init__(target, make_fixture) 225 self.path = joinpath(make_dir, target) 226 self.recompile = recompile 227 228 def setup(self, testitem): 229 # Check if the program exists if it does then only compile if 230 # recompile was given. 231 if self.recompile: 232 super(MakeTarget, self).setup() 233 elif not os.path.exists(self.path): 234 super(MakeTarget, self).setup() 235 236class DownloadedProgram(Fixture): 237 """ Like TestProgram, but checks the version in the gem5 binary repository 238 and downloads an updated version if it is needed. 239 """ 240 urlbase = "http://gem5.org/dist/current/" 241 242 def __init__(self, path, program, **kwargs): 243 """ 244 path: string 245 The path to the directory containing the binary relative to 246 $GEM5_BASE/tests 247 program: string 248 The name of the binary file 249 """ 250 super(DownloadedProgram, self).__init__("download-" + program, 251 build_once=True, **kwargs) 252 253 self.program_dir = path 254 relative_path = joinpath(self.program_dir, program) 255 self.url = self.urlbase + relative_path 256 self.path = os.path.realpath( 257 joinpath(absdirpath(__file__), '../', relative_path) 258 ) 259 260 def _download(self): 261 import urllib 262 import errno 263 log.test_log.debug("Downloading " + self.url + " to " + self.path) 264 if not os.path.exists(self.program_dir): 265 try: 266 os.makedirs(self.program_dir) 267 except OSError as e: 268 if e.errno != errno.EEXIST: 269 raise 270 urllib.urlretrieve(self.url, self.path) 271 272 def _getremotetime(self): 273 import urllib2, datetime, time 274 import _strptime # Needed for python threading bug 275 276 u = urllib2.urlopen(self.url) 277 return time.mktime(datetime.datetime.strptime( \ 278 u.info().getheaders("Last-Modified")[0], 279 "%a, %d %b %Y %X GMT").timetuple()) 280 281 def setup(self, testitem): 282 import urllib2 283 # Check to see if there is a file downloaded 284 if not os.path.exists(self.path): 285 self._download() 286 else: 287 try: 288 t = self._getremotetime() 289 except urllib2.URLError: 290 # Problem checking the server, use the old files. 291 log.debug("Could not contact server. Binaries may be old.") 292 return 293 # If the server version is more recent, download it 294 if t > os.path.getmtime(self.path): 295 self._download()
| 115class SConsFixture(Fixture): 116 ''' 117 Fixture will wait until all SCons targets are collected and tests are 118 about to be ran, then will invocate a single instance of SCons for all 119 targets. 120 121 :param directory: The directory which scons will -C (cd) into before 122 executing. If None is provided, will choose the config base_dir. 123 ''' 124 def __init__(self, directory=None, target_class=None, options=[]): 125 self.directory = directory if directory else config.base_dir 126 self.target_class = target_class if target_class else SConsTarget 127 self.threads = config.threads 128 self.targets = set() 129 self.options = options 130 super(SConsFixture, self).__init__() 131 132 def setup(self, testitem): 133 if config.skip_build: 134 return 135 136 command = [ 137 'scons', '-C', self.directory, 138 '-j', str(self.threads), 139 '--ignore-style' 140 ] 141 142 if not self.targets: 143 log.test_log.warn( 144 'No SCons targets specified, this will' 145 ' build the default all target.\n' 146 'This is likely unintended, and you' 147 ' may wish to kill testlib and reconfigure.') 148 else: 149 log.test_log.message( 150 'Building the following targets.' 151 ' This may take a while.') 152 log.test_log.message('%s' % (', '.join(self.targets))) 153 log.test_log.message( 154 "You may want to run with only a single ISA" 155 "(--isa=), use --skip-build, or use 'rerun'.") 156 157 command.extend(self.targets) 158 if self.options: 159 command.extend(self.options) 160 log_call(log.test_log, command) 161 162 163class SConsTarget(Fixture): 164 # The singleton scons fixture we'll use for all targets. 165 default_scons_invocation = None 166 167 def __init__(self, target, build_dir=None, invocation=None): 168 ''' 169 Represents a target to be built by an 'invocation' of scons. 170 171 :param target: The target known to scons. 172 173 :param build_dir: The 'build' directory path which will be prepended 174 to the target name. 175 176 :param invocation: Represents an invocation of scons which we will 177 automatically attach this target to. If None provided, uses the 178 main 'scons' invocation. 179 ''' 180 181 if build_dir is None: 182 build_dir = config.build_dir 183 self.target = os.path.join(build_dir, target) 184 super(SConsTarget, self).__init__(name=target) 185 186 if invocation is None: 187 if self.default_scons_invocation is None: 188 SConsTarget.default_scons_invocation = SConsFixture() 189 globalfixture(SConsTarget.default_scons_invocation) 190 191 invocation = self.default_scons_invocation 192 self.invocation = invocation 193 194 def schedule_finalized(self, schedule): 195 self.invocation.targets.add(self.target) 196 return Fixture.schedule_finalized(self, schedule) 197 198class Gem5Fixture(SConsTarget): 199 other_invocations = {} # stores scons invocations other than the default 200 201 def __init__(self, isa, variant, protocol=None): 202 if protocol: 203 # When specifying an non-default protocol, we have to make a 204 # separate scons invocation with specific parameters. However, if 205 # more than one tests needs the same target, we need to make sure 206 # that we don't call scons too many times. 207 target_dir = isa.upper()+'-'+protocol 208 target = joinpath(target_dir, 'gem5.%s' % variant) 209 if target_dir in self.other_invocations.keys(): 210 invocation = self.other_invocations[target_dir] 211 else: 212 options = ['PROTOCOL='+protocol, '--default='+isa.upper()] 213 invocation = SConsFixture(options=options) 214 globalfixture(invocation) 215 Gem5Fixture.other_invocations[target_dir] = invocation 216 else: 217 target = joinpath(isa.upper(), 'gem5.%s' % variant) 218 invocation = None # use default 219 super(Gem5Fixture, self).__init__(target, invocation=invocation) 220 221 self.name = constants.gem5_binary_fixture_name 222 self.path = self.target 223 self.isa = isa 224 self.variant = variant 225 226 227class MakeFixture(Fixture): 228 def __init__(self, directory, *args, **kwargs): 229 name = 'make -C %s' % directory 230 super(MakeFixture, self).__init__(build_once=True, lazy_init=False, 231 name=name, 232 *args, **kwargs) 233 self.targets = [] 234 self.directory = directory 235 236 def setup(self): 237 super(MakeFixture, self).setup() 238 targets = set(self.required_by) 239 command = ['make', '-C', self.directory] 240 command.extend([target.target for target in targets]) 241 log_call(command) 242 243 244class MakeTarget(Fixture): 245 def __init__(self, target, make_fixture=None, *args, **kwargs): 246 ''' 247 :param make_fixture: The make invocation we will be attached to. 248 Since we don't have a single global instance of make in gem5 like we do 249 scons we need to know what invocation to attach to. If none given, 250 creates its own. 251 ''' 252 super(MakeTarget, self).__init__(name=target, *args, **kwargs) 253 self.target = self.name 254 255 if make_fixture is None: 256 make_fixture = MakeFixture( 257 absdirpath(target), 258 lazy_init=True, 259 build_once=False) 260 261 self.make_fixture = make_fixture 262 263 # Add our self to the required targets of the main MakeFixture 264 self.require(self.make_fixture) 265 266 def setup(self, testitem): 267 super(MakeTarget, self).setup() 268 self.make_fixture.setup() 269 return self 270 271class TestProgram(MakeTarget): 272 def __init__(self, program, isa, os, recompile=False): 273 make_dir = joinpath('test-progs', program) 274 make_fixture = MakeFixture(make_dir) 275 target = joinpath('bin', isa, os, program) 276 super(TestProgram, self).__init__(target, make_fixture) 277 self.path = joinpath(make_dir, target) 278 self.recompile = recompile 279 280 def setup(self, testitem): 281 # Check if the program exists if it does then only compile if 282 # recompile was given. 283 if self.recompile: 284 super(MakeTarget, self).setup() 285 elif not os.path.exists(self.path): 286 super(MakeTarget, self).setup() 287 288class DownloadedProgram(Fixture): 289 """ Like TestProgram, but checks the version in the gem5 binary repository 290 and downloads an updated version if it is needed. 291 """ 292 urlbase = "http://gem5.org/dist/current/" 293 294 def __init__(self, path, program, **kwargs): 295 """ 296 path: string 297 The path to the directory containing the binary relative to 298 $GEM5_BASE/tests 299 program: string 300 The name of the binary file 301 """ 302 super(DownloadedProgram, self).__init__("download-" + program, 303 build_once=True, **kwargs) 304 305 self.program_dir = path 306 relative_path = joinpath(self.program_dir, program) 307 self.url = self.urlbase + relative_path 308 self.path = os.path.realpath( 309 joinpath(absdirpath(__file__), '../', relative_path) 310 ) 311 312 def _download(self): 313 import urllib 314 import errno 315 log.test_log.debug("Downloading " + self.url + " to " + self.path) 316 if not os.path.exists(self.program_dir): 317 try: 318 os.makedirs(self.program_dir) 319 except OSError as e: 320 if e.errno != errno.EEXIST: 321 raise 322 urllib.urlretrieve(self.url, self.path) 323 324 def _getremotetime(self): 325 import urllib2, datetime, time 326 import _strptime # Needed for python threading bug 327 328 u = urllib2.urlopen(self.url) 329 return time.mktime(datetime.datetime.strptime( \ 330 u.info().getheaders("Last-Modified")[0], 331 "%a, %d %b %Y %X GMT").timetuple()) 332 333 def setup(self, testitem): 334 import urllib2 335 # Check to see if there is a file downloaded 336 if not os.path.exists(self.path): 337 self._download() 338 else: 339 try: 340 t = self._getremotetime() 341 except urllib2.URLError: 342 # Problem checking the server, use the old files. 343 log.debug("Could not contact server. Binaries may be old.") 344 return 345 # If the server version is more recent, download it 346 if t > os.path.getmtime(self.path): 347 self._download()
|