1# Copyright (c) 2019 ARM Limited 2# All rights reserved 3# 4# The license below extends only to copyright in the software and shall 5# not be construed as granting a license to any other intellectual 6# property including but not limited to intellectual property relating 7# to a hardware implementation of the functionality of the software 8# licensed hereunder. You may use the software subject to the license 9# terms below provided that you ensure that this notice is replicated 10# unmodified and in its entirety in all distributions of the software, 11# modified or unmodified, in source code or in binary form. 12# 13# Copyright (c) 2017 Mark D. Hill and David A. Wood 14# All rights reserved. 15# 16# Redistribution and use in source and binary forms, with or without 17# modification, are permitted provided that the following conditions are 18# met: redistributions of source code must retain the above copyright 19# notice, this list of conditions and the following disclaimer; 20# redistributions in binary form must reproduce the above copyright 21# notice, this list of conditions and the following disclaimer in the 22# documentation and/or other materials provided with the distribution; 23# neither the name of the copyright holders nor the names of its 24# contributors may be used to endorse or promote products derived from 25# this software without specific prior written permission. 26# 27# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS 28# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT 29# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR 30# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT 31# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, 32# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT 33# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, 34# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY 35# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT 36# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE 37# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 38# 39# Authors: Sean Wilson 40# Nikos Nikoleris 41 42import os 43import tempfile 44import shutil 45import threading 46import urllib 47import urllib2 48 49from testlib.fixture import Fixture 50from testlib.config import config, constants 51from testlib.helper import log_call, cacheresult, joinpath, absdirpath 52import testlib.log as log 53 54 55class VariableFixture(Fixture): 56 def __init__(self, value=None, name=None): 57 super(VariableFixture, self).__init__(name=name) 58 self.value = value 59 60 61class TempdirFixture(Fixture): 62 def __init__(self): 63 self.path = None 64 super(TempdirFixture, self).__init__( 65 name=constants.tempdir_fixture_name) 66 67 def setup(self, testitem): 68 self.path = tempfile.mkdtemp(prefix='gem5out') 69 70 def teardown(self, testitem): 71 if self.path is not None: 72 shutil.rmtree(self.path) 73 74 def skip_cleanup(self): 75 # Set path to none so it's not deleted 76 self.path = None 77 78class UniqueFixture(Fixture): 79 ''' 80 Base class for fixtures that generate a target in the 81 filesystem. If the same fixture is used by more than one 82 test/suite, rather than creating a copy of the fixture, it returns 83 the same object and makes sure that setup is only executed 84 once. Devired classses should override the _init and _setup 85 functions. 86 87 :param target: The absolute path of the target in the filesystem. 88 89 ''' 90 fixtures = {} 91 92 def __new__(cls, target): 93 if target in cls.fixtures: 94 obj = cls.fixtures[target] 95 else: 96 obj = super(UniqueFixture, cls).__new__(cls) 97 obj.lock = threading.Lock() 98 obj.target = target 99 cls.fixtures[target] = obj 100 return obj 101 102 def __init__(self, *args, **kwargs): 103 with self.lock: 104 if hasattr(self, '_init_done'): 105 return 106 super(UniqueFixture, self).__init__(self, **kwargs) 107 self._init(*args, **kwargs) 108 self._init_done = True 109 110 def setup(self, testitem): 111 with self.lock: 112 if hasattr(self, '_setup_done'): 113 return 114 self._setup_done = True 115 self._setup(testitem) 116 117 118class SConsFixture(UniqueFixture): 119 ''' 120 Fixture will wait until all SCons targets are collected and tests are 121 about to be ran, then will invocate a single instance of SCons for all 122 targets. 123 124 :param directory: The directory which scons will -C (cd) into before 125 executing. If None is provided, will choose the config base_dir. 126 ''' 127 128 def __new__(cls, target): 129 obj = super(SConsFixture, cls).__new__(cls, target) 130 return obj 131 132 def _setup(self, testitem): 133 if config.skip_build: 134 return 135 136 command = [ 137 'scons', '-C', self.directory, 138 '-j', str(config.threads), 139 '--ignore-style' 140 ] 141 142 if not self.targets: 143 log.test_log.warn( 144 'No SCons targets specified, this will' 145 ' build the default all target.\n' 146 'This is likely unintended, and you' 147 ' may wish to kill testlib and reconfigure.') 148 else: 149 log.test_log.message( 150 'Building the following targets.' 151 ' This may take a while.') 152 log.test_log.message('%s' % (', '.join(self.targets))) 153 log.test_log.message( 154 "You may want to run with only a single ISA" 155 "(--isa=), use --skip-build, or use 'rerun'.") 156 157 command.extend(self.targets) 158 if self.options: 159 command.extend(self.options) 160 log_call(log.test_log, command) 161 162class Gem5Fixture(SConsFixture): 163 def __new__(cls, isa, variant, protocol=None): 164 target_dir = joinpath(config.build_dir, isa.upper()) 165 if protocol: 166 target_dir += '_' + protocol 167 target = joinpath(target_dir, 'gem5.%s' % variant) 168 obj = super(Gem5Fixture, cls).__new__(cls, target) 169 return obj 170 171 def _init(self, isa, variant, protocol=None): 172 self.name = constants.gem5_binary_fixture_name 173 174 self.targets = [self.target] 175 self.path = self.target 176 self.directory = config.base_dir 177 178 self.options = [] 179 if protocol: 180 self.options = [ '--default=' + isa.upper(), 181 'PROTOCOL=' + protocol ] 182 self.set_global() 183 184class MakeFixture(Fixture): 185 def __init__(self, directory, *args, **kwargs): 186 name = 'make -C %s' % directory 187 super(MakeFixture, self).__init__(build_once=True, lazy_init=False, 188 name=name, 189 *args, **kwargs) 190 self.targets = [] 191 self.directory = directory 192 193 def setup(self): 194 super(MakeFixture, self).setup() 195 targets = set(self.required_by) 196 command = ['make', '-C', self.directory] 197 command.extend([target.target for target in targets]) 198 log_call(command) 199 200 201class MakeTarget(Fixture): 202 def __init__(self, target, make_fixture=None, *args, **kwargs): 203 ''' 204 :param make_fixture: The make invocation we will be attached to. 205 Since we don't have a single global instance of make in gem5 like we do 206 scons we need to know what invocation to attach to. If none given, 207 creates its own. 208 ''' 209 super(MakeTarget, self).__init__(name=target, *args, **kwargs) 210 self.target = self.name 211 212 if make_fixture is None: 213 make_fixture = MakeFixture( 214 absdirpath(target), 215 lazy_init=True, 216 build_once=False) 217 218 self.make_fixture = make_fixture 219 220 # Add our self to the required targets of the main MakeFixture 221 self.require(self.make_fixture) 222 223 def setup(self, testitem): 224 super(MakeTarget, self).setup() 225 self.make_fixture.setup() 226 return self 227 228class TestProgram(MakeTarget): 229 def __init__(self, program, isa, os, recompile=False): 230 make_dir = joinpath('test-progs', program) 231 make_fixture = MakeFixture(make_dir) 232 target = joinpath('bin', isa, os, program) 233 super(TestProgram, self).__init__(target, make_fixture) 234 self.path = joinpath(make_dir, target) 235 self.recompile = recompile 236 237 def setup(self, testitem): 238 # Check if the program exists if it does then only compile if 239 # recompile was given. 240 if self.recompile: 241 super(MakeTarget, self).setup() 242 elif not os.path.exists(self.path): 243 super(MakeTarget, self).setup() 244 245class DownloadedProgram(UniqueFixture): 246 """ Like TestProgram, but checks the version in the gem5 binary repository 247 and downloads an updated version if it is needed. 248 """ 249 250 def __new__(cls, url, path, filename): 251 target = joinpath(path, filename) 252 return super(DownloadedProgram, cls).__new__(cls, target) 253 254 def _init(self, url, path, filename, **kwargs): 255 """ 256 url: string 257 The url of the archive 258 path: string 259 The absolute path of the directory containing the archive 260 filename: string 261 The name of the archive 262 """ 263 264 self.url = url 265 self.path = path 266 self.filename = joinpath(path, filename) 267 self.name = "Downloaded:" + self.filename 268 269 def _download(self): 270 import errno 271 log.test_log.debug("Downloading " + self.url + " to " + self.path) 272 if not os.path.exists(self.path): 273 try: 274 os.makedirs(self.path) 275 except OSError as e: 276 if e.errno != errno.EEXIST: 277 raise 278 urllib.urlretrieve(self.url, self.filename) 279 280 def _getremotetime(self): 281 import datetime, time 282 import _strptime # Needed for python threading bug 283 284 u = urllib2.urlopen(self.url) 285 return time.mktime(datetime.datetime.strptime( \ 286 u.info().getheaders("Last-Modified")[0], 287 "%a, %d %b %Y %X GMT").timetuple()) 288 289 def _setup(self, testitem): 290 # Check to see if there is a file downloaded 291 if not os.path.exists(self.filename): 292 self._download() 293 else: 294 try: 295 t = self._getremotetime() 296 except urllib2.URLError: 297 # Problem checking the server, use the old files. 298 log.test_log.debug("Could not contact server. Binaries may be old.") 299 return 300 # If the server version is more recent, download it 301 if t > os.path.getmtime(self.filename): 302 self._download() 303 304class DownloadedArchive(DownloadedProgram): 305 """ Like TestProgram, but checks the version in the gem5 binary repository 306 and downloads an updated version if it is needed. 307 """ 308 309 def _extract(self): 310 import tarfile 311 with tarfile.open(self.filename) as tf: 312 tf.extractall(self.path) 313 314 def _setup(self, testitem): 315 # Check to see if there is a file downloaded 316 if not os.path.exists(self.filename): 317 self._download() 318 self._extract() 319 else: 320 try: 321 t = self._getremotetime() 322 except urllib2.URLError: 323 # Problem checking the server, use the old files. 324 log.test_log.debug("Could not contact server. Binaries may be old.") 325 return 326 # If the server version is more recent, download it 327 if t > os.path.getmtime(self.filename): 328 self._download() 329 self._extract() 330