fixture.py revision 13851
112882Sspwilson2@wisc.edu# Copyright (c) 2017 Mark D. Hill and David A. Wood 212882Sspwilson2@wisc.edu# All rights reserved. 312882Sspwilson2@wisc.edu# 412882Sspwilson2@wisc.edu# Redistribution and use in source and binary forms, with or without 512882Sspwilson2@wisc.edu# modification, are permitted provided that the following conditions are 612882Sspwilson2@wisc.edu# met: redistributions of source code must retain the above copyright 712882Sspwilson2@wisc.edu# notice, this list of conditions and the following disclaimer; 812882Sspwilson2@wisc.edu# redistributions in binary form must reproduce the above copyright 912882Sspwilson2@wisc.edu# notice, this list of conditions and the following disclaimer in the 1012882Sspwilson2@wisc.edu# documentation and/or other materials provided with the distribution; 1112882Sspwilson2@wisc.edu# neither the name of the copyright holders nor the names of its 1212882Sspwilson2@wisc.edu# contributors may be used to endorse or promote products derived from 1312882Sspwilson2@wisc.edu# this software without specific prior written permission. 1412882Sspwilson2@wisc.edu# 1512882Sspwilson2@wisc.edu# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS 1612882Sspwilson2@wisc.edu# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT 1712882Sspwilson2@wisc.edu# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR 1812882Sspwilson2@wisc.edu# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT 1912882Sspwilson2@wisc.edu# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, 2012882Sspwilson2@wisc.edu# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT 2112882Sspwilson2@wisc.edu# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, 2212882Sspwilson2@wisc.edu# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY 2312882Sspwilson2@wisc.edu# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT 2412882Sspwilson2@wisc.edu# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE 2512882Sspwilson2@wisc.edu# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 2612882Sspwilson2@wisc.edu# 2712882Sspwilson2@wisc.edu# Authors: Sean Wilson 2812882Sspwilson2@wisc.edu 2912882Sspwilson2@wisc.eduimport os 3012882Sspwilson2@wisc.eduimport tempfile 3112882Sspwilson2@wisc.eduimport shutil 3212882Sspwilson2@wisc.edu 3312882Sspwilson2@wisc.edufrom testlib.fixture import Fixture, globalfixture 3412882Sspwilson2@wisc.edufrom testlib.config import config, constants 3512882Sspwilson2@wisc.edufrom testlib.helper import log_call, cacheresult, joinpath, absdirpath 3612882Sspwilson2@wisc.eduimport testlib.log as log 3712882Sspwilson2@wisc.edu 3812882Sspwilson2@wisc.edu 3912882Sspwilson2@wisc.educlass VariableFixture(Fixture): 4012882Sspwilson2@wisc.edu def __init__(self, value=None, name=None): 4112882Sspwilson2@wisc.edu super(VariableFixture, self).__init__(name=name) 4212882Sspwilson2@wisc.edu self.value = value 4312882Sspwilson2@wisc.edu 4412882Sspwilson2@wisc.edu 4512882Sspwilson2@wisc.educlass TempdirFixture(Fixture): 4612882Sspwilson2@wisc.edu def __init__(self): 4712882Sspwilson2@wisc.edu self.path = None 4812882Sspwilson2@wisc.edu super(TempdirFixture, self).__init__( 4912882Sspwilson2@wisc.edu name=constants.tempdir_fixture_name) 5012882Sspwilson2@wisc.edu 5112882Sspwilson2@wisc.edu def setup(self, testitem): 5212882Sspwilson2@wisc.edu self.path = tempfile.mkdtemp(prefix='gem5out') 5312882Sspwilson2@wisc.edu 5412882Sspwilson2@wisc.edu def teardown(self, testitem): 5512882Sspwilson2@wisc.edu if self.path is not None: 5612882Sspwilson2@wisc.edu shutil.rmtree(self.path) 5712882Sspwilson2@wisc.edu 5813790Sjason@lowepower.com def skip_cleanup(self): 5913790Sjason@lowepower.com # Set path to none so it's not deleted 6013790Sjason@lowepower.com self.path = None 6113790Sjason@lowepower.com 6212882Sspwilson2@wisc.edu 6312882Sspwilson2@wisc.educlass SConsFixture(Fixture): 6412882Sspwilson2@wisc.edu ''' 6512882Sspwilson2@wisc.edu Fixture will wait until all SCons targets are collected and tests are 6612882Sspwilson2@wisc.edu about to be ran, then will invocate a single instance of SCons for all 6712882Sspwilson2@wisc.edu targets. 6812882Sspwilson2@wisc.edu 6912882Sspwilson2@wisc.edu :param directory: The directory which scons will -C (cd) into before 7012882Sspwilson2@wisc.edu executing. If None is provided, will choose the config base_dir. 7112882Sspwilson2@wisc.edu ''' 7213851Sjason@lowepower.com def __init__(self, directory=None, target_class=None, options=[]): 7312882Sspwilson2@wisc.edu self.directory = directory if directory else config.base_dir 7412882Sspwilson2@wisc.edu self.target_class = target_class if target_class else SConsTarget 7512882Sspwilson2@wisc.edu self.threads = config.threads 7612882Sspwilson2@wisc.edu self.targets = set() 7713851Sjason@lowepower.com self.options = options 7812882Sspwilson2@wisc.edu super(SConsFixture, self).__init__() 7912882Sspwilson2@wisc.edu 8012882Sspwilson2@wisc.edu def setup(self, testitem): 8112882Sspwilson2@wisc.edu if config.skip_build: 8212882Sspwilson2@wisc.edu return 8312882Sspwilson2@wisc.edu 8412882Sspwilson2@wisc.edu command = [ 8512882Sspwilson2@wisc.edu 'scons', '-C', self.directory, 8612882Sspwilson2@wisc.edu '-j', str(self.threads), 8712882Sspwilson2@wisc.edu '--ignore-style' 8812882Sspwilson2@wisc.edu ] 8912882Sspwilson2@wisc.edu 9012882Sspwilson2@wisc.edu if not self.targets: 9112882Sspwilson2@wisc.edu log.test_log.warn( 9212882Sspwilson2@wisc.edu 'No SCons targets specified, this will' 9312882Sspwilson2@wisc.edu ' build the default all target.\n' 9412882Sspwilson2@wisc.edu 'This is likely unintended, and you' 9512882Sspwilson2@wisc.edu ' may wish to kill testlib and reconfigure.') 9612882Sspwilson2@wisc.edu else: 9712882Sspwilson2@wisc.edu log.test_log.message( 9812882Sspwilson2@wisc.edu 'Building the following targets.' 9912882Sspwilson2@wisc.edu ' This may take a while.') 10012882Sspwilson2@wisc.edu log.test_log.message('%s' % (', '.join(self.targets))) 10112882Sspwilson2@wisc.edu log.test_log.message( 10212882Sspwilson2@wisc.edu "You may want to run with only a single ISA" 10312882Sspwilson2@wisc.edu "(--isa=), use --skip-build, or use 'rerun'.") 10412882Sspwilson2@wisc.edu 10512882Sspwilson2@wisc.edu command.extend(self.targets) 10613851Sjason@lowepower.com if self.options: 10713851Sjason@lowepower.com command.extend(self.options) 10812882Sspwilson2@wisc.edu log_call(log.test_log, command) 10912882Sspwilson2@wisc.edu 11012882Sspwilson2@wisc.edu 11112882Sspwilson2@wisc.educlass SConsTarget(Fixture): 11212882Sspwilson2@wisc.edu # The singleton scons fixture we'll use for all targets. 11312882Sspwilson2@wisc.edu default_scons_invocation = None 11412882Sspwilson2@wisc.edu 11512882Sspwilson2@wisc.edu def __init__(self, target, build_dir=None, invocation=None): 11612882Sspwilson2@wisc.edu ''' 11712882Sspwilson2@wisc.edu Represents a target to be built by an 'invocation' of scons. 11812882Sspwilson2@wisc.edu 11912882Sspwilson2@wisc.edu :param target: The target known to scons. 12012882Sspwilson2@wisc.edu 12112882Sspwilson2@wisc.edu :param build_dir: The 'build' directory path which will be prepended 12212882Sspwilson2@wisc.edu to the target name. 12312882Sspwilson2@wisc.edu 12412882Sspwilson2@wisc.edu :param invocation: Represents an invocation of scons which we will 12512882Sspwilson2@wisc.edu automatically attach this target to. If None provided, uses the 12612882Sspwilson2@wisc.edu main 'scons' invocation. 12712882Sspwilson2@wisc.edu ''' 12812882Sspwilson2@wisc.edu 12912882Sspwilson2@wisc.edu if build_dir is None: 13012882Sspwilson2@wisc.edu build_dir = config.build_dir 13112882Sspwilson2@wisc.edu self.target = os.path.join(build_dir, target) 13212882Sspwilson2@wisc.edu super(SConsTarget, self).__init__(name=target) 13312882Sspwilson2@wisc.edu 13412882Sspwilson2@wisc.edu if invocation is None: 13512882Sspwilson2@wisc.edu if self.default_scons_invocation is None: 13612882Sspwilson2@wisc.edu SConsTarget.default_scons_invocation = SConsFixture() 13712882Sspwilson2@wisc.edu globalfixture(SConsTarget.default_scons_invocation) 13812882Sspwilson2@wisc.edu 13912882Sspwilson2@wisc.edu invocation = self.default_scons_invocation 14012882Sspwilson2@wisc.edu self.invocation = invocation 14112882Sspwilson2@wisc.edu 14212882Sspwilson2@wisc.edu def schedule_finalized(self, schedule): 14312882Sspwilson2@wisc.edu self.invocation.targets.add(self.target) 14412882Sspwilson2@wisc.edu return Fixture.schedule_finalized(self, schedule) 14512882Sspwilson2@wisc.edu 14612882Sspwilson2@wisc.educlass Gem5Fixture(SConsTarget): 14713851Sjason@lowepower.com other_invocations = {} # stores scons invocations other than the default 14813851Sjason@lowepower.com 14913851Sjason@lowepower.com def __init__(self, isa, variant, protocol=None): 15013851Sjason@lowepower.com if protocol: 15113851Sjason@lowepower.com # When specifying an non-default protocol, we have to make a 15213851Sjason@lowepower.com # separate scons invocation with specific parameters. However, if 15313851Sjason@lowepower.com # more than one tests needs the same target, we need to make sure 15413851Sjason@lowepower.com # that we don't call scons too many times. 15513851Sjason@lowepower.com target_dir = isa.upper()+'-'+protocol 15613851Sjason@lowepower.com target = joinpath(target_dir, 'gem5.%s' % variant) 15713851Sjason@lowepower.com if target_dir in self.other_invocations.keys(): 15813851Sjason@lowepower.com invocation = self.other_invocations[target_dir] 15913851Sjason@lowepower.com else: 16013851Sjason@lowepower.com options = ['PROTOCOL='+protocol, '--default='+isa.upper()] 16113851Sjason@lowepower.com invocation = SConsFixture(options=options) 16213851Sjason@lowepower.com globalfixture(invocation) 16313851Sjason@lowepower.com Gem5Fixture.other_invocations[target_dir] = invocation 16413851Sjason@lowepower.com else: 16513851Sjason@lowepower.com target = joinpath(isa.upper(), 'gem5.%s' % variant) 16613851Sjason@lowepower.com invocation = None # use default 16713851Sjason@lowepower.com super(Gem5Fixture, self).__init__(target, invocation=invocation) 16812882Sspwilson2@wisc.edu 16912882Sspwilson2@wisc.edu self.name = constants.gem5_binary_fixture_name 17012882Sspwilson2@wisc.edu self.path = self.target 17112882Sspwilson2@wisc.edu self.isa = isa 17212882Sspwilson2@wisc.edu self.variant = variant 17312882Sspwilson2@wisc.edu 17412882Sspwilson2@wisc.edu 17512882Sspwilson2@wisc.educlass MakeFixture(Fixture): 17612882Sspwilson2@wisc.edu def __init__(self, directory, *args, **kwargs): 17712882Sspwilson2@wisc.edu name = 'make -C %s' % directory 17812882Sspwilson2@wisc.edu super(MakeFixture, self).__init__(build_once=True, lazy_init=False, 17912882Sspwilson2@wisc.edu name=name, 18012882Sspwilson2@wisc.edu *args, **kwargs) 18112882Sspwilson2@wisc.edu self.targets = [] 18212882Sspwilson2@wisc.edu self.directory = directory 18312882Sspwilson2@wisc.edu 18412882Sspwilson2@wisc.edu def setup(self): 18512882Sspwilson2@wisc.edu super(MakeFixture, self).setup() 18612882Sspwilson2@wisc.edu targets = set(self.required_by) 18712882Sspwilson2@wisc.edu command = ['make', '-C', self.directory] 18812882Sspwilson2@wisc.edu command.extend([target.target for target in targets]) 18912882Sspwilson2@wisc.edu log_call(command) 19012882Sspwilson2@wisc.edu 19112882Sspwilson2@wisc.edu 19212882Sspwilson2@wisc.educlass MakeTarget(Fixture): 19312882Sspwilson2@wisc.edu def __init__(self, target, make_fixture=None, *args, **kwargs): 19412882Sspwilson2@wisc.edu ''' 19512882Sspwilson2@wisc.edu :param make_fixture: The make invocation we will be attached to. 19612882Sspwilson2@wisc.edu Since we don't have a single global instance of make in gem5 like we do 19712882Sspwilson2@wisc.edu scons we need to know what invocation to attach to. If none given, 19812882Sspwilson2@wisc.edu creates its own. 19912882Sspwilson2@wisc.edu ''' 20012882Sspwilson2@wisc.edu super(MakeTarget, self).__init__(name=target, *args, **kwargs) 20112882Sspwilson2@wisc.edu self.target = self.name 20212882Sspwilson2@wisc.edu 20312882Sspwilson2@wisc.edu if make_fixture is None: 20412882Sspwilson2@wisc.edu make_fixture = MakeFixture( 20512882Sspwilson2@wisc.edu absdirpath(target), 20612882Sspwilson2@wisc.edu lazy_init=True, 20712882Sspwilson2@wisc.edu build_once=False) 20812882Sspwilson2@wisc.edu 20912882Sspwilson2@wisc.edu self.make_fixture = make_fixture 21012882Sspwilson2@wisc.edu 21112882Sspwilson2@wisc.edu # Add our self to the required targets of the main MakeFixture 21212882Sspwilson2@wisc.edu self.require(self.make_fixture) 21312882Sspwilson2@wisc.edu 21412882Sspwilson2@wisc.edu def setup(self, testitem): 21512882Sspwilson2@wisc.edu super(MakeTarget, self).setup() 21612882Sspwilson2@wisc.edu self.make_fixture.setup() 21712882Sspwilson2@wisc.edu return self 21812882Sspwilson2@wisc.edu 21912882Sspwilson2@wisc.educlass TestProgram(MakeTarget): 22012882Sspwilson2@wisc.edu def __init__(self, program, isa, os, recompile=False): 22112882Sspwilson2@wisc.edu make_dir = joinpath('test-progs', program) 22212882Sspwilson2@wisc.edu make_fixture = MakeFixture(make_dir) 22312882Sspwilson2@wisc.edu target = joinpath('bin', isa, os, program) 22412882Sspwilson2@wisc.edu super(TestProgram, self).__init__(target, make_fixture) 22512882Sspwilson2@wisc.edu self.path = joinpath(make_dir, target) 22612882Sspwilson2@wisc.edu self.recompile = recompile 22712882Sspwilson2@wisc.edu 22812882Sspwilson2@wisc.edu def setup(self, testitem): 22912882Sspwilson2@wisc.edu # Check if the program exists if it does then only compile if 23012882Sspwilson2@wisc.edu # recompile was given. 23112882Sspwilson2@wisc.edu if self.recompile: 23212882Sspwilson2@wisc.edu super(MakeTarget, self).setup() 23312882Sspwilson2@wisc.edu elif not os.path.exists(self.path): 23412882Sspwilson2@wisc.edu super(MakeTarget, self).setup() 23512882Sspwilson2@wisc.edu 23612882Sspwilson2@wisc.educlass DownloadedProgram(Fixture): 23712882Sspwilson2@wisc.edu """ Like TestProgram, but checks the version in the gem5 binary repository 23812882Sspwilson2@wisc.edu and downloads an updated version if it is needed. 23912882Sspwilson2@wisc.edu """ 24012882Sspwilson2@wisc.edu urlbase = "http://gem5.org/dist/current/" 24112882Sspwilson2@wisc.edu 24212882Sspwilson2@wisc.edu def __init__(self, path, program, **kwargs): 24313793Sjason@lowepower.com """ 24413793Sjason@lowepower.com path: string 24513793Sjason@lowepower.com The path to the directory containing the binary relative to 24613793Sjason@lowepower.com $GEM5_BASE/tests 24713793Sjason@lowepower.com program: string 24813793Sjason@lowepower.com The name of the binary file 24913793Sjason@lowepower.com """ 25012882Sspwilson2@wisc.edu super(DownloadedProgram, self).__init__("download-" + program, 25112882Sspwilson2@wisc.edu build_once=True, **kwargs) 25212882Sspwilson2@wisc.edu 25313679Syazakram@ucdavis.edu self.program_dir = path 25413793Sjason@lowepower.com relative_path = joinpath(self.program_dir, program) 25513793Sjason@lowepower.com self.url = self.urlbase + relative_path 25613793Sjason@lowepower.com self.path = os.path.realpath( 25713793Sjason@lowepower.com joinpath(absdirpath(__file__), '../', relative_path) 25813793Sjason@lowepower.com ) 25913793Sjason@lowepower.com 26012882Sspwilson2@wisc.edu def _download(self): 26112882Sspwilson2@wisc.edu import urllib 26213792Sjason@lowepower.com import errno 26312882Sspwilson2@wisc.edu log.test_log.debug("Downloading " + self.url + " to " + self.path) 26412882Sspwilson2@wisc.edu if not os.path.exists(self.program_dir): 26513792Sjason@lowepower.com try: 26613792Sjason@lowepower.com os.makedirs(self.program_dir) 26713792Sjason@lowepower.com except OSError as e: 26813792Sjason@lowepower.com if e.errno != errno.EEXIST: 26913792Sjason@lowepower.com raise 27012882Sspwilson2@wisc.edu urllib.urlretrieve(self.url, self.path) 27112882Sspwilson2@wisc.edu 27212882Sspwilson2@wisc.edu def _getremotetime(self): 27312882Sspwilson2@wisc.edu import urllib2, datetime, time 27412882Sspwilson2@wisc.edu import _strptime # Needed for python threading bug 27512882Sspwilson2@wisc.edu 27612882Sspwilson2@wisc.edu u = urllib2.urlopen(self.url) 27712882Sspwilson2@wisc.edu return time.mktime(datetime.datetime.strptime( \ 27812882Sspwilson2@wisc.edu u.info().getheaders("Last-Modified")[0], 27912882Sspwilson2@wisc.edu "%a, %d %b %Y %X GMT").timetuple()) 28012882Sspwilson2@wisc.edu 28112882Sspwilson2@wisc.edu def setup(self, testitem): 28212882Sspwilson2@wisc.edu import urllib2 28312882Sspwilson2@wisc.edu # Check to see if there is a file downloaded 28412882Sspwilson2@wisc.edu if not os.path.exists(self.path): 28512882Sspwilson2@wisc.edu self._download() 28612882Sspwilson2@wisc.edu else: 28712882Sspwilson2@wisc.edu try: 28812882Sspwilson2@wisc.edu t = self._getremotetime() 28912882Sspwilson2@wisc.edu except urllib2.URLError: 29012882Sspwilson2@wisc.edu # Problem checking the server, use the old files. 29112882Sspwilson2@wisc.edu log.debug("Could not contact server. Binaries may be old.") 29212882Sspwilson2@wisc.edu return 29312882Sspwilson2@wisc.edu # If the server version is more recent, download it 29412882Sspwilson2@wisc.edu if t > os.path.getmtime(self.path): 29512882Sspwilson2@wisc.edu self._download() 296