1# Copyright (c) 2019 ARM Limited
2# All rights reserved
3#
4# The license below extends only to copyright in the software and shall
5# not be construed as granting a license to any other intellectual
6# property including but not limited to intellectual property relating
7# to a hardware implementation of the functionality of the software
8# licensed hereunder.  You may use the software subject to the license
9# terms below provided that you ensure that this notice is replicated
10# unmodified and in its entirety in all distributions of the software,
11# modified or unmodified, in source code or in binary form.
12#
13# Copyright (c) 2017 Mark D. Hill and David A. Wood
14# All rights reserved.
15#
16# Redistribution and use in source and binary forms, with or without
17# modification, are permitted provided that the following conditions are
18# met: redistributions of source code must retain the above copyright
19# notice, this list of conditions and the following disclaimer;
20# redistributions in binary form must reproduce the above copyright
21# notice, this list of conditions and the following disclaimer in the
22# documentation and/or other materials provided with the distribution;
23# neither the name of the copyright holders nor the names of its
24# contributors may be used to endorse or promote products derived from
25# this software without specific prior written permission.
26#
27# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
28# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
29# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
30# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
31# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
32# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
33# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
34# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
35# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
36# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
37# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
38#
39# Authors: Sean Wilson
40#          Nikos Nikoleris
41
42import os
43import tempfile
44import shutil
45import threading
46import urllib
47import urllib2
48
49from testlib.fixture import Fixture
50from testlib.config import config, constants
51from testlib.helper import log_call, cacheresult, joinpath, absdirpath
52import testlib.log as log
53
54
55class VariableFixture(Fixture):
56    def __init__(self, value=None, name=None):
57        super(VariableFixture, self).__init__(name=name)
58        self.value = value
59
60
61class TempdirFixture(Fixture):
62    def __init__(self):
63        self.path = None
64        super(TempdirFixture, self).__init__(
65                name=constants.tempdir_fixture_name)
66
67    def setup(self, testitem):
68        self.path = tempfile.mkdtemp(prefix='gem5out')
69
70    def teardown(self, testitem):
71        if self.path is not None:
72            shutil.rmtree(self.path)
73
74    def skip_cleanup(self):
75        # Set path to none so it's not deleted
76        self.path = None
77
78class UniqueFixture(Fixture):
79    '''
80    Base class for fixtures that generate a target in the
81    filesystem. If the same fixture is used by more than one
82    test/suite, rather than creating a copy of the fixture, it returns
83    the same object and makes sure that setup is only executed
84    once. Devired classses should override the _init and _setup
85    functions.
86
87    :param target: The absolute path of the target in the filesystem.
88
89    '''
90    fixtures = {}
91
92    def __new__(cls, target):
93        if target in cls.fixtures:
94            obj = cls.fixtures[target]
95        else:
96            obj = super(UniqueFixture, cls).__new__(cls)
97            obj.lock = threading.Lock()
98            obj.target = target
99            cls.fixtures[target] = obj
100        return obj
101
102    def __init__(self, *args, **kwargs):
103        with self.lock:
104            if hasattr(self, '_init_done'):
105                return
106            super(UniqueFixture, self).__init__(self, **kwargs)
107            self._init(*args, **kwargs)
108            self._init_done = True
109
110    def setup(self, testitem):
111        with self.lock:
112            if hasattr(self, '_setup_done'):
113                return
114            self._setup_done = True
115            self._setup(testitem)
116
117
118class SConsFixture(UniqueFixture):
119    '''
120    Fixture will wait until all SCons targets are collected and tests are
121    about to be ran, then will invocate a single instance of SCons for all
122    targets.
123
124    :param directory: The directory which scons will -C (cd) into before
125        executing. If None is provided, will choose the config base_dir.
126    '''
127
128    def __new__(cls, target):
129        obj = super(SConsFixture, cls).__new__(cls, target)
130        return obj
131
132    def _setup(self, testitem):
133        if config.skip_build:
134            return
135
136        command = [
137            'scons', '-C', self.directory,
138            '-j', str(config.threads),
139            '--ignore-style'
140        ]
141
142        if not self.targets:
143            log.test_log.warn(
144                'No SCons targets specified, this will'
145                ' build the default all target.\n'
146                'This is likely unintended, and you'
147                ' may wish to kill testlib and reconfigure.')
148        else:
149            log.test_log.message(
150                    'Building the following targets.'
151                    ' This may take a while.')
152            log.test_log.message('%s' % (', '.join(self.targets)))
153            log.test_log.message(
154                    "You may want to run with only a single ISA"
155                    "(--isa=), use --skip-build, or use 'rerun'.")
156
157        command.extend(self.targets)
158        if self.options:
159            command.extend(self.options)
160        log_call(log.test_log, command)
161
162class Gem5Fixture(SConsFixture):
163    def __new__(cls, isa, variant, protocol=None):
164        target_dir = joinpath(config.build_dir, isa.upper())
165        if protocol:
166            target_dir += '_' + protocol
167        target = joinpath(target_dir, 'gem5.%s' % variant)
168        obj = super(Gem5Fixture, cls).__new__(cls, target)
169        return obj
170
171    def _init(self, isa, variant, protocol=None):
172        self.name = constants.gem5_binary_fixture_name
173
174        self.targets = [self.target]
175        self.path = self.target
176        self.directory = config.base_dir
177
178        self.options = []
179        if protocol:
180            self.options = [ '--default=' + isa.upper(),
181                             'PROTOCOL=' + protocol ]
182        self.set_global()
183
184class MakeFixture(Fixture):
185    def __init__(self, directory, *args, **kwargs):
186        name = 'make -C %s' % directory
187        super(MakeFixture, self).__init__(build_once=True, lazy_init=False,
188                                          name=name,
189                                          *args, **kwargs)
190        self.targets = []
191        self.directory = directory
192
193    def setup(self):
194        super(MakeFixture, self).setup()
195        targets = set(self.required_by)
196        command = ['make', '-C', self.directory]
197        command.extend([target.target for target in targets])
198        log_call(command)
199
200
201class MakeTarget(Fixture):
202    def __init__(self, target, make_fixture=None, *args, **kwargs):
203        '''
204        :param make_fixture: The make invocation we will be attached to.
205        Since we don't have a single global instance of make in gem5 like we do
206        scons we need to know what invocation to attach to. If none given,
207        creates its own.
208        '''
209        super(MakeTarget, self).__init__(name=target, *args, **kwargs)
210        self.target = self.name
211
212        if make_fixture is None:
213            make_fixture = MakeFixture(
214                    absdirpath(target),
215                    lazy_init=True,
216                    build_once=False)
217
218        self.make_fixture = make_fixture
219
220        # Add our self to the required targets of the main MakeFixture
221        self.require(self.make_fixture)
222
223    def setup(self, testitem):
224        super(MakeTarget, self).setup()
225        self.make_fixture.setup()
226        return self
227
228class TestProgram(MakeTarget):
229    def __init__(self, program, isa, os, recompile=False):
230        make_dir = joinpath('test-progs', program)
231        make_fixture = MakeFixture(make_dir)
232        target = joinpath('bin', isa, os, program)
233        super(TestProgram, self).__init__(target, make_fixture)
234        self.path = joinpath(make_dir, target)
235        self.recompile = recompile
236
237    def setup(self, testitem):
238        # Check if the program exists if it does then only compile if
239        # recompile was given.
240        if self.recompile:
241            super(MakeTarget, self).setup()
242        elif not os.path.exists(self.path):
243            super(MakeTarget, self).setup()
244
245class DownloadedProgram(UniqueFixture):
246    """ Like TestProgram, but checks the version in the gem5 binary repository
247        and downloads an updated version if it is needed.
248    """
249
250    def __new__(cls, url, path, filename):
251        target = joinpath(path, filename)
252        return super(DownloadedProgram, cls).__new__(cls, target)
253
254    def _init(self, url, path, filename, **kwargs):
255        """
256        url: string
257            The url of the archive
258        path: string
259            The absolute path of the directory containing the archive
260        filename: string
261            The name of the archive
262        """
263
264        self.url = url
265        self.path = path
266        self.filename = joinpath(path, filename)
267        self.name = "Downloaded:" + self.filename
268
269    def _download(self):
270        import errno
271        log.test_log.debug("Downloading " + self.url + " to " + self.path)
272        if not os.path.exists(self.path):
273            try:
274                os.makedirs(self.path)
275            except OSError as e:
276                if e.errno != errno.EEXIST:
277                    raise
278        urllib.urlretrieve(self.url, self.filename)
279
280    def _getremotetime(self):
281        import datetime, time
282        import _strptime # Needed for python threading bug
283
284        u = urllib2.urlopen(self.url)
285        return time.mktime(datetime.datetime.strptime( \
286                    u.info().getheaders("Last-Modified")[0],
287                    "%a, %d %b %Y %X GMT").timetuple())
288
289    def _setup(self, testitem):
290        # Check to see if there is a file downloaded
291        if not os.path.exists(self.filename):
292            self._download()
293        else:
294            try:
295                t = self._getremotetime()
296            except urllib2.URLError:
297                # Problem checking the server, use the old files.
298                log.test_log.debug("Could not contact server. Binaries may be old.")
299                return
300            # If the server version is more recent, download it
301            if t > os.path.getmtime(self.filename):
302                self._download()
303
304class DownloadedArchive(DownloadedProgram):
305    """ Like TestProgram, but checks the version in the gem5 binary repository
306        and downloads an updated version if it is needed.
307    """
308
309    def _extract(self):
310        import tarfile
311        with tarfile.open(self.filename) as tf:
312            tf.extractall(self.path)
313
314    def _setup(self, testitem):
315        # Check to see if there is a file downloaded
316        if not os.path.exists(self.filename):
317            self._download()
318            self._extract()
319        else:
320            try:
321                t = self._getremotetime()
322            except urllib2.URLError:
323                # Problem checking the server, use the old files.
324                log.test_log.debug("Could not contact server. Binaries may be old.")
325                return
326            # If the server version is more recent, download it
327            if t > os.path.getmtime(self.filename):
328                self._download()
329                self._extract()
330