tests.py revision 13012 
1#!/usr/bin/env python2
2#
3# Copyright (c) 2016-2017 ARM Limited
4# All rights reserved
5#
6# The license below extends only to copyright in the software and shall
7# not be construed as granting a license to any other intellectual
8# property including but not limited to intellectual property relating
9# to a hardware implementation of the functionality of the software
10# licensed hereunder.  You may use the software subject to the license
11# terms below provided that you ensure that this notice is replicated
12# unmodified and in its entirety in all distributions of the software,
13# modified or unmodified, in source code or in binary form.
14#
15# Redistribution and use in source and binary forms, with or without
16# modification, are permitted provided that the following conditions are
17# met: redistributions of source code must retain the above copyright
18# notice, this list of conditions and the following disclaimer;
19# redistributions in binary form must reproduce the above copyright
20# notice, this list of conditions and the following disclaimer in the
21# documentation and/or other materials provided with the distribution;
22# neither the name of the copyright holders nor the names of its
23# contributors may be used to endorse or promote products derived from
24# this software without specific prior written permission.
25#
26# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
27# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
28# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
29# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
30# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
31# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
32# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
33# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
34# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
35# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
36# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
37#
38# Authors: Andreas Sandberg
39
40from abc import ABCMeta, abstractmethod
41import os
42from collections import namedtuple
43from units import *
44from results import TestResult
45import shutil
46
47_test_base = os.path.join(os.path.dirname(__file__), "..")
48
49ClassicConfig = namedtuple("ClassicConfig", (
50    "category",
51    "mode",
52    "workload",
53    "isa",
54    "os",
55    "config",
56))
57
58# There are currently two "classes" of test
59# configurations. Architecture-specific ones and generic ones
60# (typically SE mode tests). In both cases, the configuration name
61# matches a file in tests/configs/ that will be picked up by the test
62# runner (run.py).
63#
64# Architecture specific configurations are listed in the arch_configs
65# dictionary. This is indexed by a (cpu architecture, gpu
66# architecture) tuple. GPU architecture is optional and may be None.
67#
68# Generic configurations are listed in the generic_configs tuple.
69#
70# When discovering available test cases, this script look uses the
71# test list as a list of /candidate/ configurations. A configuration
72# is only used if a test has a reference output for that
73# configuration. In addition to the base configurations from
74# arch_configs and generic_configs, a Ruby configuration may be
75# appended to the base name (this is probed /in addition/ to the
76# original name. See get_tests() for details.
77#
78arch_configs = {
79    ("alpha", None) : (
80        'tsunami-simple-atomic',
81        'tsunami-simple-timing',
82        'tsunami-simple-atomic-dual',
83        'tsunami-simple-timing-dual',
84        'twosys-tsunami-simple-atomic',
85        'tsunami-o3', 'tsunami-o3-dual',
86        'tsunami-minor', 'tsunami-minor-dual',
87        'tsunami-switcheroo-full',
88    ),
89
90    ("arm", None) : (
91        'simple-atomic-dummychecker',
92        'o3-timing-checker',
93        'realview-simple-atomic',
94        'realview-simple-atomic-dual',
95        'realview-simple-atomic-checkpoint',
96        'realview-simple-timing',
97        'realview-simple-timing-dual',
98        'realview-o3',
99        'realview-o3-checker',
100        'realview-o3-dual',
101        'realview-minor',
102        'realview-minor-dual',
103        'realview-switcheroo-atomic',
104        'realview-switcheroo-timing',
105        'realview-switcheroo-noncaching-timing',
106        'realview-switcheroo-o3',
107        'realview-switcheroo-full',
108        'realview64-simple-atomic',
109        'realview64-simple-atomic-checkpoint',
110        'realview64-simple-atomic-dual',
111        'realview64-simple-timing',
112        'realview64-simple-timing-dual',
113        'realview64-o3',
114        'realview64-o3-checker',
115        'realview64-o3-dual',
116        'realview64-minor',
117        'realview64-minor-dual',
118        'realview64-switcheroo-atomic',
119        'realview64-switcheroo-timing',
120        'realview64-switcheroo-o3',
121        'realview64-switcheroo-full',
122    ),
123
124    ("sparc", None) : (
125        't1000-simple-atomic',
126        't1000-simple-x86',
127    ),
128
129    ("timing", None) : (
130        'pc-simple-atomic',
131        'pc-simple-timing',
132        'pc-o3-timing',
133        'pc-switcheroo-full',
134    ),
135
136    ("x86", "hsail") : (
137        'gpu',
138    ),
139}
140
141generic_configs = (
142    'simple-atomic',
143    'simple-atomic-mp',
144    'simple-timing',
145    'simple-timing-mp',
146
147    'minor-timing',
148    'minor-timing-mp',
149
150    'o3-timing',
151    'o3-timing-mt',
152    'o3-timing-mp',
153
154    'rubytest',
155    'memcheck',
156    'memtest',
157    'memtest-filter',
158    'tgen-simple-mem',
159    'tgen-dram-ctrl',
160    'dram-lowp',
161
162    'learning-gem5-p1-simple',
163    'learning-gem5-p1-two-level',
164)
165
166default_ruby_protocol = {
167    "arm" : "MOESI_CMP_directory",
168}
169
170def get_default_protocol(arch):
171    return default_ruby_protocol.get(arch, 'MI_example')
172
173all_categories = ("quick", "long")
174all_modes = ("fs", "se")
175
176class Test(object):
177    """Test case base class.
178
179    Test cases consists of one or more test units that are run in two
180    phases. A run phase (units produced by run_units() and a verify
181    phase (units from verify_units()). The verify phase is skipped if
182    the run phase fails.
183
184    """
185
186    __metaclass__ = ABCMeta
187
188    def __init__(self, name):
189        self.test_name = name
190
191    @abstractmethod
192    def ref_files(self):
193        """Get a list of reference files used by this test case"""
194        pass
195
196    @abstractmethod
197    def run_units(self):
198        """Units (typically RunGem5 instances) that describe the run phase of
199        this test.
200
201        """
202        pass
203
204    @abstractmethod
205    def verify_units(self):
206        """Verify the output from the run phase (see run_units())."""
207        pass
208
209    @abstractmethod
210    def update_ref(self):
211        """Update reference files with files from a test run"""
212        pass
213
214    def run(self):
215        """Run this test case and return a list of results"""
216
217        run_results = [ u.run() for u in self.run_units() ]
218        run_ok = all([not r.skipped() and r for r in run_results ])
219
220        verify_results = [
221            u.run() if run_ok else u.skip()
222            for u in self.verify_units()
223        ]
224
225        return TestResult(self.test_name,
226                          run_results=run_results,
227                          verify_results=verify_results)
228
229    def __str__(self):
230        return self.test_name
231
232class ClassicTest(Test):
233    # The diff ignore list contains all files that shouldn't be diffed
234    # using DiffOutFile. These files typically use special-purpose
235    # diff tools (e.g., DiffStatFile).
236    diff_ignore_files = FileIgnoreList(
237        names=(
238            # Stat files use a special stat differ
239            "stats.txt",
240        ), rex=(
241        ))
242
243    # These files should never be included in the list of
244    # reference files. This list should include temporary files
245    # and other files that we don't care about.
246    ref_ignore_files = FileIgnoreList(
247        names=(
248            "EMPTY",
249        ), rex=(
250            # Mercurial sometimes leaves backups when applying MQ patches
251            r"\.orig$",
252            r"\.rej$",
253        ))
254
255    def __init__(self, gem5, output_dir, config_tuple,
256                 timeout=None,
257                 skip=False, skip_diff_out=False, skip_diff_stat=False):
258
259        super(ClassicTest, self).__init__("/".join(config_tuple))
260
261        ct = config_tuple
262
263        self.gem5 = os.path.abspath(gem5)
264        self.script = os.path.join(_test_base, "run.py")
265        self.config_tuple = ct
266        self.timeout = timeout
267
268        self.output_dir = output_dir
269        self.ref_dir = os.path.join(_test_base,
270                                    ct.category, ct.mode, ct.workload,
271                                    "ref", ct.isa, ct.os, ct.config)
272        self.skip_run = skip
273        self.skip_diff_out = skip or skip_diff_out
274        self.skip_diff_stat = skip or skip_diff_stat
275
276    def ref_files(self):
277        ref_dir = os.path.abspath(self.ref_dir)
278        for root, dirs, files in os.walk(ref_dir, topdown=False):
279            for f in files:
280                fpath = os.path.join(root[len(ref_dir) + 1:], f)
281                if fpath not in ClassicTest.ref_ignore_files:
282                    yield fpath
283
284    def run_units(self):
285        args = [
286            self.script,
287            "/".join(self.config_tuple),
288        ]
289
290        return [
291            RunGem5(self.gem5, args,
292                    ref_dir=self.ref_dir, test_dir=self.output_dir,
293                    skip=self.skip_run),
294        ]
295
296    def verify_units(self):
297        ref_files = set(self.ref_files())
298        units = []
299        if "stats.txt" in ref_files:
300            units.append(
301                DiffStatFile(ref_dir=self.ref_dir, test_dir=self.output_dir,
302                             skip=self.skip_diff_stat))
303        units += [
304            DiffOutFile(f,
305                        ref_dir=self.ref_dir, test_dir=self.output_dir,
306                        skip=self.skip_diff_out)
307            for f in ref_files if f not in ClassicTest.diff_ignore_files
308        ]
309
310        return units
311
312    def update_ref(self):
313        for fname in self.ref_files():
314            shutil.copy(
315                os.path.join(self.output_dir, fname),
316                os.path.join(self.ref_dir, fname))
317
318def parse_test_filter(test_filter):
319    wildcards = ("", "*")
320
321    _filter = list(test_filter.split("/"))
322    if len(_filter) > 3:
323        raise RuntimeError("Illegal test filter string")
324    _filter += [ "", ] * (3 - len(_filter))
325
326    isa, cat, mode = _filter
327
328    if isa in wildcards:
329        raise RuntimeError("No ISA specified")
330
331    cat = all_categories if cat in wildcards else (cat, )
332    mode = all_modes if mode in wildcards else (mode, )
333
334    return isa, cat, mode
335
336def get_tests(isa,
337              categories=all_categories, modes=all_modes,
338              ruby_protocol=None, gpu_isa=None):
339
340    # Generate a list of candidate configs
341    configs = list(arch_configs.get((isa, gpu_isa), []))
342
343    if (isa, gpu_isa) == ("x86", "hsail"):
344        if ruby_protocol == "GPU_RfO":
345            configs += ['gpu-randomtest']
346    else:
347        configs += generic_configs
348
349    if ruby_protocol == get_default_protocol(isa):
350        if ruby_protocol == 'MI_example':
351            configs += [ "%s-ruby" % (c, ) for c in configs ]
352        else:
353            configs += [ "%s-ruby-%s" % (c, ruby_protocol) for c in configs ]
354    elif ruby_protocol is not None:
355        # Override generic ISA configs when using Ruby (excluding
356        # MI_example which is included in all ISAs by default). This
357        # reduces the number of generic tests we re-run for when
358        # compiling Ruby targets.
359        configs = [ "%s-ruby-%s" % (c, ruby_protocol) for c in configs ]
360
361    # /(quick|long)/(fs|se)/workload/ref/arch/guest/config/
362    for conf_script in configs:
363        for cat in categories:
364            for mode in modes:
365                mode_dir = os.path.join(_test_base, cat, mode)
366                if not os.path.exists(mode_dir):
367                    continue
368
369                for workload in os.listdir(mode_dir):
370                    isa_dir = os.path.join(mode_dir, workload, "ref", isa)
371                    if not os.path.isdir(isa_dir):
372                        continue
373
374                    for _os in os.listdir(isa_dir):
375                        test_dir = os.path.join(isa_dir, _os, conf_script)
376                        if not os.path.exists(test_dir) or \
377                           os.path.exists(os.path.join(test_dir, "skip")):
378                            continue
379
380                        yield ClassicConfig(cat, mode, workload, isa, _os,
381                                            conf_script)
382