tests.py revision 11828
111828Sjason@lowepower.com#!/usr/bin/env python2
211482Sandreas.sandberg@arm.com#
311482Sandreas.sandberg@arm.com# Copyright (c) 2016 ARM Limited
411482Sandreas.sandberg@arm.com# All rights reserved
511482Sandreas.sandberg@arm.com#
611482Sandreas.sandberg@arm.com# The license below extends only to copyright in the software and shall
711482Sandreas.sandberg@arm.com# not be construed as granting a license to any other intellectual
811482Sandreas.sandberg@arm.com# property including but not limited to intellectual property relating
911482Sandreas.sandberg@arm.com# to a hardware implementation of the functionality of the software
1011482Sandreas.sandberg@arm.com# licensed hereunder.  You may use the software subject to the license
1111482Sandreas.sandberg@arm.com# terms below provided that you ensure that this notice is replicated
1211482Sandreas.sandberg@arm.com# unmodified and in its entirety in all distributions of the software,
1311482Sandreas.sandberg@arm.com# modified or unmodified, in source code or in binary form.
1411482Sandreas.sandberg@arm.com#
1511482Sandreas.sandberg@arm.com# Redistribution and use in source and binary forms, with or without
1611482Sandreas.sandberg@arm.com# modification, are permitted provided that the following conditions are
1711482Sandreas.sandberg@arm.com# met: redistributions of source code must retain the above copyright
1811482Sandreas.sandberg@arm.com# notice, this list of conditions and the following disclaimer;
1911482Sandreas.sandberg@arm.com# redistributions in binary form must reproduce the above copyright
2011482Sandreas.sandberg@arm.com# notice, this list of conditions and the following disclaimer in the
2111482Sandreas.sandberg@arm.com# documentation and/or other materials provided with the distribution;
2211482Sandreas.sandberg@arm.com# neither the name of the copyright holders nor the names of its
2311482Sandreas.sandberg@arm.com# contributors may be used to endorse or promote products derived from
2411482Sandreas.sandberg@arm.com# this software without specific prior written permission.
2511482Sandreas.sandberg@arm.com#
2611482Sandreas.sandberg@arm.com# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
2711482Sandreas.sandberg@arm.com# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
2811482Sandreas.sandberg@arm.com# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
2911482Sandreas.sandberg@arm.com# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
3011482Sandreas.sandberg@arm.com# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
3111482Sandreas.sandberg@arm.com# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
3211482Sandreas.sandberg@arm.com# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
3311482Sandreas.sandberg@arm.com# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
3411482Sandreas.sandberg@arm.com# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
3511482Sandreas.sandberg@arm.com# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
3611482Sandreas.sandberg@arm.com# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
3711482Sandreas.sandberg@arm.com#
3811482Sandreas.sandberg@arm.com# Authors: Andreas Sandberg
3911482Sandreas.sandberg@arm.com
4011482Sandreas.sandberg@arm.comfrom abc import ABCMeta, abstractmethod
4111482Sandreas.sandberg@arm.comimport os
4211482Sandreas.sandberg@arm.comfrom collections import namedtuple
4311482Sandreas.sandberg@arm.comfrom units import *
4411482Sandreas.sandberg@arm.comfrom results import TestResult
4511482Sandreas.sandberg@arm.comimport shutil
4611482Sandreas.sandberg@arm.com
4711482Sandreas.sandberg@arm.com_test_base = os.path.join(os.path.dirname(__file__), "..")
4811482Sandreas.sandberg@arm.com
4911482Sandreas.sandberg@arm.comClassicConfig = namedtuple("ClassicConfig", (
5011482Sandreas.sandberg@arm.com    "category",
5111482Sandreas.sandberg@arm.com    "mode",
5211482Sandreas.sandberg@arm.com    "workload",
5311482Sandreas.sandberg@arm.com    "isa",
5411482Sandreas.sandberg@arm.com    "os",
5511482Sandreas.sandberg@arm.com    "config",
5611482Sandreas.sandberg@arm.com))
5711482Sandreas.sandberg@arm.com
5811482Sandreas.sandberg@arm.com# There are currently two "classes" of test
5911482Sandreas.sandberg@arm.com# configurations. Architecture-specific ones and generic ones
6011482Sandreas.sandberg@arm.com# (typically SE mode tests). In both cases, the configuration name
6111482Sandreas.sandberg@arm.com# matches a file in tests/configs/ that will be picked up by the test
6211482Sandreas.sandberg@arm.com# runner (run.py).
6311482Sandreas.sandberg@arm.com#
6411482Sandreas.sandberg@arm.com# Architecture specific configurations are listed in the arch_configs
6511482Sandreas.sandberg@arm.com# dictionary. This is indexed by a (cpu architecture, gpu
6611482Sandreas.sandberg@arm.com# architecture) tuple. GPU architecture is optional and may be None.
6711482Sandreas.sandberg@arm.com#
6811482Sandreas.sandberg@arm.com# Generic configurations are listed in the generic_configs tuple.
6911482Sandreas.sandberg@arm.com#
7011482Sandreas.sandberg@arm.com# When discovering available test cases, this script look uses the
7111482Sandreas.sandberg@arm.com# test list as a list of /candidate/ configurations. A configuration
7211482Sandreas.sandberg@arm.com# is only used if a test has a reference output for that
7311482Sandreas.sandberg@arm.com# configuration. In addition to the base configurations from
7411482Sandreas.sandberg@arm.com# arch_configs and generic_configs, a Ruby configuration may be
7511482Sandreas.sandberg@arm.com# appended to the base name (this is probed /in addition/ to the
7611482Sandreas.sandberg@arm.com# original name. See get_tests() for details.
7711482Sandreas.sandberg@arm.com#
7811482Sandreas.sandberg@arm.comarch_configs = {
7911482Sandreas.sandberg@arm.com    ("alpha", None) : (
8011482Sandreas.sandberg@arm.com        'tsunami-simple-atomic',
8111482Sandreas.sandberg@arm.com        'tsunami-simple-timing',
8211482Sandreas.sandberg@arm.com        'tsunami-simple-atomic-dual',
8311482Sandreas.sandberg@arm.com        'tsunami-simple-timing-dual',
8411482Sandreas.sandberg@arm.com        'twosys-tsunami-simple-atomic',
8511482Sandreas.sandberg@arm.com        'tsunami-o3', 'tsunami-o3-dual',
8611482Sandreas.sandberg@arm.com        'tsunami-minor', 'tsunami-minor-dual',
8711482Sandreas.sandberg@arm.com        'tsunami-switcheroo-full',
8811482Sandreas.sandberg@arm.com    ),
8911482Sandreas.sandberg@arm.com
9011482Sandreas.sandberg@arm.com    ("arm", None) : (
9111482Sandreas.sandberg@arm.com        'simple-atomic-dummychecker',
9211482Sandreas.sandberg@arm.com        'o3-timing-checker',
9311482Sandreas.sandberg@arm.com        'realview-simple-atomic',
9411482Sandreas.sandberg@arm.com        'realview-simple-atomic-dual',
9511482Sandreas.sandberg@arm.com        'realview-simple-atomic-checkpoint',
9611482Sandreas.sandberg@arm.com        'realview-simple-timing',
9711482Sandreas.sandberg@arm.com        'realview-simple-timing-dual',
9811482Sandreas.sandberg@arm.com        'realview-o3',
9911482Sandreas.sandberg@arm.com        'realview-o3-checker',
10011482Sandreas.sandberg@arm.com        'realview-o3-dual',
10111482Sandreas.sandberg@arm.com        'realview-minor',
10211482Sandreas.sandberg@arm.com        'realview-minor-dual',
10311482Sandreas.sandberg@arm.com        'realview-switcheroo-atomic',
10411482Sandreas.sandberg@arm.com        'realview-switcheroo-timing',
10511482Sandreas.sandberg@arm.com        'realview-switcheroo-o3',
10611482Sandreas.sandberg@arm.com        'realview-switcheroo-full',
10711482Sandreas.sandberg@arm.com        'realview64-simple-atomic',
10811482Sandreas.sandberg@arm.com        'realview64-simple-atomic-checkpoint',
10911482Sandreas.sandberg@arm.com        'realview64-simple-atomic-dual',
11011482Sandreas.sandberg@arm.com        'realview64-simple-timing',
11111482Sandreas.sandberg@arm.com        'realview64-simple-timing-dual',
11211482Sandreas.sandberg@arm.com        'realview64-o3',
11311482Sandreas.sandberg@arm.com        'realview64-o3-checker',
11411482Sandreas.sandberg@arm.com        'realview64-o3-dual',
11511482Sandreas.sandberg@arm.com        'realview64-minor',
11611482Sandreas.sandberg@arm.com        'realview64-minor-dual',
11711482Sandreas.sandberg@arm.com        'realview64-switcheroo-atomic',
11811482Sandreas.sandberg@arm.com        'realview64-switcheroo-timing',
11911482Sandreas.sandberg@arm.com        'realview64-switcheroo-o3',
12011482Sandreas.sandberg@arm.com        'realview64-switcheroo-full',
12111482Sandreas.sandberg@arm.com    ),
12211482Sandreas.sandberg@arm.com
12311482Sandreas.sandberg@arm.com    ("sparc", None) : (
12411482Sandreas.sandberg@arm.com        't1000-simple-atomic',
12511482Sandreas.sandberg@arm.com        't1000-simple-x86',
12611482Sandreas.sandberg@arm.com    ),
12711482Sandreas.sandberg@arm.com
12811482Sandreas.sandberg@arm.com    ("timing", None) : (
12911482Sandreas.sandberg@arm.com        'pc-simple-atomic',
13011482Sandreas.sandberg@arm.com        'pc-simple-timing',
13111482Sandreas.sandberg@arm.com        'pc-o3-timing',
13211482Sandreas.sandberg@arm.com        'pc-switcheroo-full',
13311482Sandreas.sandberg@arm.com    ),
13411482Sandreas.sandberg@arm.com
13511482Sandreas.sandberg@arm.com    ("x86", "hsail") : (
13611482Sandreas.sandberg@arm.com        'gpu',
13711482Sandreas.sandberg@arm.com    ),
13811482Sandreas.sandberg@arm.com}
13911482Sandreas.sandberg@arm.com
14011482Sandreas.sandberg@arm.comgeneric_configs = (
14111482Sandreas.sandberg@arm.com    'simple-atomic',
14211482Sandreas.sandberg@arm.com    'simple-atomic-mp',
14311482Sandreas.sandberg@arm.com    'simple-timing',
14411482Sandreas.sandberg@arm.com    'simple-timing-mp',
14511482Sandreas.sandberg@arm.com
14611482Sandreas.sandberg@arm.com    'minor-timing',
14711482Sandreas.sandberg@arm.com    'minor-timing-mp',
14811482Sandreas.sandberg@arm.com
14911482Sandreas.sandberg@arm.com    'o3-timing',
15011482Sandreas.sandberg@arm.com    'o3-timing-mt',
15111482Sandreas.sandberg@arm.com    'o3-timing-mp',
15211482Sandreas.sandberg@arm.com
15311482Sandreas.sandberg@arm.com    'rubytest',
15411482Sandreas.sandberg@arm.com    'memcheck',
15511482Sandreas.sandberg@arm.com    'memtest',
15611482Sandreas.sandberg@arm.com    'memtest-filter',
15711482Sandreas.sandberg@arm.com    'tgen-simple-mem',
15811482Sandreas.sandberg@arm.com    'tgen-dram-ctrl',
15911482Sandreas.sandberg@arm.com
16011482Sandreas.sandberg@arm.com    'learning-gem5-p1-simple',
16111482Sandreas.sandberg@arm.com    'learning-gem5-p1-two-level',
16211482Sandreas.sandberg@arm.com)
16311482Sandreas.sandberg@arm.com
16411482Sandreas.sandberg@arm.comall_categories = ("quick", "long")
16511482Sandreas.sandberg@arm.comall_modes = ("fs", "se")
16611482Sandreas.sandberg@arm.com
16711482Sandreas.sandberg@arm.comclass Test(object):
16811482Sandreas.sandberg@arm.com    """Test case base class.
16911482Sandreas.sandberg@arm.com
17011482Sandreas.sandberg@arm.com    Test cases consists of one or more test units that are run in two
17111482Sandreas.sandberg@arm.com    phases. A run phase (units produced by run_units() and a verify
17211482Sandreas.sandberg@arm.com    phase (units from verify_units()). The verify phase is skipped if
17311482Sandreas.sandberg@arm.com    the run phase fails.
17411482Sandreas.sandberg@arm.com
17511482Sandreas.sandberg@arm.com    """
17611482Sandreas.sandberg@arm.com
17711482Sandreas.sandberg@arm.com    __metaclass__ = ABCMeta
17811482Sandreas.sandberg@arm.com
17911482Sandreas.sandberg@arm.com    def __init__(self, name):
18011482Sandreas.sandberg@arm.com        self.test_name = name
18111482Sandreas.sandberg@arm.com
18211482Sandreas.sandberg@arm.com    @abstractmethod
18311482Sandreas.sandberg@arm.com    def ref_files(self):
18411482Sandreas.sandberg@arm.com        """Get a list of reference files used by this test case"""
18511482Sandreas.sandberg@arm.com        pass
18611482Sandreas.sandberg@arm.com
18711482Sandreas.sandberg@arm.com    @abstractmethod
18811482Sandreas.sandberg@arm.com    def run_units(self):
18911482Sandreas.sandberg@arm.com        """Units (typically RunGem5 instances) that describe the run phase of
19011482Sandreas.sandberg@arm.com        this test.
19111482Sandreas.sandberg@arm.com
19211482Sandreas.sandberg@arm.com        """
19311482Sandreas.sandberg@arm.com        pass
19411482Sandreas.sandberg@arm.com
19511482Sandreas.sandberg@arm.com    @abstractmethod
19611482Sandreas.sandberg@arm.com    def verify_units(self):
19711482Sandreas.sandberg@arm.com        """Verify the output from the run phase (see run_units())."""
19811482Sandreas.sandberg@arm.com        pass
19911482Sandreas.sandberg@arm.com
20011482Sandreas.sandberg@arm.com    @abstractmethod
20111482Sandreas.sandberg@arm.com    def update_ref(self):
20211482Sandreas.sandberg@arm.com        """Update reference files with files from a test run"""
20311482Sandreas.sandberg@arm.com        pass
20411482Sandreas.sandberg@arm.com
20511482Sandreas.sandberg@arm.com    def run(self):
20611482Sandreas.sandberg@arm.com        """Run this test case and return a list of results"""
20711482Sandreas.sandberg@arm.com
20811482Sandreas.sandberg@arm.com        run_results = [ u.run() for u in self.run_units() ]
20911482Sandreas.sandberg@arm.com        run_ok = all([not r.skipped() and r for r in run_results ])
21011482Sandreas.sandberg@arm.com
21111482Sandreas.sandberg@arm.com        verify_results = [
21211482Sandreas.sandberg@arm.com            u.run() if run_ok else u.skip()
21311482Sandreas.sandberg@arm.com            for u in self.verify_units()
21411482Sandreas.sandberg@arm.com        ]
21511482Sandreas.sandberg@arm.com
21611542Sandreas.sandberg@arm.com        return TestResult(self.test_name,
21711542Sandreas.sandberg@arm.com                          run_results=run_results,
21811542Sandreas.sandberg@arm.com                          verify_results=verify_results)
21911482Sandreas.sandberg@arm.com
22011482Sandreas.sandberg@arm.com    def __str__(self):
22111482Sandreas.sandberg@arm.com        return self.test_name
22211482Sandreas.sandberg@arm.com
22311482Sandreas.sandberg@arm.comclass ClassicTest(Test):
22411503Sandreas.sandberg@arm.com    # The diff ignore list contains all files that shouldn't be diffed
22511503Sandreas.sandberg@arm.com    # using DiffOutFile. These files typically use special-purpose
22611503Sandreas.sandberg@arm.com    # diff tools (e.g., DiffStatFile).
22711571Sandreas.sandberg@arm.com    diff_ignore_files = FileIgnoreList(
22811571Sandreas.sandberg@arm.com        names=(
22911571Sandreas.sandberg@arm.com            # Stat files use a special stat differ
23011571Sandreas.sandberg@arm.com            "stats.txt",
23111571Sandreas.sandberg@arm.com        ), rex=(
23211571Sandreas.sandberg@arm.com        ))
23311503Sandreas.sandberg@arm.com
23411503Sandreas.sandberg@arm.com    # These files should never be included in the list of
23511503Sandreas.sandberg@arm.com    # reference files. This list should include temporary files
23611503Sandreas.sandberg@arm.com    # and other files that we don't care about.
23711571Sandreas.sandberg@arm.com    ref_ignore_files = FileIgnoreList(
23811571Sandreas.sandberg@arm.com        names=(
23911636Sandreas.sandberg@arm.com            "EMPTY",
24011571Sandreas.sandberg@arm.com        ), rex=(
24111571Sandreas.sandberg@arm.com            # Mercurial sometimes leaves backups when applying MQ patches
24211571Sandreas.sandberg@arm.com            r"\.orig$",
24311571Sandreas.sandberg@arm.com            r"\.rej$",
24411571Sandreas.sandberg@arm.com        ))
24511482Sandreas.sandberg@arm.com
24611482Sandreas.sandberg@arm.com    def __init__(self, gem5, output_dir, config_tuple,
24711482Sandreas.sandberg@arm.com                 timeout=None,
24811482Sandreas.sandberg@arm.com                 skip=False, skip_diff_out=False, skip_diff_stat=False):
24911482Sandreas.sandberg@arm.com
25011482Sandreas.sandberg@arm.com        super(ClassicTest, self).__init__("/".join(config_tuple))
25111482Sandreas.sandberg@arm.com
25211482Sandreas.sandberg@arm.com        ct = config_tuple
25311482Sandreas.sandberg@arm.com
25411482Sandreas.sandberg@arm.com        self.gem5 = os.path.abspath(gem5)
25511482Sandreas.sandberg@arm.com        self.script = os.path.join(_test_base, "run.py")
25611482Sandreas.sandberg@arm.com        self.config_tuple = ct
25711482Sandreas.sandberg@arm.com        self.timeout = timeout
25811482Sandreas.sandberg@arm.com
25911482Sandreas.sandberg@arm.com        self.output_dir = output_dir
26011482Sandreas.sandberg@arm.com        self.ref_dir = os.path.join(_test_base,
26111482Sandreas.sandberg@arm.com                                    ct.category, ct.mode, ct.workload,
26211482Sandreas.sandberg@arm.com                                    "ref", ct.isa, ct.os, ct.config)
26311482Sandreas.sandberg@arm.com        self.skip_run = skip
26411482Sandreas.sandberg@arm.com        self.skip_diff_out = skip or skip_diff_out
26511482Sandreas.sandberg@arm.com        self.skip_diff_stat = skip or skip_diff_stat
26611482Sandreas.sandberg@arm.com
26711482Sandreas.sandberg@arm.com    def ref_files(self):
26811482Sandreas.sandberg@arm.com        ref_dir = os.path.abspath(self.ref_dir)
26911482Sandreas.sandberg@arm.com        for root, dirs, files in os.walk(ref_dir, topdown=False):
27011482Sandreas.sandberg@arm.com            for f in files:
27111482Sandreas.sandberg@arm.com                fpath = os.path.join(root[len(ref_dir) + 1:], f)
27211503Sandreas.sandberg@arm.com                if fpath not in ClassicTest.ref_ignore_files:
27311482Sandreas.sandberg@arm.com                    yield fpath
27411482Sandreas.sandberg@arm.com
27511482Sandreas.sandberg@arm.com    def run_units(self):
27611482Sandreas.sandberg@arm.com        args = [
27711482Sandreas.sandberg@arm.com            self.script,
27811482Sandreas.sandberg@arm.com            "/".join(self.config_tuple),
27911482Sandreas.sandberg@arm.com        ]
28011482Sandreas.sandberg@arm.com
28111482Sandreas.sandberg@arm.com        return [
28211482Sandreas.sandberg@arm.com            RunGem5(self.gem5, args,
28311482Sandreas.sandberg@arm.com                    ref_dir=self.ref_dir, test_dir=self.output_dir,
28411482Sandreas.sandberg@arm.com                    skip=self.skip_run),
28511482Sandreas.sandberg@arm.com        ]
28611482Sandreas.sandberg@arm.com
28711482Sandreas.sandberg@arm.com    def verify_units(self):
28811636Sandreas.sandberg@arm.com        ref_files = set(self.ref_files())
28911636Sandreas.sandberg@arm.com        units = []
29011636Sandreas.sandberg@arm.com        if "stats.txt" in ref_files:
29111636Sandreas.sandberg@arm.com            units.append(
29211636Sandreas.sandberg@arm.com                DiffStatFile(ref_dir=self.ref_dir, test_dir=self.output_dir,
29311636Sandreas.sandberg@arm.com                             skip=self.skip_diff_stat))
29411636Sandreas.sandberg@arm.com        units += [
29511482Sandreas.sandberg@arm.com            DiffOutFile(f,
29611482Sandreas.sandberg@arm.com                        ref_dir=self.ref_dir, test_dir=self.output_dir,
29711482Sandreas.sandberg@arm.com                        skip=self.skip_diff_out)
29811636Sandreas.sandberg@arm.com            for f in ref_files if f not in ClassicTest.diff_ignore_files
29911482Sandreas.sandberg@arm.com        ]
30011482Sandreas.sandberg@arm.com
30111636Sandreas.sandberg@arm.com        return units
30211636Sandreas.sandberg@arm.com
30311482Sandreas.sandberg@arm.com    def update_ref(self):
30411482Sandreas.sandberg@arm.com        for fname in self.ref_files():
30511482Sandreas.sandberg@arm.com            shutil.copy(
30611482Sandreas.sandberg@arm.com                os.path.join(self.output_dir, fname),
30711482Sandreas.sandberg@arm.com                os.path.join(self.ref_dir, fname))
30811482Sandreas.sandberg@arm.com
30911482Sandreas.sandberg@arm.comdef parse_test_filter(test_filter):
31011482Sandreas.sandberg@arm.com    wildcards = ("", "*")
31111482Sandreas.sandberg@arm.com
31211482Sandreas.sandberg@arm.com    _filter = list(test_filter.split("/"))
31311482Sandreas.sandberg@arm.com    if len(_filter) > 3:
31411482Sandreas.sandberg@arm.com        raise RuntimeError("Illegal test filter string")
31511482Sandreas.sandberg@arm.com    _filter += [ "", ] * (3 - len(_filter))
31611482Sandreas.sandberg@arm.com
31711482Sandreas.sandberg@arm.com    isa, cat, mode = _filter
31811482Sandreas.sandberg@arm.com
31911482Sandreas.sandberg@arm.com    if isa in wildcards:
32011482Sandreas.sandberg@arm.com        raise RuntimeError("No ISA specified")
32111482Sandreas.sandberg@arm.com
32211482Sandreas.sandberg@arm.com    cat = all_categories if cat in wildcards else (cat, )
32311482Sandreas.sandberg@arm.com    mode = all_modes if mode in wildcards else (mode, )
32411482Sandreas.sandberg@arm.com
32511482Sandreas.sandberg@arm.com    return isa, cat, mode
32611482Sandreas.sandberg@arm.com
32711482Sandreas.sandberg@arm.comdef get_tests(isa,
32811482Sandreas.sandberg@arm.com              categories=all_categories, modes=all_modes,
32911482Sandreas.sandberg@arm.com              ruby_protocol=None, gpu_isa=None):
33011482Sandreas.sandberg@arm.com
33111482Sandreas.sandberg@arm.com    # Generate a list of candidate configs
33211482Sandreas.sandberg@arm.com    configs = list(arch_configs.get((isa, gpu_isa), []))
33311482Sandreas.sandberg@arm.com
33411482Sandreas.sandberg@arm.com    if (isa, gpu_isa) == ("x86", "hsail"):
33511482Sandreas.sandberg@arm.com        if ruby_protocol == "GPU_RfO":
33611482Sandreas.sandberg@arm.com            configs += ['gpu-randomtest']
33711482Sandreas.sandberg@arm.com    else:
33811482Sandreas.sandberg@arm.com        configs += generic_configs
33911482Sandreas.sandberg@arm.com
34011482Sandreas.sandberg@arm.com    if ruby_protocol == 'MI_example':
34111482Sandreas.sandberg@arm.com        configs += [ "%s-ruby" % (c, ) for c in configs ]
34211482Sandreas.sandberg@arm.com    elif ruby_protocol is not None:
34311511Sandreas.sandberg@arm.com        # Override generic ISA configs when using Ruby (excluding
34411511Sandreas.sandberg@arm.com        # MI_example which is included in all ISAs by default). This
34511511Sandreas.sandberg@arm.com        # reduces the number of generic tests we re-run for when
34611511Sandreas.sandberg@arm.com        # compiling Ruby targets.
34711511Sandreas.sandberg@arm.com        configs = [ "%s-ruby-%s" % (c, ruby_protocol) for c in configs ]
34811482Sandreas.sandberg@arm.com
34911482Sandreas.sandberg@arm.com    # /(quick|long)/(fs|se)/workload/ref/arch/guest/config/
35011482Sandreas.sandberg@arm.com    for conf_script in configs:
35111482Sandreas.sandberg@arm.com        for cat in categories:
35211482Sandreas.sandberg@arm.com            for mode in modes:
35311482Sandreas.sandberg@arm.com                mode_dir = os.path.join(_test_base, cat, mode)
35411482Sandreas.sandberg@arm.com                if not os.path.exists(mode_dir):
35511482Sandreas.sandberg@arm.com                    continue
35611482Sandreas.sandberg@arm.com
35711482Sandreas.sandberg@arm.com                for workload in os.listdir(mode_dir):
35811482Sandreas.sandberg@arm.com                    isa_dir = os.path.join(mode_dir, workload, "ref", isa)
35911482Sandreas.sandberg@arm.com                    if not os.path.isdir(isa_dir):
36011482Sandreas.sandberg@arm.com                        continue
36111482Sandreas.sandberg@arm.com
36211482Sandreas.sandberg@arm.com                    for _os in os.listdir(isa_dir):
36311482Sandreas.sandberg@arm.com                        test_dir = os.path.join(isa_dir, _os, conf_script)
36411482Sandreas.sandberg@arm.com                        if not os.path.exists(test_dir) or \
36511482Sandreas.sandberg@arm.com                           os.path.exists(os.path.join(test_dir, "skip")):
36611482Sandreas.sandberg@arm.com                            continue
36711482Sandreas.sandberg@arm.com
36811482Sandreas.sandberg@arm.com                        yield ClassicConfig(cat, mode, workload, isa, _os,
36911482Sandreas.sandberg@arm.com                                            conf_script)
370