tests.py revision 11542:ecd058e3dcbe
16657Snate@binkert.org#!/usr/bin/env python 26657Snate@binkert.org# 36657Snate@binkert.org# Copyright (c) 2016 ARM Limited 46657Snate@binkert.org# All rights reserved 56657Snate@binkert.org# 66657Snate@binkert.org# The license below extends only to copyright in the software and shall 76657Snate@binkert.org# not be construed as granting a license to any other intellectual 86657Snate@binkert.org# property including but not limited to intellectual property relating 96657Snate@binkert.org# to a hardware implementation of the functionality of the software 106657Snate@binkert.org# licensed hereunder. You may use the software subject to the license 116657Snate@binkert.org# terms below provided that you ensure that this notice is replicated 126657Snate@binkert.org# unmodified and in its entirety in all distributions of the software, 136657Snate@binkert.org# modified or unmodified, in source code or in binary form. 146657Snate@binkert.org# 156657Snate@binkert.org# Redistribution and use in source and binary forms, with or without 166657Snate@binkert.org# modification, are permitted provided that the following conditions are 176657Snate@binkert.org# met: redistributions of source code must retain the above copyright 186657Snate@binkert.org# notice, this list of conditions and the following disclaimer; 196657Snate@binkert.org# redistributions in binary form must reproduce the above copyright 206657Snate@binkert.org# notice, this list of conditions and the following disclaimer in the 216657Snate@binkert.org# documentation and/or other materials provided with the distribution; 226657Snate@binkert.org# neither the name of the copyright holders nor the names of its 236657Snate@binkert.org# contributors may be used to endorse or promote products derived from 246657Snate@binkert.org# this software without specific prior written permission. 256657Snate@binkert.org# 266657Snate@binkert.org# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS 276657Snate@binkert.org# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT 286999Snate@binkert.org# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR 296657Snate@binkert.org# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT 306657Snate@binkert.org# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, 316657Snate@binkert.org# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT 326657Snate@binkert.org# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, 338189SLisa.Hsu@amd.com# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY 346657Snate@binkert.org# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT 356882SBrad.Beckmann@amd.com# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE 367055Snate@binkert.org# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 376882SBrad.Beckmann@amd.com# 386882SBrad.Beckmann@amd.com# Authors: Andreas Sandberg 398191SLisa.Hsu@amd.com 406882SBrad.Beckmann@amd.comfrom abc import ABCMeta, abstractmethod 416882SBrad.Beckmann@amd.comimport os 426882SBrad.Beckmann@amd.comfrom collections import namedtuple 436888SBrad.Beckmann@amd.comfrom units import * 446882SBrad.Beckmann@amd.comfrom results import TestResult 456882SBrad.Beckmann@amd.comimport shutil 466657Snate@binkert.org 476657Snate@binkert.org_test_base = os.path.join(os.path.dirname(__file__), "..") 486657Snate@binkert.org 496657Snate@binkert.orgClassicConfig = namedtuple("ClassicConfig", ( 506657Snate@binkert.org "category", 517839Snilay@cs.wisc.edu "mode", 526657Snate@binkert.org "workload", 536882SBrad.Beckmann@amd.com "isa", 546882SBrad.Beckmann@amd.com "os", 556882SBrad.Beckmann@amd.com "config", 566882SBrad.Beckmann@amd.com)) 576882SBrad.Beckmann@amd.com 586882SBrad.Beckmann@amd.com# There are currently two "classes" of test 596657Snate@binkert.org# configurations. Architecture-specific ones and generic ones 606657Snate@binkert.org# (typically SE mode tests). In both cases, the configuration name 616657Snate@binkert.org# matches a file in tests/configs/ that will be picked up by the test 626657Snate@binkert.org# runner (run.py). 636657Snate@binkert.org# 646657Snate@binkert.org# Architecture specific configurations are listed in the arch_configs 656657Snate@binkert.org# dictionary. This is indexed by a (cpu architecture, gpu 666657Snate@binkert.org# architecture) tuple. GPU architecture is optional and may be None. 676657Snate@binkert.org# 687839Snilay@cs.wisc.edu# Generic configurations are listed in the generic_configs tuple. 697839Snilay@cs.wisc.edu# 706657Snate@binkert.org# When discovering available test cases, this script look uses the 716657Snate@binkert.org# test list as a list of /candidate/ configurations. A configuration 726657Snate@binkert.org# is only used if a test has a reference output for that 736657Snate@binkert.org# configuration. In addition to the base configurations from 746657Snate@binkert.org# arch_configs and generic_configs, a Ruby configuration may be 756657Snate@binkert.org# appended to the base name (this is probed /in addition/ to the 766657Snate@binkert.org# original name. See get_tests() for details. 776657Snate@binkert.org# 786657Snate@binkert.orgarch_configs = { 796657Snate@binkert.org ("alpha", None) : ( 806657Snate@binkert.org 'tsunami-simple-atomic', 816657Snate@binkert.org 'tsunami-simple-timing', 826657Snate@binkert.org 'tsunami-simple-atomic-dual', 836657Snate@binkert.org 'tsunami-simple-timing-dual', 846657Snate@binkert.org 'twosys-tsunami-simple-atomic', 856657Snate@binkert.org 'tsunami-o3', 'tsunami-o3-dual', 866657Snate@binkert.org 'tsunami-minor', 'tsunami-minor-dual', 876657Snate@binkert.org 'tsunami-switcheroo-full', 886657Snate@binkert.org ), 896657Snate@binkert.org 906779SBrad.Beckmann@amd.com ("arm", None) : ( 916657Snate@binkert.org 'simple-atomic-dummychecker', 926657Snate@binkert.org 'o3-timing-checker', 936657Snate@binkert.org 'realview-simple-atomic', 946657Snate@binkert.org 'realview-simple-atomic-dual', 956657Snate@binkert.org 'realview-simple-atomic-checkpoint', 966657Snate@binkert.org 'realview-simple-timing', 976657Snate@binkert.org 'realview-simple-timing-dual', 986657Snate@binkert.org 'realview-o3', 996657Snate@binkert.org 'realview-o3-checker', 1006657Snate@binkert.org 'realview-o3-dual', 1016657Snate@binkert.org 'realview-minor', 1026657Snate@binkert.org 'realview-minor-dual', 1036657Snate@binkert.org 'realview-switcheroo-atomic', 1046657Snate@binkert.org 'realview-switcheroo-timing', 1056657Snate@binkert.org 'realview-switcheroo-o3', 1066657Snate@binkert.org 'realview-switcheroo-full', 1076657Snate@binkert.org 'realview64-simple-atomic', 1086657Snate@binkert.org 'realview64-simple-atomic-checkpoint', 1096657Snate@binkert.org 'realview64-simple-atomic-dual', 1106657Snate@binkert.org 'realview64-simple-timing', 1116657Snate@binkert.org 'realview64-simple-timing-dual', 1126657Snate@binkert.org 'realview64-o3', 1136657Snate@binkert.org 'realview64-o3-checker', 1146657Snate@binkert.org 'realview64-o3-dual', 1157839Snilay@cs.wisc.edu 'realview64-minor', 1167839Snilay@cs.wisc.edu 'realview64-minor-dual', 1177839Snilay@cs.wisc.edu 'realview64-switcheroo-atomic', 1187839Snilay@cs.wisc.edu 'realview64-switcheroo-timing', 1197839Snilay@cs.wisc.edu 'realview64-switcheroo-o3', 1207839Snilay@cs.wisc.edu 'realview64-switcheroo-full', 1217839Snilay@cs.wisc.edu ), 1227839Snilay@cs.wisc.edu 1237839Snilay@cs.wisc.edu ("sparc", None) : ( 1247839Snilay@cs.wisc.edu 't1000-simple-atomic', 1257839Snilay@cs.wisc.edu 't1000-simple-x86', 1267839Snilay@cs.wisc.edu ), 1277839Snilay@cs.wisc.edu 1287839Snilay@cs.wisc.edu ("timing", None) : ( 1297839Snilay@cs.wisc.edu 'pc-simple-atomic', 1306657Snate@binkert.org 'pc-simple-timing', 1316657Snate@binkert.org 'pc-o3-timing', 1326657Snate@binkert.org 'pc-switcheroo-full', 1336657Snate@binkert.org ), 1346657Snate@binkert.org 1356657Snate@binkert.org ("x86", "hsail") : ( 1366657Snate@binkert.org 'gpu', 1376657Snate@binkert.org ), 1386657Snate@binkert.org} 1396657Snate@binkert.org 1406657Snate@binkert.orggeneric_configs = ( 1416657Snate@binkert.org 'simple-atomic', 1426657Snate@binkert.org 'simple-atomic-mp', 1436657Snate@binkert.org 'simple-timing', 1446657Snate@binkert.org 'simple-timing-mp', 1456657Snate@binkert.org 1466657Snate@binkert.org 'minor-timing', 1476657Snate@binkert.org 'minor-timing-mp', 1486657Snate@binkert.org 1496657Snate@binkert.org 'o3-timing', 1506657Snate@binkert.org 'o3-timing-mt', 1516657Snate@binkert.org 'o3-timing-mp', 1526657Snate@binkert.org 1536657Snate@binkert.org 'rubytest', 1546657Snate@binkert.org 'memcheck', 1556657Snate@binkert.org 'memtest', 1566657Snate@binkert.org 'memtest-filter', 1576657Snate@binkert.org 'tgen-simple-mem', 1586657Snate@binkert.org 'tgen-dram-ctrl', 1596657Snate@binkert.org 1606657Snate@binkert.org 'learning-gem5-p1-simple', 1616877Ssteve.reinhardt@amd.com 'learning-gem5-p1-two-level', 1626657Snate@binkert.org) 1636657Snate@binkert.org 1646657Snate@binkert.orgall_categories = ("quick", "long") 1656657Snate@binkert.orgall_modes = ("fs", "se") 1666657Snate@binkert.org 1676657Snate@binkert.orgclass Test(object): 1687542SBrad.Beckmann@amd.com """Test case base class. 1697542SBrad.Beckmann@amd.com 1706657Snate@binkert.org Test cases consists of one or more test units that are run in two 1716877Ssteve.reinhardt@amd.com phases. A run phase (units produced by run_units() and a verify 1726999Snate@binkert.org phase (units from verify_units()). The verify phase is skipped if 1736877Ssteve.reinhardt@amd.com the run phase fails. 1746877Ssteve.reinhardt@amd.com 1756877Ssteve.reinhardt@amd.com """ 1766877Ssteve.reinhardt@amd.com 1776877Ssteve.reinhardt@amd.com __metaclass__ = ABCMeta 1786877Ssteve.reinhardt@amd.com 1796877Ssteve.reinhardt@amd.com def __init__(self, name): 1806877Ssteve.reinhardt@amd.com self.test_name = name 1816877Ssteve.reinhardt@amd.com 1826877Ssteve.reinhardt@amd.com @abstractmethod 1836877Ssteve.reinhardt@amd.com def ref_files(self): 1846877Ssteve.reinhardt@amd.com """Get a list of reference files used by this test case""" 1856877Ssteve.reinhardt@amd.com pass 1866877Ssteve.reinhardt@amd.com 1876877Ssteve.reinhardt@amd.com @abstractmethod 1886877Ssteve.reinhardt@amd.com def run_units(self): 1896882SBrad.Beckmann@amd.com """Units (typically RunGem5 instances) that describe the run phase of 1906882SBrad.Beckmann@amd.com this test. 1916882SBrad.Beckmann@amd.com 1926882SBrad.Beckmann@amd.com """ 1936882SBrad.Beckmann@amd.com pass 1946882SBrad.Beckmann@amd.com 1956882SBrad.Beckmann@amd.com @abstractmethod 1966877Ssteve.reinhardt@amd.com def verify_units(self): 1976877Ssteve.reinhardt@amd.com """Verify the output from the run phase (see run_units()).""" 1986877Ssteve.reinhardt@amd.com pass 1996877Ssteve.reinhardt@amd.com 2006657Snate@binkert.org @abstractmethod 2016657Snate@binkert.org def update_ref(self): 2026999Snate@binkert.org """Update reference files with files from a test run""" 2036657Snate@binkert.org pass 2046657Snate@binkert.org 2056657Snate@binkert.org def run(self): 2066657Snate@binkert.org """Run this test case and return a list of results""" 2076657Snate@binkert.org 2086657Snate@binkert.org run_results = [ u.run() for u in self.run_units() ] 2097007Snate@binkert.org run_ok = all([not r.skipped() and r for r in run_results ]) 2106657Snate@binkert.org 2116657Snate@binkert.org verify_results = [ 2126657Snate@binkert.org u.run() if run_ok else u.skip() 2136657Snate@binkert.org for u in self.verify_units() 2146657Snate@binkert.org ] 2157007Snate@binkert.org 2167007Snate@binkert.org return TestResult(self.test_name, 2176657Snate@binkert.org run_results=run_results, 2187002Snate@binkert.org verify_results=verify_results) 2197002Snate@binkert.org 2207002Snate@binkert.org def __str__(self): 2217002Snate@binkert.org return self.test_name 2228229Snate@binkert.org 2238229Snate@binkert.orgclass ClassicTest(Test): 2246657Snate@binkert.org # The diff ignore list contains all files that shouldn't be diffed 2256657Snate@binkert.org # using DiffOutFile. These files typically use special-purpose 2268229Snate@binkert.org # diff tools (e.g., DiffStatFile). 2278229Snate@binkert.org diff_ignore_files = ( 2288229Snate@binkert.org # Stat files use a special stat differ 2298229Snate@binkert.org "stats.txt", 2306657Snate@binkert.org ) 2316657Snate@binkert.org 2326657Snate@binkert.org # These files should never be included in the list of 2336657Snate@binkert.org # reference files. This list should include temporary files 2346793SBrad.Beckmann@amd.com # and other files that we don't care about. 2356657Snate@binkert.org ref_ignore_files = ( 2366657Snate@binkert.org ) 2376657Snate@binkert.org 2386657Snate@binkert.org def __init__(self, gem5, output_dir, config_tuple, 2396657Snate@binkert.org timeout=None, 2407002Snate@binkert.org skip=False, skip_diff_out=False, skip_diff_stat=False): 2416657Snate@binkert.org 2427007Snate@binkert.org super(ClassicTest, self).__init__("/".join(config_tuple)) 2437007Snate@binkert.org 2447007Snate@binkert.org ct = config_tuple 2457007Snate@binkert.org 2467007Snate@binkert.org self.gem5 = os.path.abspath(gem5) 2476657Snate@binkert.org self.script = os.path.join(_test_base, "run.py") 2486877Ssteve.reinhardt@amd.com self.config_tuple = ct 2496877Ssteve.reinhardt@amd.com self.timeout = timeout 2506657Snate@binkert.org 2516877Ssteve.reinhardt@amd.com self.output_dir = output_dir 2526657Snate@binkert.org self.ref_dir = os.path.join(_test_base, 2536657Snate@binkert.org ct.category, ct.mode, ct.workload, 2547002Snate@binkert.org "ref", ct.isa, ct.os, ct.config) 2557002Snate@binkert.org self.skip_run = skip 2567567SBrad.Beckmann@amd.com self.skip_diff_out = skip or skip_diff_out 2577567SBrad.Beckmann@amd.com self.skip_diff_stat = skip or skip_diff_stat 2587922SBrad.Beckmann@amd.com 2596881SBrad.Beckmann@amd.com def ref_files(self): 2607002Snate@binkert.org ref_dir = os.path.abspath(self.ref_dir) 2617002Snate@binkert.org for root, dirs, files in os.walk(ref_dir, topdown=False): 2626657Snate@binkert.org for f in files: 2637002Snate@binkert.org fpath = os.path.join(root[len(ref_dir) + 1:], f) 2646902SBrad.Beckmann@amd.com if fpath not in ClassicTest.ref_ignore_files: 2656863Sdrh5@cs.wisc.edu yield fpath 2666863Sdrh5@cs.wisc.edu 2678683Snilay@cs.wisc.edu def run_units(self): 2688683Snilay@cs.wisc.edu args = [ 2697007Snate@binkert.org self.script, 2706657Snate@binkert.org "/".join(self.config_tuple), 2716657Snate@binkert.org ] 2726657Snate@binkert.org 2736657Snate@binkert.org return [ 2746657Snate@binkert.org RunGem5(self.gem5, args, 2756657Snate@binkert.org ref_dir=self.ref_dir, test_dir=self.output_dir, 2766882SBrad.Beckmann@amd.com skip=self.skip_run), 2776882SBrad.Beckmann@amd.com ] 2786882SBrad.Beckmann@amd.com 2796882SBrad.Beckmann@amd.com def verify_units(self): 2806657Snate@binkert.org return [ 2816657Snate@binkert.org DiffStatFile(ref_dir=self.ref_dir, test_dir=self.output_dir, 2826657Snate@binkert.org skip=self.skip_diff_stat) 2836657Snate@binkert.org ] + [ 2847007Snate@binkert.org DiffOutFile(f, 2857839Snilay@cs.wisc.edu ref_dir=self.ref_dir, test_dir=self.output_dir, 2867839Snilay@cs.wisc.edu skip=self.skip_diff_out) 2877839Snilay@cs.wisc.edu for f in self.ref_files() 2887839Snilay@cs.wisc.edu if f not in ClassicTest.diff_ignore_files 2897839Snilay@cs.wisc.edu ] 2907839Snilay@cs.wisc.edu 2917839Snilay@cs.wisc.edu def update_ref(self): 2927839Snilay@cs.wisc.edu for fname in self.ref_files(): 2937839Snilay@cs.wisc.edu shutil.copy( 2947839Snilay@cs.wisc.edu os.path.join(self.output_dir, fname), 2957839Snilay@cs.wisc.edu os.path.join(self.ref_dir, fname)) 2967839Snilay@cs.wisc.edu 2977007Snate@binkert.orgdef parse_test_filter(test_filter): 2987007Snate@binkert.org wildcards = ("", "*") 2997007Snate@binkert.org 3007007Snate@binkert.org _filter = list(test_filter.split("/")) 3017007Snate@binkert.org if len(_filter) > 3: 3027839Snilay@cs.wisc.edu raise RuntimeError("Illegal test filter string") 3037839Snilay@cs.wisc.edu _filter += [ "", ] * (3 - len(_filter)) 3047839Snilay@cs.wisc.edu 3057839Snilay@cs.wisc.edu isa, cat, mode = _filter 3067839Snilay@cs.wisc.edu 3077839Snilay@cs.wisc.edu if isa in wildcards: 3087839Snilay@cs.wisc.edu raise RuntimeError("No ISA specified") 3097839Snilay@cs.wisc.edu 3107839Snilay@cs.wisc.edu cat = all_categories if cat in wildcards else (cat, ) 3117839Snilay@cs.wisc.edu mode = all_modes if mode in wildcards else (mode, ) 3127839Snilay@cs.wisc.edu 3137839Snilay@cs.wisc.edu return isa, cat, mode 3147007Snate@binkert.org 3157007Snate@binkert.orgdef get_tests(isa, 3167002Snate@binkert.org categories=all_categories, modes=all_modes, 3176657Snate@binkert.org ruby_protocol=None, gpu_isa=None): 3186657Snate@binkert.org 3196657Snate@binkert.org # Generate a list of candidate configs 3207055Snate@binkert.org configs = list(arch_configs.get((isa, gpu_isa), [])) 3216657Snate@binkert.org 3226657Snate@binkert.org if (isa, gpu_isa) == ("x86", "hsail"): 3236657Snate@binkert.org if ruby_protocol == "GPU_RfO": 3246863Sdrh5@cs.wisc.edu configs += ['gpu-randomtest'] 3257055Snate@binkert.org else: 3267567SBrad.Beckmann@amd.com configs += generic_configs 3278943Sandreas.hansson@arm.com 3287567SBrad.Beckmann@amd.com if ruby_protocol == 'MI_example': 3297567SBrad.Beckmann@amd.com configs += [ "%s-ruby" % (c, ) for c in configs ] 3307567SBrad.Beckmann@amd.com elif ruby_protocol is not None: 3317542SBrad.Beckmann@amd.com # Override generic ISA configs when using Ruby (excluding 3327542SBrad.Beckmann@amd.com # MI_example which is included in all ISAs by default). This 3336657Snate@binkert.org # reduces the number of generic tests we re-run for when 3347007Snate@binkert.org # compiling Ruby targets. 3356657Snate@binkert.org configs = [ "%s-ruby-%s" % (c, ruby_protocol) for c in configs ] 3366657Snate@binkert.org 3376657Snate@binkert.org # /(quick|long)/(fs|se)/workload/ref/arch/guest/config/ 3386657Snate@binkert.org for conf_script in configs: 3396657Snate@binkert.org for cat in categories: 3406657Snate@binkert.org for mode in modes: 3416657Snate@binkert.org mode_dir = os.path.join(_test_base, cat, mode) 3426657Snate@binkert.org if not os.path.exists(mode_dir): 3437839Snilay@cs.wisc.edu continue 3447839Snilay@cs.wisc.edu 3457839Snilay@cs.wisc.edu for workload in os.listdir(mode_dir): 3467839Snilay@cs.wisc.edu isa_dir = os.path.join(mode_dir, workload, "ref", isa) 3477839Snilay@cs.wisc.edu if not os.path.isdir(isa_dir): 3487839Snilay@cs.wisc.edu continue 3497839Snilay@cs.wisc.edu 3507839Snilay@cs.wisc.edu for _os in os.listdir(isa_dir): 3517839Snilay@cs.wisc.edu test_dir = os.path.join(isa_dir, _os, conf_script) 3527839Snilay@cs.wisc.edu if not os.path.exists(test_dir) or \ 3537839Snilay@cs.wisc.edu os.path.exists(os.path.join(test_dir, "skip")): 3547839Snilay@cs.wisc.edu continue 3557839Snilay@cs.wisc.edu 3567839Snilay@cs.wisc.edu yield ClassicConfig(cat, mode, workload, isa, _os, 3577839Snilay@cs.wisc.edu conf_script) 3587839Snilay@cs.wisc.edu