tests.py revision 11542
16657Snate@binkert.org#!/usr/bin/env python 26657Snate@binkert.org# 36657Snate@binkert.org# Copyright (c) 2016 ARM Limited 46657Snate@binkert.org# All rights reserved 56657Snate@binkert.org# 66657Snate@binkert.org# The license below extends only to copyright in the software and shall 76657Snate@binkert.org# not be construed as granting a license to any other intellectual 86657Snate@binkert.org# property including but not limited to intellectual property relating 96657Snate@binkert.org# to a hardware implementation of the functionality of the software 106657Snate@binkert.org# licensed hereunder. You may use the software subject to the license 116657Snate@binkert.org# terms below provided that you ensure that this notice is replicated 126657Snate@binkert.org# unmodified and in its entirety in all distributions of the software, 136657Snate@binkert.org# modified or unmodified, in source code or in binary form. 146657Snate@binkert.org# 156657Snate@binkert.org# Redistribution and use in source and binary forms, with or without 166657Snate@binkert.org# modification, are permitted provided that the following conditions are 176657Snate@binkert.org# met: redistributions of source code must retain the above copyright 186657Snate@binkert.org# notice, this list of conditions and the following disclaimer; 196657Snate@binkert.org# redistributions in binary form must reproduce the above copyright 206657Snate@binkert.org# notice, this list of conditions and the following disclaimer in the 216657Snate@binkert.org# documentation and/or other materials provided with the distribution; 226657Snate@binkert.org# neither the name of the copyright holders nor the names of its 236657Snate@binkert.org# contributors may be used to endorse or promote products derived from 246657Snate@binkert.org# this software without specific prior written permission. 256657Snate@binkert.org# 266657Snate@binkert.org# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS 276657Snate@binkert.org# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT 286999Snate@binkert.org# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR 296657Snate@binkert.org# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT 306657Snate@binkert.org# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, 316657Snate@binkert.org# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT 326657Snate@binkert.org# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, 338189SLisa.Hsu@amd.com# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY 346657Snate@binkert.org# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT 359499Snilay@cs.wisc.edu# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE 369499Snilay@cs.wisc.edu# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 379364Snilay@cs.wisc.edu# 387055Snate@binkert.org# Authors: Andreas Sandberg 396882SBrad.Beckmann@amd.com 406882SBrad.Beckmann@amd.comfrom abc import ABCMeta, abstractmethod 418191SLisa.Hsu@amd.comimport os 426882SBrad.Beckmann@amd.comfrom collections import namedtuple 436882SBrad.Beckmann@amd.comfrom units import * 449102SNuwan.Jayasena@amd.comfrom results import TestResult 459366Snilay@cs.wisc.eduimport shutil 469499Snilay@cs.wisc.edu 479499Snilay@cs.wisc.edu_test_base = os.path.join(os.path.dirname(__file__), "..") 489499Snilay@cs.wisc.edu 496882SBrad.Beckmann@amd.comClassicConfig = namedtuple("ClassicConfig", ( 506657Snate@binkert.org "category", 516657Snate@binkert.org "mode", 526657Snate@binkert.org "workload", 536657Snate@binkert.org "isa", 5410311Snilay@cs.wisc.edu "os", 5510311Snilay@cs.wisc.edu "config", 5610311Snilay@cs.wisc.edu)) 5710311Snilay@cs.wisc.edu 586657Snate@binkert.org# There are currently two "classes" of test 5910311Snilay@cs.wisc.edu# configurations. Architecture-specific ones and generic ones 609366Snilay@cs.wisc.edu# (typically SE mode tests). In both cases, the configuration name 617839Snilay@cs.wisc.edu# matches a file in tests/configs/ that will be picked up by the test 626657Snate@binkert.org# runner (run.py). 636882SBrad.Beckmann@amd.com# 6410308Snilay@cs.wisc.edu# Architecture specific configurations are listed in the arch_configs 6510308Snilay@cs.wisc.edu# dictionary. This is indexed by a (cpu architecture, gpu 666882SBrad.Beckmann@amd.com# architecture) tuple. GPU architecture is optional and may be None. 6710308Snilay@cs.wisc.edu# 6810308Snilay@cs.wisc.edu# Generic configurations are listed in the generic_configs tuple. 6910308Snilay@cs.wisc.edu# 7010308Snilay@cs.wisc.edu# When discovering available test cases, this script look uses the 7110308Snilay@cs.wisc.edu# test list as a list of /candidate/ configurations. A configuration 729366Snilay@cs.wisc.edu# is only used if a test has a reference output for that 739366Snilay@cs.wisc.edu# configuration. In addition to the base configurations from 746657Snate@binkert.org# arch_configs and generic_configs, a Ruby configuration may be 756657Snate@binkert.org# appended to the base name (this is probed /in addition/ to the 766657Snate@binkert.org# original name. See get_tests() for details. 776657Snate@binkert.org# 789104Shestness@cs.utexas.eduarch_configs = { 796657Snate@binkert.org ("alpha", None) : ( 806657Snate@binkert.org 'tsunami-simple-atomic', 816657Snate@binkert.org 'tsunami-simple-timing', 8210311Snilay@cs.wisc.edu 'tsunami-simple-atomic-dual', 8310311Snilay@cs.wisc.edu 'tsunami-simple-timing-dual', 8410311Snilay@cs.wisc.edu 'twosys-tsunami-simple-atomic', 8510311Snilay@cs.wisc.edu 'tsunami-o3', 'tsunami-o3-dual', 866657Snate@binkert.org 'tsunami-minor', 'tsunami-minor-dual', 877839Snilay@cs.wisc.edu 'tsunami-switcheroo-full', 887839Snilay@cs.wisc.edu ), 896657Snate@binkert.org 906657Snate@binkert.org ("arm", None) : ( 916657Snate@binkert.org 'simple-atomic-dummychecker', 926657Snate@binkert.org 'o3-timing-checker', 936657Snate@binkert.org 'realview-simple-atomic', 946657Snate@binkert.org 'realview-simple-atomic-dual', 956657Snate@binkert.org 'realview-simple-atomic-checkpoint', 966657Snate@binkert.org 'realview-simple-timing', 976657Snate@binkert.org 'realview-simple-timing-dual', 986657Snate@binkert.org 'realview-o3', 996657Snate@binkert.org 'realview-o3-checker', 1006657Snate@binkert.org 'realview-o3-dual', 1016657Snate@binkert.org 'realview-minor', 1026657Snate@binkert.org 'realview-minor-dual', 1036657Snate@binkert.org 'realview-switcheroo-atomic', 1046657Snate@binkert.org 'realview-switcheroo-timing', 1056657Snate@binkert.org 'realview-switcheroo-o3', 1066657Snate@binkert.org 'realview-switcheroo-full', 1076779SBrad.Beckmann@amd.com 'realview64-simple-atomic', 1086657Snate@binkert.org 'realview64-simple-atomic-checkpoint', 1096657Snate@binkert.org 'realview64-simple-atomic-dual', 1106657Snate@binkert.org 'realview64-simple-timing', 1116657Snate@binkert.org 'realview64-simple-timing-dual', 1126657Snate@binkert.org 'realview64-o3', 1136657Snate@binkert.org 'realview64-o3-checker', 1146657Snate@binkert.org 'realview64-o3-dual', 1156657Snate@binkert.org 'realview64-minor', 1166657Snate@binkert.org 'realview64-minor-dual', 1179104Shestness@cs.utexas.edu 'realview64-switcheroo-atomic', 1189104Shestness@cs.utexas.edu 'realview64-switcheroo-timing', 1199104Shestness@cs.utexas.edu 'realview64-switcheroo-o3', 1209104Shestness@cs.utexas.edu 'realview64-switcheroo-full', 1216657Snate@binkert.org ), 1226657Snate@binkert.org 1236657Snate@binkert.org ("sparc", None) : ( 1246657Snate@binkert.org 't1000-simple-atomic', 1256657Snate@binkert.org 't1000-simple-x86', 1266657Snate@binkert.org ), 1276657Snate@binkert.org 1286657Snate@binkert.org ("timing", None) : ( 1296657Snate@binkert.org 'pc-simple-atomic', 1306657Snate@binkert.org 'pc-simple-timing', 1316657Snate@binkert.org 'pc-o3-timing', 1326657Snate@binkert.org 'pc-switcheroo-full', 1336657Snate@binkert.org ), 13410307Snilay@cs.wisc.edu 1356657Snate@binkert.org ("x86", "hsail") : ( 1366657Snate@binkert.org 'gpu', 1377839Snilay@cs.wisc.edu ), 1387839Snilay@cs.wisc.edu} 1397839Snilay@cs.wisc.edu 1407839Snilay@cs.wisc.edugeneric_configs = ( 1417839Snilay@cs.wisc.edu 'simple-atomic', 1427839Snilay@cs.wisc.edu 'simple-atomic-mp', 1437839Snilay@cs.wisc.edu 'simple-timing', 1447839Snilay@cs.wisc.edu 'simple-timing-mp', 1457839Snilay@cs.wisc.edu 1467839Snilay@cs.wisc.edu 'minor-timing', 1477839Snilay@cs.wisc.edu 'minor-timing-mp', 1487839Snilay@cs.wisc.edu 1497839Snilay@cs.wisc.edu 'o3-timing', 1507839Snilay@cs.wisc.edu 'o3-timing-mt', 1517839Snilay@cs.wisc.edu 'o3-timing-mp', 1526657Snate@binkert.org 1536657Snate@binkert.org 'rubytest', 1546657Snate@binkert.org 'memcheck', 1556657Snate@binkert.org 'memtest', 1566657Snate@binkert.org 'memtest-filter', 1576657Snate@binkert.org 'tgen-simple-mem', 1586657Snate@binkert.org 'tgen-dram-ctrl', 1596657Snate@binkert.org 1606657Snate@binkert.org 'learning-gem5-p1-simple', 1616657Snate@binkert.org 'learning-gem5-p1-two-level', 1626657Snate@binkert.org) 1636657Snate@binkert.org 1646657Snate@binkert.orgall_categories = ("quick", "long") 1656657Snate@binkert.orgall_modes = ("fs", "se") 1666657Snate@binkert.org 1676657Snate@binkert.orgclass Test(object): 1686657Snate@binkert.org """Test case base class. 1696657Snate@binkert.org 1706657Snate@binkert.org Test cases consists of one or more test units that are run in two 1716657Snate@binkert.org phases. A run phase (units produced by run_units() and a verify 1726657Snate@binkert.org phase (units from verify_units()). The verify phase is skipped if 1736657Snate@binkert.org the run phase fails. 1746657Snate@binkert.org 1756657Snate@binkert.org """ 1766657Snate@binkert.org 1776657Snate@binkert.org __metaclass__ = ABCMeta 1786657Snate@binkert.org 1796657Snate@binkert.org def __init__(self, name): 1806657Snate@binkert.org self.test_name = name 1816657Snate@binkert.org 1829219Spower.jg@gmail.com @abstractmethod 1836877Ssteve.reinhardt@amd.com def ref_files(self): 1846657Snate@binkert.org """Get a list of reference files used by this test case""" 1859219Spower.jg@gmail.com pass 1866657Snate@binkert.org 1879219Spower.jg@gmail.com @abstractmethod 1886657Snate@binkert.org def run_units(self): 1896877Ssteve.reinhardt@amd.com """Units (typically RunGem5 instances) that describe the run phase of 1906999Snate@binkert.org this test. 1916877Ssteve.reinhardt@amd.com 19210308Snilay@cs.wisc.edu """ 1936877Ssteve.reinhardt@amd.com pass 1946877Ssteve.reinhardt@amd.com 19510308Snilay@cs.wisc.edu @abstractmethod 1966877Ssteve.reinhardt@amd.com def verify_units(self): 1976877Ssteve.reinhardt@amd.com """Verify the output from the run phase (see run_units()).""" 1986877Ssteve.reinhardt@amd.com pass 1996877Ssteve.reinhardt@amd.com 2006877Ssteve.reinhardt@amd.com @abstractmethod 2016877Ssteve.reinhardt@amd.com def update_ref(self): 2026877Ssteve.reinhardt@amd.com """Update reference files with files from a test run""" 2039338SAndreas.Sandberg@arm.com pass 2046877Ssteve.reinhardt@amd.com 2056877Ssteve.reinhardt@amd.com def run(self): 2066877Ssteve.reinhardt@amd.com """Run this test case and return a list of results""" 2076877Ssteve.reinhardt@amd.com 20810308Snilay@cs.wisc.edu run_results = [ u.run() for u in self.run_units() ] 20910308Snilay@cs.wisc.edu run_ok = all([not r.skipped() and r for r in run_results ]) 21010308Snilay@cs.wisc.edu 21110308Snilay@cs.wisc.edu verify_results = [ 21210311Snilay@cs.wisc.edu u.run() if run_ok else u.skip() 21310311Snilay@cs.wisc.edu for u in self.verify_units() 21410311Snilay@cs.wisc.edu ] 21510311Snilay@cs.wisc.edu 21610311Snilay@cs.wisc.edu return TestResult(self.test_name, 21710311Snilay@cs.wisc.edu run_results=run_results, 21810311Snilay@cs.wisc.edu verify_results=verify_results) 2196882SBrad.Beckmann@amd.com 22010308Snilay@cs.wisc.edu def __str__(self): 22110308Snilay@cs.wisc.edu return self.test_name 2226882SBrad.Beckmann@amd.com 2236882SBrad.Beckmann@amd.comclass ClassicTest(Test): 2246882SBrad.Beckmann@amd.com # The diff ignore list contains all files that shouldn't be diffed 2256882SBrad.Beckmann@amd.com # using DiffOutFile. These files typically use special-purpose 2266877Ssteve.reinhardt@amd.com # diff tools (e.g., DiffStatFile). 2276877Ssteve.reinhardt@amd.com diff_ignore_files = ( 22810917Sbrandon.potter@amd.com # Stat files use a special stat differ 2296877Ssteve.reinhardt@amd.com "stats.txt", 2306657Snate@binkert.org ) 2316657Snate@binkert.org 2326999Snate@binkert.org # These files should never be included in the list of 2336657Snate@binkert.org # reference files. This list should include temporary files 2346657Snate@binkert.org # and other files that we don't care about. 2356657Snate@binkert.org ref_ignore_files = ( 2366657Snate@binkert.org ) 2377007Snate@binkert.org 2386657Snate@binkert.org def __init__(self, gem5, output_dir, config_tuple, 2396657Snate@binkert.org timeout=None, 2406657Snate@binkert.org skip=False, skip_diff_out=False, skip_diff_stat=False): 2416657Snate@binkert.org 2426657Snate@binkert.org super(ClassicTest, self).__init__("/".join(config_tuple)) 2437007Snate@binkert.org 2447007Snate@binkert.org ct = config_tuple 2456657Snate@binkert.org 2467002Snate@binkert.org self.gem5 = os.path.abspath(gem5) 2477002Snate@binkert.org self.script = os.path.join(_test_base, "run.py") 2487002Snate@binkert.org self.config_tuple = ct 2497002Snate@binkert.org self.timeout = timeout 2506657Snate@binkert.org 2516657Snate@binkert.org self.output_dir = output_dir 2528229Snate@binkert.org self.ref_dir = os.path.join(_test_base, 2538229Snate@binkert.org ct.category, ct.mode, ct.workload, 2548229Snate@binkert.org "ref", ct.isa, ct.os, ct.config) 2558229Snate@binkert.org self.skip_run = skip 2566657Snate@binkert.org self.skip_diff_out = skip or skip_diff_out 2576657Snate@binkert.org self.skip_diff_stat = skip or skip_diff_stat 2586657Snate@binkert.org 2596657Snate@binkert.org def ref_files(self): 2606793SBrad.Beckmann@amd.com ref_dir = os.path.abspath(self.ref_dir) 2616657Snate@binkert.org for root, dirs, files in os.walk(ref_dir, topdown=False): 26210311Snilay@cs.wisc.edu for f in files: 2636657Snate@binkert.org fpath = os.path.join(root[len(ref_dir) + 1:], f) 2646657Snate@binkert.org if fpath not in ClassicTest.ref_ignore_files: 2656657Snate@binkert.org yield fpath 2667002Snate@binkert.org 2676657Snate@binkert.org def run_units(self): 2687007Snate@binkert.org args = [ 2697007Snate@binkert.org self.script, 2709271Snilay@cs.wisc.edu "/".join(self.config_tuple), 2716877Ssteve.reinhardt@amd.com ] 2726877Ssteve.reinhardt@amd.com 2736657Snate@binkert.org return [ 2746877Ssteve.reinhardt@amd.com RunGem5(self.gem5, args, 27510311Snilay@cs.wisc.edu ref_dir=self.ref_dir, test_dir=self.output_dir, 2766657Snate@binkert.org skip=self.skip_run), 27710311Snilay@cs.wisc.edu ] 2789745Snilay@cs.wisc.edu 2797002Snate@binkert.org def verify_units(self): 2806657Snate@binkert.org return [ 28110012Snilay@cs.wisc.edu DiffStatFile(ref_dir=self.ref_dir, test_dir=self.output_dir, 2829745Snilay@cs.wisc.edu skip=self.skip_diff_stat) 2839745Snilay@cs.wisc.edu ] + [ 2849745Snilay@cs.wisc.edu DiffOutFile(f, 2858683Snilay@cs.wisc.edu ref_dir=self.ref_dir, test_dir=self.output_dir, 2868683Snilay@cs.wisc.edu skip=self.skip_diff_out) 2877007Snate@binkert.org for f in self.ref_files() 28810524Snilay@cs.wisc.edu if f not in ClassicTest.diff_ignore_files 2899302Snilay@cs.wisc.edu ] 2909745Snilay@cs.wisc.edu 2919745Snilay@cs.wisc.edu def update_ref(self): 2929745Snilay@cs.wisc.edu for fname in self.ref_files(): 2939745Snilay@cs.wisc.edu shutil.copy( 2949745Snilay@cs.wisc.edu os.path.join(self.output_dir, fname), 2959745Snilay@cs.wisc.edu os.path.join(self.ref_dir, fname)) 2966657Snate@binkert.org 2976657Snate@binkert.orgdef parse_test_filter(test_filter): 2986657Snate@binkert.org wildcards = ("", "*") 2996657Snate@binkert.org 3006657Snate@binkert.org _filter = list(test_filter.split("/")) 3016657Snate@binkert.org if len(_filter) > 3: 3026882SBrad.Beckmann@amd.com raise RuntimeError("Illegal test filter string") 3036882SBrad.Beckmann@amd.com _filter += [ "", ] * (3 - len(_filter)) 3046882SBrad.Beckmann@amd.com 3056882SBrad.Beckmann@amd.com isa, cat, mode = _filter 3066657Snate@binkert.org 3076657Snate@binkert.org if isa in wildcards: 3087007Snate@binkert.org raise RuntimeError("No ISA specified") 3097839Snilay@cs.wisc.edu 3107839Snilay@cs.wisc.edu cat = all_categories if cat in wildcards else (cat, ) 3117839Snilay@cs.wisc.edu mode = all_modes if mode in wildcards else (mode, ) 3127839Snilay@cs.wisc.edu 3137839Snilay@cs.wisc.edu return isa, cat, mode 3147839Snilay@cs.wisc.edu 3157839Snilay@cs.wisc.edudef get_tests(isa, 3167839Snilay@cs.wisc.edu categories=all_categories, modes=all_modes, 3177839Snilay@cs.wisc.edu ruby_protocol=None, gpu_isa=None): 3187839Snilay@cs.wisc.edu 3197839Snilay@cs.wisc.edu # Generate a list of candidate configs 3207839Snilay@cs.wisc.edu configs = list(arch_configs.get((isa, gpu_isa), [])) 32110010Snilay@cs.wisc.edu 3227007Snate@binkert.org if (isa, gpu_isa) == ("x86", "hsail"): 3237007Snate@binkert.org if ruby_protocol == "GPU_RfO": 3247007Snate@binkert.org configs += ['gpu-randomtest'] 3257007Snate@binkert.org else: 3267839Snilay@cs.wisc.edu configs += generic_configs 3277839Snilay@cs.wisc.edu 3287839Snilay@cs.wisc.edu if ruby_protocol == 'MI_example': 3297839Snilay@cs.wisc.edu configs += [ "%s-ruby" % (c, ) for c in configs ] 3307839Snilay@cs.wisc.edu elif ruby_protocol is not None: 3317839Snilay@cs.wisc.edu # Override generic ISA configs when using Ruby (excluding 3327839Snilay@cs.wisc.edu # MI_example which is included in all ISAs by default). This 3337839Snilay@cs.wisc.edu # reduces the number of generic tests we re-run for when 3347839Snilay@cs.wisc.edu # compiling Ruby targets. 3357839Snilay@cs.wisc.edu configs = [ "%s-ruby-%s" % (c, ruby_protocol) for c in configs ] 3367839Snilay@cs.wisc.edu 3377839Snilay@cs.wisc.edu # /(quick|long)/(fs|se)/workload/ref/arch/guest/config/ 3387007Snate@binkert.org for conf_script in configs: 3397007Snate@binkert.org for cat in categories: 3409745Snilay@cs.wisc.edu for mode in modes: 3419745Snilay@cs.wisc.edu mode_dir = os.path.join(_test_base, cat, mode) 3429745Snilay@cs.wisc.edu if not os.path.exists(mode_dir): 3439745Snilay@cs.wisc.edu continue 3449745Snilay@cs.wisc.edu 3459745Snilay@cs.wisc.edu for workload in os.listdir(mode_dir): 3466657Snate@binkert.org isa_dir = os.path.join(mode_dir, workload, "ref", isa) 3477007Snate@binkert.org if not os.path.isdir(isa_dir): 3486657Snate@binkert.org continue 3496657Snate@binkert.org 3506657Snate@binkert.org for _os in os.listdir(isa_dir): 3516657Snate@binkert.org test_dir = os.path.join(isa_dir, _os, conf_script) 3526657Snate@binkert.org if not os.path.exists(test_dir) or \ 3536657Snate@binkert.org os.path.exists(os.path.join(test_dir, "skip")): 3546657Snate@binkert.org continue 3556657Snate@binkert.org 3567839Snilay@cs.wisc.edu yield ClassicConfig(cat, mode, workload, isa, _os, 3577839Snilay@cs.wisc.edu conf_script) 3587839Snilay@cs.wisc.edu