tests.py revision 11636:2cdb85a2e980
15086Sgblack@eecs.umich.edu#!/usr/bin/env python 25086Sgblack@eecs.umich.edu# 35086Sgblack@eecs.umich.edu# Copyright (c) 2016 ARM Limited 45086Sgblack@eecs.umich.edu# All rights reserved 55086Sgblack@eecs.umich.edu# 65086Sgblack@eecs.umich.edu# The license below extends only to copyright in the software and shall 75086Sgblack@eecs.umich.edu# not be construed as granting a license to any other intellectual 85086Sgblack@eecs.umich.edu# property including but not limited to intellectual property relating 95086Sgblack@eecs.umich.edu# to a hardware implementation of the functionality of the software 105086Sgblack@eecs.umich.edu# licensed hereunder. You may use the software subject to the license 115086Sgblack@eecs.umich.edu# terms below provided that you ensure that this notice is replicated 125086Sgblack@eecs.umich.edu# unmodified and in its entirety in all distributions of the software, 135086Sgblack@eecs.umich.edu# modified or unmodified, in source code or in binary form. 145086Sgblack@eecs.umich.edu# 155086Sgblack@eecs.umich.edu# Redistribution and use in source and binary forms, with or without 165086Sgblack@eecs.umich.edu# modification, are permitted provided that the following conditions are 175086Sgblack@eecs.umich.edu# met: redistributions of source code must retain the above copyright 185086Sgblack@eecs.umich.edu# notice, this list of conditions and the following disclaimer; 195086Sgblack@eecs.umich.edu# redistributions in binary form must reproduce the above copyright 205086Sgblack@eecs.umich.edu# notice, this list of conditions and the following disclaimer in the 215086Sgblack@eecs.umich.edu# documentation and/or other materials provided with the distribution; 225086Sgblack@eecs.umich.edu# neither the name of the copyright holders nor the names of its 235086Sgblack@eecs.umich.edu# contributors may be used to endorse or promote products derived from 245086Sgblack@eecs.umich.edu# this software without specific prior written permission. 255086Sgblack@eecs.umich.edu# 265086Sgblack@eecs.umich.edu# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS 275086Sgblack@eecs.umich.edu# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT 285086Sgblack@eecs.umich.edu# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR 295086Sgblack@eecs.umich.edu# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT 305086Sgblack@eecs.umich.edu# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, 315086Sgblack@eecs.umich.edu# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT 325086Sgblack@eecs.umich.edu# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, 335086Sgblack@eecs.umich.edu# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY 345086Sgblack@eecs.umich.edu# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT 355086Sgblack@eecs.umich.edu# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE 365086Sgblack@eecs.umich.edu# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 375086Sgblack@eecs.umich.edu# 385086Sgblack@eecs.umich.edu# Authors: Andreas Sandberg 395086Sgblack@eecs.umich.edu 405086Sgblack@eecs.umich.edufrom abc import ABCMeta, abstractmethod 415086Sgblack@eecs.umich.eduimport os 425086Sgblack@eecs.umich.edufrom collections import namedtuple 435086Sgblack@eecs.umich.edufrom units import * 445086Sgblack@eecs.umich.edufrom results import TestResult 455086Sgblack@eecs.umich.eduimport shutil 465086Sgblack@eecs.umich.edu 475086Sgblack@eecs.umich.edu_test_base = os.path.join(os.path.dirname(__file__), "..") 485086Sgblack@eecs.umich.edu 495086Sgblack@eecs.umich.eduClassicConfig = namedtuple("ClassicConfig", ( 505086Sgblack@eecs.umich.edu "category", 515086Sgblack@eecs.umich.edu "mode", 525086Sgblack@eecs.umich.edu "workload", 535086Sgblack@eecs.umich.edu "isa", 545086Sgblack@eecs.umich.edu "os", 555086Sgblack@eecs.umich.edu "config", 565086Sgblack@eecs.umich.edu)) 575086Sgblack@eecs.umich.edu 585647Sgblack@eecs.umich.edu# There are currently two "classes" of test 595647Sgblack@eecs.umich.edu# configurations. Architecture-specific ones and generic ones 605647Sgblack@eecs.umich.edu# (typically SE mode tests). In both cases, the configuration name 615647Sgblack@eecs.umich.edu# matches a file in tests/configs/ that will be picked up by the test 625647Sgblack@eecs.umich.edu# runner (run.py). 635135Sgblack@eecs.umich.edu# 645135Sgblack@eecs.umich.edu# Architecture specific configurations are listed in the arch_configs 655135Sgblack@eecs.umich.edu# dictionary. This is indexed by a (cpu architecture, gpu 665086Sgblack@eecs.umich.edu# architecture) tuple. GPU architecture is optional and may be None. 675135Sgblack@eecs.umich.edu# 685647Sgblack@eecs.umich.edu# Generic configurations are listed in the generic_configs tuple. 695234Sgblack@eecs.umich.edu# 705086Sgblack@eecs.umich.edu# When discovering available test cases, this script look uses the 715086Sgblack@eecs.umich.edu# test list as a list of /candidate/ configurations. A configuration 725086Sgblack@eecs.umich.edu# is only used if a test has a reference output for that 735086Sgblack@eecs.umich.edu# configuration. In addition to the base configurations from 745086Sgblack@eecs.umich.edu# arch_configs and generic_configs, a Ruby configuration may be 755086Sgblack@eecs.umich.edu# appended to the base name (this is probed /in addition/ to the 765086Sgblack@eecs.umich.edu# original name. See get_tests() for details. 775086Sgblack@eecs.umich.edu# 785086Sgblack@eecs.umich.eduarch_configs = { 795086Sgblack@eecs.umich.edu ("alpha", None) : ( 805086Sgblack@eecs.umich.edu 'tsunami-simple-atomic', 815135Sgblack@eecs.umich.edu 'tsunami-simple-timing', 825135Sgblack@eecs.umich.edu 'tsunami-simple-atomic-dual', 835135Sgblack@eecs.umich.edu 'tsunami-simple-timing-dual', 845135Sgblack@eecs.umich.edu 'twosys-tsunami-simple-atomic', 856048Sgblack@eecs.umich.edu 'tsunami-o3', 'tsunami-o3-dual', 866048Sgblack@eecs.umich.edu 'tsunami-minor', 'tsunami-minor-dual', 876048Sgblack@eecs.umich.edu 'tsunami-switcheroo-full', 886048Sgblack@eecs.umich.edu ), 896048Sgblack@eecs.umich.edu 906048Sgblack@eecs.umich.edu ("arm", None) : ( 916048Sgblack@eecs.umich.edu 'simple-atomic-dummychecker', 926048Sgblack@eecs.umich.edu 'o3-timing-checker', 935135Sgblack@eecs.umich.edu 'realview-simple-atomic', 945135Sgblack@eecs.umich.edu 'realview-simple-atomic-dual', 955135Sgblack@eecs.umich.edu 'realview-simple-atomic-checkpoint', 965135Sgblack@eecs.umich.edu 'realview-simple-timing', 975135Sgblack@eecs.umich.edu 'realview-simple-timing-dual', 985135Sgblack@eecs.umich.edu 'realview-o3', 995135Sgblack@eecs.umich.edu 'realview-o3-checker', 1005135Sgblack@eecs.umich.edu 'realview-o3-dual', 1015135Sgblack@eecs.umich.edu 'realview-minor', 1025135Sgblack@eecs.umich.edu 'realview-minor-dual', 1035135Sgblack@eecs.umich.edu 'realview-switcheroo-atomic', 1045135Sgblack@eecs.umich.edu 'realview-switcheroo-timing', 1055135Sgblack@eecs.umich.edu 'realview-switcheroo-o3', 1065135Sgblack@eecs.umich.edu 'realview-switcheroo-full', 1075135Sgblack@eecs.umich.edu 'realview64-simple-atomic', 1085135Sgblack@eecs.umich.edu 'realview64-simple-atomic-checkpoint', 1095135Sgblack@eecs.umich.edu 'realview64-simple-atomic-dual', 1105264Sgblack@eecs.umich.edu 'realview64-simple-timing', 1115135Sgblack@eecs.umich.edu 'realview64-simple-timing-dual', 1125135Sgblack@eecs.umich.edu 'realview64-o3', 1135135Sgblack@eecs.umich.edu 'realview64-o3-checker', 1145135Sgblack@eecs.umich.edu 'realview64-o3-dual', 1155141Sgblack@eecs.umich.edu 'realview64-minor', 1165141Sgblack@eecs.umich.edu 'realview64-minor-dual', 1175141Sgblack@eecs.umich.edu 'realview64-switcheroo-atomic', 1185141Sgblack@eecs.umich.edu 'realview64-switcheroo-timing', 1195141Sgblack@eecs.umich.edu 'realview64-switcheroo-o3', 1205141Sgblack@eecs.umich.edu 'realview64-switcheroo-full', 1215141Sgblack@eecs.umich.edu ), 1225141Sgblack@eecs.umich.edu 1235141Sgblack@eecs.umich.edu ("sparc", None) : ( 1245182Sgblack@eecs.umich.edu 't1000-simple-atomic', 1255141Sgblack@eecs.umich.edu 't1000-simple-x86', 1265141Sgblack@eecs.umich.edu ), 1275141Sgblack@eecs.umich.edu 1285141Sgblack@eecs.umich.edu ("timing", None) : ( 1295141Sgblack@eecs.umich.edu 'pc-simple-atomic', 1305141Sgblack@eecs.umich.edu 'pc-simple-timing', 1315135Sgblack@eecs.umich.edu 'pc-o3-timing', 1325141Sgblack@eecs.umich.edu 'pc-switcheroo-full', 1335141Sgblack@eecs.umich.edu ), 1345141Sgblack@eecs.umich.edu 1355141Sgblack@eecs.umich.edu ("x86", "hsail") : ( 1365141Sgblack@eecs.umich.edu 'gpu', 1375141Sgblack@eecs.umich.edu ), 1385141Sgblack@eecs.umich.edu} 1395141Sgblack@eecs.umich.edu 1405141Sgblack@eecs.umich.edugeneric_configs = ( 1415141Sgblack@eecs.umich.edu 'simple-atomic', 1425141Sgblack@eecs.umich.edu 'simple-atomic-mp', 1435141Sgblack@eecs.umich.edu 'simple-timing', 1445135Sgblack@eecs.umich.edu 'simple-timing-mp', 1455141Sgblack@eecs.umich.edu 1465141Sgblack@eecs.umich.edu 'minor-timing', 1475135Sgblack@eecs.umich.edu 'minor-timing-mp', 1485141Sgblack@eecs.umich.edu 1495141Sgblack@eecs.umich.edu 'o3-timing', 1505141Sgblack@eecs.umich.edu 'o3-timing-mt', 1515141Sgblack@eecs.umich.edu 'o3-timing-mp', 1525135Sgblack@eecs.umich.edu 1535141Sgblack@eecs.umich.edu 'rubytest', 1545141Sgblack@eecs.umich.edu 'memcheck', 1555141Sgblack@eecs.umich.edu 'memtest', 1565141Sgblack@eecs.umich.edu 'memtest-filter', 1575141Sgblack@eecs.umich.edu 'tgen-simple-mem', 1585141Sgblack@eecs.umich.edu 'tgen-dram-ctrl', 1595141Sgblack@eecs.umich.edu 1605141Sgblack@eecs.umich.edu 'learning-gem5-p1-simple', 1615141Sgblack@eecs.umich.edu 'learning-gem5-p1-two-level', 1625141Sgblack@eecs.umich.edu) 1635141Sgblack@eecs.umich.edu 1645141Sgblack@eecs.umich.eduall_categories = ("quick", "long") 1655264Sgblack@eecs.umich.eduall_modes = ("fs", "se") 1665141Sgblack@eecs.umich.edu 1675141Sgblack@eecs.umich.educlass Test(object): 1685141Sgblack@eecs.umich.edu """Test case base class. 1695141Sgblack@eecs.umich.edu 1705141Sgblack@eecs.umich.edu Test cases consists of one or more test units that are run in two 1715141Sgblack@eecs.umich.edu phases. A run phase (units produced by run_units() and a verify 1725141Sgblack@eecs.umich.edu phase (units from verify_units()). The verify phase is skipped if 1735141Sgblack@eecs.umich.edu the run phase fails. 1745141Sgblack@eecs.umich.edu 1755141Sgblack@eecs.umich.edu """ 1765141Sgblack@eecs.umich.edu 1775141Sgblack@eecs.umich.edu __metaclass__ = ABCMeta 1785141Sgblack@eecs.umich.edu 1795141Sgblack@eecs.umich.edu def __init__(self, name): 1805141Sgblack@eecs.umich.edu self.test_name = name 1815141Sgblack@eecs.umich.edu 1825141Sgblack@eecs.umich.edu @abstractmethod 1835135Sgblack@eecs.umich.edu def ref_files(self): 1845135Sgblack@eecs.umich.edu """Get a list of reference files used by this test case""" 1855135Sgblack@eecs.umich.edu pass 1865360Sgblack@eecs.umich.edu 1875360Sgblack@eecs.umich.edu @abstractmethod 1885360Sgblack@eecs.umich.edu def run_units(self): 1895360Sgblack@eecs.umich.edu """Units (typically RunGem5 instances) that describe the run phase of 1905360Sgblack@eecs.umich.edu this test. 1915360Sgblack@eecs.umich.edu 1925647Sgblack@eecs.umich.edu """ 1935647Sgblack@eecs.umich.edu pass 1945647Sgblack@eecs.umich.edu 1955360Sgblack@eecs.umich.edu @abstractmethod 1965647Sgblack@eecs.umich.edu def verify_units(self): 1975647Sgblack@eecs.umich.edu """Verify the output from the run phase (see run_units()).""" 1985647Sgblack@eecs.umich.edu pass 1995648Sgblack@eecs.umich.edu 2005648Sgblack@eecs.umich.edu @abstractmethod 2015360Sgblack@eecs.umich.edu def update_ref(self): 2025141Sgblack@eecs.umich.edu """Update reference files with files from a test run""" 2035141Sgblack@eecs.umich.edu pass 2045141Sgblack@eecs.umich.edu 2055141Sgblack@eecs.umich.edu def run(self): 2065141Sgblack@eecs.umich.edu """Run this test case and return a list of results""" 2075141Sgblack@eecs.umich.edu 2085135Sgblack@eecs.umich.edu run_results = [ u.run() for u in self.run_units() ] 2095135Sgblack@eecs.umich.edu run_ok = all([not r.skipped() and r for r in run_results ]) 2105135Sgblack@eecs.umich.edu 2115135Sgblack@eecs.umich.edu verify_results = [ 2125135Sgblack@eecs.umich.edu u.run() if run_ok else u.skip() 2135135Sgblack@eecs.umich.edu for u in self.verify_units() 2146042Sgblack@eecs.umich.edu ] 2155135Sgblack@eecs.umich.edu 2165135Sgblack@eecs.umich.edu return TestResult(self.test_name, 2175135Sgblack@eecs.umich.edu run_results=run_results, 2185135Sgblack@eecs.umich.edu verify_results=verify_results) 2195135Sgblack@eecs.umich.edu 2205135Sgblack@eecs.umich.edu def __str__(self): 2216042Sgblack@eecs.umich.edu return self.test_name 2225135Sgblack@eecs.umich.edu 2236042Sgblack@eecs.umich.educlass ClassicTest(Test): 2246042Sgblack@eecs.umich.edu # The diff ignore list contains all files that shouldn't be diffed 2256042Sgblack@eecs.umich.edu # using DiffOutFile. These files typically use special-purpose 2265135Sgblack@eecs.umich.edu # diff tools (e.g., DiffStatFile). 2275135Sgblack@eecs.umich.edu diff_ignore_files = FileIgnoreList( 2285086Sgblack@eecs.umich.edu names=( 229 # Stat files use a special stat differ 230 "stats.txt", 231 ), rex=( 232 )) 233 234 # These files should never be included in the list of 235 # reference files. This list should include temporary files 236 # and other files that we don't care about. 237 ref_ignore_files = FileIgnoreList( 238 names=( 239 "EMPTY", 240 ), rex=( 241 # Mercurial sometimes leaves backups when applying MQ patches 242 r"\.orig$", 243 r"\.rej$", 244 )) 245 246 def __init__(self, gem5, output_dir, config_tuple, 247 timeout=None, 248 skip=False, skip_diff_out=False, skip_diff_stat=False): 249 250 super(ClassicTest, self).__init__("/".join(config_tuple)) 251 252 ct = config_tuple 253 254 self.gem5 = os.path.abspath(gem5) 255 self.script = os.path.join(_test_base, "run.py") 256 self.config_tuple = ct 257 self.timeout = timeout 258 259 self.output_dir = output_dir 260 self.ref_dir = os.path.join(_test_base, 261 ct.category, ct.mode, ct.workload, 262 "ref", ct.isa, ct.os, ct.config) 263 self.skip_run = skip 264 self.skip_diff_out = skip or skip_diff_out 265 self.skip_diff_stat = skip or skip_diff_stat 266 267 def ref_files(self): 268 ref_dir = os.path.abspath(self.ref_dir) 269 for root, dirs, files in os.walk(ref_dir, topdown=False): 270 for f in files: 271 fpath = os.path.join(root[len(ref_dir) + 1:], f) 272 if fpath not in ClassicTest.ref_ignore_files: 273 yield fpath 274 275 def run_units(self): 276 args = [ 277 self.script, 278 "/".join(self.config_tuple), 279 ] 280 281 return [ 282 RunGem5(self.gem5, args, 283 ref_dir=self.ref_dir, test_dir=self.output_dir, 284 skip=self.skip_run), 285 ] 286 287 def verify_units(self): 288 ref_files = set(self.ref_files()) 289 units = [] 290 if "stats.txt" in ref_files: 291 units.append( 292 DiffStatFile(ref_dir=self.ref_dir, test_dir=self.output_dir, 293 skip=self.skip_diff_stat)) 294 units += [ 295 DiffOutFile(f, 296 ref_dir=self.ref_dir, test_dir=self.output_dir, 297 skip=self.skip_diff_out) 298 for f in ref_files if f not in ClassicTest.diff_ignore_files 299 ] 300 301 return units 302 303 def update_ref(self): 304 for fname in self.ref_files(): 305 shutil.copy( 306 os.path.join(self.output_dir, fname), 307 os.path.join(self.ref_dir, fname)) 308 309def parse_test_filter(test_filter): 310 wildcards = ("", "*") 311 312 _filter = list(test_filter.split("/")) 313 if len(_filter) > 3: 314 raise RuntimeError("Illegal test filter string") 315 _filter += [ "", ] * (3 - len(_filter)) 316 317 isa, cat, mode = _filter 318 319 if isa in wildcards: 320 raise RuntimeError("No ISA specified") 321 322 cat = all_categories if cat in wildcards else (cat, ) 323 mode = all_modes if mode in wildcards else (mode, ) 324 325 return isa, cat, mode 326 327def get_tests(isa, 328 categories=all_categories, modes=all_modes, 329 ruby_protocol=None, gpu_isa=None): 330 331 # Generate a list of candidate configs 332 configs = list(arch_configs.get((isa, gpu_isa), [])) 333 334 if (isa, gpu_isa) == ("x86", "hsail"): 335 if ruby_protocol == "GPU_RfO": 336 configs += ['gpu-randomtest'] 337 else: 338 configs += generic_configs 339 340 if ruby_protocol == 'MI_example': 341 configs += [ "%s-ruby" % (c, ) for c in configs ] 342 elif ruby_protocol is not None: 343 # Override generic ISA configs when using Ruby (excluding 344 # MI_example which is included in all ISAs by default). This 345 # reduces the number of generic tests we re-run for when 346 # compiling Ruby targets. 347 configs = [ "%s-ruby-%s" % (c, ruby_protocol) for c in configs ] 348 349 # /(quick|long)/(fs|se)/workload/ref/arch/guest/config/ 350 for conf_script in configs: 351 for cat in categories: 352 for mode in modes: 353 mode_dir = os.path.join(_test_base, cat, mode) 354 if not os.path.exists(mode_dir): 355 continue 356 357 for workload in os.listdir(mode_dir): 358 isa_dir = os.path.join(mode_dir, workload, "ref", isa) 359 if not os.path.isdir(isa_dir): 360 continue 361 362 for _os in os.listdir(isa_dir): 363 test_dir = os.path.join(isa_dir, _os, conf_script) 364 if not os.path.exists(test_dir) or \ 365 os.path.exists(os.path.join(test_dir, "skip")): 366 continue 367 368 yield ClassicConfig(cat, mode, workload, isa, _os, 369 conf_script) 370