1# Copyright (c) 2017 Mark D. Hill and David A. Wood 2# All rights reserved. 3# 4# Redistribution and use in source and binary forms, with or without 5# modification, are permitted provided that the following conditions are 6# met: redistributions of source code must retain the above copyright 7# notice, this list of conditions and the following disclaimer; 8# redistributions in binary form must reproduce the above copyright 9# notice, this list of conditions and the following disclaimer in the 10# documentation and/or other materials provided with the distribution; 11# neither the name of the copyright holders nor the names of its 12# contributors may be used to endorse or promote products derived from 13# this software without specific prior written permission. 14# 15# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS 16# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT 17# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR 18# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT 19# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, 20# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT 21# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, 22# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY 23# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT 24# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE 25# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 26# 27# Authors: Sean Wilson 28 29''' 30Built in test cases that verify particular details about a gem5 run. 31''' 32import re 33 34from testlib import test 35from testlib.config import constants 36from testlib.helper import joinpath, diff_out_file 37 38class Verifier(object): 39 def __init__(self, fixtures=tuple()): 40 self.fixtures = fixtures 41 42 def _test(self, *args, **kwargs): 43 # Use a callback wrapper to make stack 44 # traces easier to understand. 45 self.test(*args, **kwargs) 46 47 def instantiate_test(self, name_pfx): 48 name = '-'.join([name_pfx, self.__class__.__name__]) 49 return test.TestFunction(self._test, 50 name=name, fixtures=self.fixtures) 51 52 def failed(self, fixtures): 53 ''' 54 Called if this verifier fails to cleanup (or not) as needed. 55 ''' 56 try: 57 fixtures[constants.tempdir_fixture_name].skip_cleanup() 58 except KeyError: 59 pass # No need to do anything if the tempdir fixture doesn't exist 60 61 62class MatchGoldStandard(Verifier): 63 ''' 64 Compares a standard output to the test output and passes if they match, 65 fails if they do not. 66 ''' 67 def __init__(self, standard_filename, ignore_regex=None, 68 test_filename='simout'): 69 ''' 70 :param standard_filename: The path of the standard file to compare 71 output to. 72 73 :param ignore_regex: A string, compiled regex, or iterable containing 74 either which will be ignored in 'standard' and test output files when 75 diffing. 76 ''' 77 super(MatchGoldStandard, self).__init__() 78 self.standard_filename = standard_filename 79 self.test_filename = test_filename 80 81 self.ignore_regex = _iterable_regex(ignore_regex) 82 83 def test(self, params): 84 # We need a tempdir fixture from our parent verifier suite. 85 fixtures = params.fixtures 86 # Get the file from the tempdir of the test. 87 tempdir = fixtures[constants.tempdir_fixture_name].path 88 self.test_filename = joinpath(tempdir, self.test_filename) 89 90 diff = diff_out_file(self.standard_filename, 91 self.test_filename, 92 ignore_regexes=self.ignore_regex, 93 logger=params.log) 94 if diff is not None: 95 self.failed(fixtures) 96 test.fail('Stdout did not match:\n%s\nSee %s for full results' 97 % (diff, tempdir)) 98 99 def _generic_instance_warning(self, kwargs): 100 ''' 101 Method for helper classes to tell users to use this more generic class 102 if they are going to manually override the test_filename param. 103 ''' 104 if 'test_filename' in kwargs: 105 raise ValueError('If you are setting test_filename use the more' 106 ' generic %s' 107 ' instead' % MatchGoldStandard.__name__) 108 109class DerivedGoldStandard(MatchGoldStandard): 110 __ignore_regex_sentinel = object() 111 _file = None 112 _default_ignore_regex = [] 113 114 def __init__(self, standard_filename, 115 ignore_regex=__ignore_regex_sentinel, **kwargs): 116 117 if ignore_regex == self.__ignore_regex_sentinel: 118 ignore_regex = self._default_ignore_regex 119 120 self._generic_instance_warning(kwargs) 121 122 super(DerivedGoldStandard, self).__init__( 123 standard_filename, 124 test_filename=self._file, 125 ignore_regex=ignore_regex, 126 **kwargs) 127 128class MatchStdout(DerivedGoldStandard): 129 _file = constants.gem5_simulation_stdout 130 _default_ignore_regex = [ 131 re.compile('^Redirecting (stdout|stderr) to'), 132 re.compile('^gem5 compiled '), 133 re.compile('^gem5 started '), 134 re.compile('^gem5 executing on '), 135 re.compile('^command line:'), 136 re.compile("^Couldn't import dot_parser,"), 137 re.compile("^info: kernel located at:"), 138 re.compile("^info: Standard input is not a terminal"), 139 re.compile("^Couldn't unlink "), 140 re.compile("^Using GPU kernel code file\(s\) "), 141 ] 142 143class MatchStdoutNoPerf(MatchStdout): 144 _file = constants.gem5_simulation_stdout 145 _default_ignore_regex = MatchStdout._default_ignore_regex + [ 146 re.compile('^Exiting @ tick'), 147 ] 148 149class MatchStderr(DerivedGoldStandard): 150 _file = constants.gem5_simulation_stderr 151 _default_ignore_regex = [] 152 153class MatchStats(DerivedGoldStandard): 154 # TODO: Likely will want to change this verifier since we have the weird 155 # perl script right now. A simple diff probably isn't going to work. 156 _file = constants.gem5_simulation_stats 157 _default_ignore_regex = [] 158 159class MatchConfigINI(DerivedGoldStandard): 160 _file = constants.gem5_simulation_config_ini 161 _default_ignore_regex = ( 162 re.compile("^(executable|readfile|kernel|image_file)="), 163 re.compile("^(cwd|input|codefile)="), 164 ) 165 166class MatchConfigJSON(DerivedGoldStandard): 167 _file = constants.gem5_simulation_config_json 168 _default_ignore_regex = ( 169 re.compile(r'''^\s*"(executable|readfile|kernel|image_file)":'''), 170 re.compile(r'''^\s*"(cwd|input|codefile)":'''), 171 ) 172 173class MatchRegex(Verifier): 174 def __init__(self, regex, match_stderr=True, match_stdout=True): 175 super(MatchRegex, self).__init__() 176 self.regex = _iterable_regex(regex) 177 self.match_stderr = match_stderr 178 self.match_stdout = match_stdout 179 180 def test(self, params): 181 fixtures = params.fixtures 182 # Get the file from the tempdir of the test. 183 tempdir = fixtures[constants.tempdir_fixture_name].path 184 185 def parse_file(fname): 186 with open(fname, 'r') as file_: 187 for line in file_: 188 for regex in self.regex: 189 if re.match(regex, line): 190 return True 191 if self.match_stdout: 192 if parse_file(joinpath(tempdir, 193 constants.gem5_simulation_stdout)): 194 return # Success 195 if self.match_stderr: 196 if parse_file(joinpath(tempdir, 197 constants.gem5_simulation_stderr)): 198 return # Success 199 self.failed(fixtures) 200 test.fail('Could not match regex.') 201 202_re_type = type(re.compile('')) 203def _iterable_regex(regex): 204 if isinstance(regex, _re_type) or isinstance(regex, str): 205 regex = (regex,) 206 return regex 207