112882Sspwilson2@wisc.edu# Copyright (c) 2017 Mark D. Hill and David A. Wood 212882Sspwilson2@wisc.edu# All rights reserved. 312882Sspwilson2@wisc.edu# 412882Sspwilson2@wisc.edu# Redistribution and use in source and binary forms, with or without 512882Sspwilson2@wisc.edu# modification, are permitted provided that the following conditions are 612882Sspwilson2@wisc.edu# met: redistributions of source code must retain the above copyright 712882Sspwilson2@wisc.edu# notice, this list of conditions and the following disclaimer; 812882Sspwilson2@wisc.edu# redistributions in binary form must reproduce the above copyright 912882Sspwilson2@wisc.edu# notice, this list of conditions and the following disclaimer in the 1012882Sspwilson2@wisc.edu# documentation and/or other materials provided with the distribution; 1112882Sspwilson2@wisc.edu# neither the name of the copyright holders nor the names of its 1212882Sspwilson2@wisc.edu# contributors may be used to endorse or promote products derived from 1312882Sspwilson2@wisc.edu# this software without specific prior written permission. 1412882Sspwilson2@wisc.edu# 1512882Sspwilson2@wisc.edu# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS 1612882Sspwilson2@wisc.edu# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT 1712882Sspwilson2@wisc.edu# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR 1812882Sspwilson2@wisc.edu# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT 1912882Sspwilson2@wisc.edu# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, 2012882Sspwilson2@wisc.edu# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT 2112882Sspwilson2@wisc.edu# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, 2212882Sspwilson2@wisc.edu# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY 2312882Sspwilson2@wisc.edu# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT 2412882Sspwilson2@wisc.edu# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE 2512882Sspwilson2@wisc.edu# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 2612882Sspwilson2@wisc.edu# 2712882Sspwilson2@wisc.edu# Authors: Sean Wilson 2812882Sspwilson2@wisc.edu 2912882Sspwilson2@wisc.edu''' 3012882Sspwilson2@wisc.eduBuilt in test cases that verify particular details about a gem5 run. 3112882Sspwilson2@wisc.edu''' 3212882Sspwilson2@wisc.eduimport re 3312882Sspwilson2@wisc.edu 3412882Sspwilson2@wisc.edufrom testlib import test 3512882Sspwilson2@wisc.edufrom testlib.config import constants 3612882Sspwilson2@wisc.edufrom testlib.helper import joinpath, diff_out_file 3712882Sspwilson2@wisc.edu 3812882Sspwilson2@wisc.educlass Verifier(object): 3912882Sspwilson2@wisc.edu def __init__(self, fixtures=tuple()): 4012882Sspwilson2@wisc.edu self.fixtures = fixtures 4112882Sspwilson2@wisc.edu 4212882Sspwilson2@wisc.edu def _test(self, *args, **kwargs): 4312882Sspwilson2@wisc.edu # Use a callback wrapper to make stack 4412882Sspwilson2@wisc.edu # traces easier to understand. 4512882Sspwilson2@wisc.edu self.test(*args, **kwargs) 4612882Sspwilson2@wisc.edu 4712882Sspwilson2@wisc.edu def instantiate_test(self, name_pfx): 4812882Sspwilson2@wisc.edu name = '-'.join([name_pfx, self.__class__.__name__]) 4912882Sspwilson2@wisc.edu return test.TestFunction(self._test, 5012882Sspwilson2@wisc.edu name=name, fixtures=self.fixtures) 5112882Sspwilson2@wisc.edu 5212882Sspwilson2@wisc.edu def failed(self, fixtures): 5312882Sspwilson2@wisc.edu ''' 5412882Sspwilson2@wisc.edu Called if this verifier fails to cleanup (or not) as needed. 5512882Sspwilson2@wisc.edu ''' 5612882Sspwilson2@wisc.edu try: 5712882Sspwilson2@wisc.edu fixtures[constants.tempdir_fixture_name].skip_cleanup() 5812882Sspwilson2@wisc.edu except KeyError: 5912882Sspwilson2@wisc.edu pass # No need to do anything if the tempdir fixture doesn't exist 6012882Sspwilson2@wisc.edu 6112882Sspwilson2@wisc.edu 6212882Sspwilson2@wisc.educlass MatchGoldStandard(Verifier): 6312882Sspwilson2@wisc.edu ''' 6412882Sspwilson2@wisc.edu Compares a standard output to the test output and passes if they match, 6512882Sspwilson2@wisc.edu fails if they do not. 6612882Sspwilson2@wisc.edu ''' 6712882Sspwilson2@wisc.edu def __init__(self, standard_filename, ignore_regex=None, 6812882Sspwilson2@wisc.edu test_filename='simout'): 6912882Sspwilson2@wisc.edu ''' 7012882Sspwilson2@wisc.edu :param standard_filename: The path of the standard file to compare 7112882Sspwilson2@wisc.edu output to. 7212882Sspwilson2@wisc.edu 7312882Sspwilson2@wisc.edu :param ignore_regex: A string, compiled regex, or iterable containing 7412882Sspwilson2@wisc.edu either which will be ignored in 'standard' and test output files when 7512882Sspwilson2@wisc.edu diffing. 7612882Sspwilson2@wisc.edu ''' 7712882Sspwilson2@wisc.edu super(MatchGoldStandard, self).__init__() 7812882Sspwilson2@wisc.edu self.standard_filename = standard_filename 7912882Sspwilson2@wisc.edu self.test_filename = test_filename 8012882Sspwilson2@wisc.edu 8112882Sspwilson2@wisc.edu self.ignore_regex = _iterable_regex(ignore_regex) 8212882Sspwilson2@wisc.edu 8312882Sspwilson2@wisc.edu def test(self, params): 8412882Sspwilson2@wisc.edu # We need a tempdir fixture from our parent verifier suite. 8512882Sspwilson2@wisc.edu fixtures = params.fixtures 8612882Sspwilson2@wisc.edu # Get the file from the tempdir of the test. 8712882Sspwilson2@wisc.edu tempdir = fixtures[constants.tempdir_fixture_name].path 8812882Sspwilson2@wisc.edu self.test_filename = joinpath(tempdir, self.test_filename) 8912882Sspwilson2@wisc.edu 9012882Sspwilson2@wisc.edu diff = diff_out_file(self.standard_filename, 9112882Sspwilson2@wisc.edu self.test_filename, 9212882Sspwilson2@wisc.edu ignore_regexes=self.ignore_regex, 9312882Sspwilson2@wisc.edu logger=params.log) 9412882Sspwilson2@wisc.edu if diff is not None: 9512882Sspwilson2@wisc.edu self.failed(fixtures) 9612882Sspwilson2@wisc.edu test.fail('Stdout did not match:\n%s\nSee %s for full results' 9712882Sspwilson2@wisc.edu % (diff, tempdir)) 9812882Sspwilson2@wisc.edu 9912882Sspwilson2@wisc.edu def _generic_instance_warning(self, kwargs): 10012882Sspwilson2@wisc.edu ''' 10112882Sspwilson2@wisc.edu Method for helper classes to tell users to use this more generic class 10212882Sspwilson2@wisc.edu if they are going to manually override the test_filename param. 10312882Sspwilson2@wisc.edu ''' 10412882Sspwilson2@wisc.edu if 'test_filename' in kwargs: 10512882Sspwilson2@wisc.edu raise ValueError('If you are setting test_filename use the more' 10612882Sspwilson2@wisc.edu ' generic %s' 10712882Sspwilson2@wisc.edu ' instead' % MatchGoldStandard.__name__) 10812882Sspwilson2@wisc.edu 10912882Sspwilson2@wisc.educlass DerivedGoldStandard(MatchGoldStandard): 11012882Sspwilson2@wisc.edu __ignore_regex_sentinel = object() 11112882Sspwilson2@wisc.edu _file = None 11212882Sspwilson2@wisc.edu _default_ignore_regex = [] 11312882Sspwilson2@wisc.edu 11412882Sspwilson2@wisc.edu def __init__(self, standard_filename, 11512882Sspwilson2@wisc.edu ignore_regex=__ignore_regex_sentinel, **kwargs): 11612882Sspwilson2@wisc.edu 11712882Sspwilson2@wisc.edu if ignore_regex == self.__ignore_regex_sentinel: 11812882Sspwilson2@wisc.edu ignore_regex = self._default_ignore_regex 11912882Sspwilson2@wisc.edu 12012882Sspwilson2@wisc.edu self._generic_instance_warning(kwargs) 12112882Sspwilson2@wisc.edu 12212882Sspwilson2@wisc.edu super(DerivedGoldStandard, self).__init__( 12312882Sspwilson2@wisc.edu standard_filename, 12412882Sspwilson2@wisc.edu test_filename=self._file, 12512882Sspwilson2@wisc.edu ignore_regex=ignore_regex, 12612882Sspwilson2@wisc.edu **kwargs) 12712882Sspwilson2@wisc.edu 12812882Sspwilson2@wisc.educlass MatchStdout(DerivedGoldStandard): 12912882Sspwilson2@wisc.edu _file = constants.gem5_simulation_stdout 13012882Sspwilson2@wisc.edu _default_ignore_regex = [ 13112882Sspwilson2@wisc.edu re.compile('^Redirecting (stdout|stderr) to'), 13212882Sspwilson2@wisc.edu re.compile('^gem5 compiled '), 13312882Sspwilson2@wisc.edu re.compile('^gem5 started '), 13412882Sspwilson2@wisc.edu re.compile('^gem5 executing on '), 13512882Sspwilson2@wisc.edu re.compile('^command line:'), 13612882Sspwilson2@wisc.edu re.compile("^Couldn't import dot_parser,"), 13712882Sspwilson2@wisc.edu re.compile("^info: kernel located at:"), 13813794Sjason@lowepower.com re.compile("^info: Standard input is not a terminal"), 13912882Sspwilson2@wisc.edu re.compile("^Couldn't unlink "), 14012882Sspwilson2@wisc.edu re.compile("^Using GPU kernel code file\(s\) "), 14112882Sspwilson2@wisc.edu ] 14212882Sspwilson2@wisc.edu 14312882Sspwilson2@wisc.educlass MatchStdoutNoPerf(MatchStdout): 14412882Sspwilson2@wisc.edu _file = constants.gem5_simulation_stdout 14512882Sspwilson2@wisc.edu _default_ignore_regex = MatchStdout._default_ignore_regex + [ 14612882Sspwilson2@wisc.edu re.compile('^Exiting @ tick'), 14712882Sspwilson2@wisc.edu ] 14812882Sspwilson2@wisc.edu 14912882Sspwilson2@wisc.educlass MatchStderr(DerivedGoldStandard): 15012882Sspwilson2@wisc.edu _file = constants.gem5_simulation_stderr 15112882Sspwilson2@wisc.edu _default_ignore_regex = [] 15212882Sspwilson2@wisc.edu 15312882Sspwilson2@wisc.educlass MatchStats(DerivedGoldStandard): 15412882Sspwilson2@wisc.edu # TODO: Likely will want to change this verifier since we have the weird 15512882Sspwilson2@wisc.edu # perl script right now. A simple diff probably isn't going to work. 15612882Sspwilson2@wisc.edu _file = constants.gem5_simulation_stats 15712882Sspwilson2@wisc.edu _default_ignore_regex = [] 15812882Sspwilson2@wisc.edu 15912882Sspwilson2@wisc.educlass MatchConfigINI(DerivedGoldStandard): 16012882Sspwilson2@wisc.edu _file = constants.gem5_simulation_config_ini 16112882Sspwilson2@wisc.edu _default_ignore_regex = ( 16212882Sspwilson2@wisc.edu re.compile("^(executable|readfile|kernel|image_file)="), 16312882Sspwilson2@wisc.edu re.compile("^(cwd|input|codefile)="), 16412882Sspwilson2@wisc.edu ) 16512882Sspwilson2@wisc.edu 16612882Sspwilson2@wisc.educlass MatchConfigJSON(DerivedGoldStandard): 16712882Sspwilson2@wisc.edu _file = constants.gem5_simulation_config_json 16812882Sspwilson2@wisc.edu _default_ignore_regex = ( 16912882Sspwilson2@wisc.edu re.compile(r'''^\s*"(executable|readfile|kernel|image_file)":'''), 17012882Sspwilson2@wisc.edu re.compile(r'''^\s*"(cwd|input|codefile)":'''), 17112882Sspwilson2@wisc.edu ) 17212882Sspwilson2@wisc.edu 17312882Sspwilson2@wisc.educlass MatchRegex(Verifier): 17412882Sspwilson2@wisc.edu def __init__(self, regex, match_stderr=True, match_stdout=True): 17512882Sspwilson2@wisc.edu super(MatchRegex, self).__init__() 17612882Sspwilson2@wisc.edu self.regex = _iterable_regex(regex) 17712882Sspwilson2@wisc.edu self.match_stderr = match_stderr 17812882Sspwilson2@wisc.edu self.match_stdout = match_stdout 17912882Sspwilson2@wisc.edu 18012882Sspwilson2@wisc.edu def test(self, params): 18112882Sspwilson2@wisc.edu fixtures = params.fixtures 18212882Sspwilson2@wisc.edu # Get the file from the tempdir of the test. 18312882Sspwilson2@wisc.edu tempdir = fixtures[constants.tempdir_fixture_name].path 18412882Sspwilson2@wisc.edu 18512882Sspwilson2@wisc.edu def parse_file(fname): 18612882Sspwilson2@wisc.edu with open(fname, 'r') as file_: 18712882Sspwilson2@wisc.edu for line in file_: 18812882Sspwilson2@wisc.edu for regex in self.regex: 18912882Sspwilson2@wisc.edu if re.match(regex, line): 19012882Sspwilson2@wisc.edu return True 19112882Sspwilson2@wisc.edu if self.match_stdout: 19212882Sspwilson2@wisc.edu if parse_file(joinpath(tempdir, 19312882Sspwilson2@wisc.edu constants.gem5_simulation_stdout)): 19412882Sspwilson2@wisc.edu return # Success 19512882Sspwilson2@wisc.edu if self.match_stderr: 19612882Sspwilson2@wisc.edu if parse_file(joinpath(tempdir, 19712882Sspwilson2@wisc.edu constants.gem5_simulation_stderr)): 19812882Sspwilson2@wisc.edu return # Success 19912882Sspwilson2@wisc.edu self.failed(fixtures) 20012882Sspwilson2@wisc.edu test.fail('Could not match regex.') 20112882Sspwilson2@wisc.edu 20212882Sspwilson2@wisc.edu_re_type = type(re.compile('')) 20312882Sspwilson2@wisc.edudef _iterable_regex(regex): 20412882Sspwilson2@wisc.edu if isinstance(regex, _re_type) or isinstance(regex, str): 20512882Sspwilson2@wisc.edu regex = (regex,) 20612882Sspwilson2@wisc.edu return regex 207