verifier.py revision 12882:dd87d7f2f3e5 
1# Copyright (c) 2017 Mark D. Hill and David A. Wood
2# All rights reserved.
3#
4# Redistribution and use in source and binary forms, with or without
5# modification, are permitted provided that the following conditions are
6# met: redistributions of source code must retain the above copyright
7# notice, this list of conditions and the following disclaimer;
8# redistributions in binary form must reproduce the above copyright
9# notice, this list of conditions and the following disclaimer in the
10# documentation and/or other materials provided with the distribution;
11# neither the name of the copyright holders nor the names of its
12# contributors may be used to endorse or promote products derived from
13# this software without specific prior written permission.
14#
15# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
16# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
17# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
18# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
19# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
20# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
21# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
22# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
23# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
24# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
25# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
26#
27# Authors: Sean Wilson
28
29'''
30Built in test cases that verify particular details about a gem5 run.
31'''
32import re
33
34from testlib import test
35from testlib.config import constants
36from testlib.helper import joinpath, diff_out_file
37
38class Verifier(object):
39    def __init__(self, fixtures=tuple()):
40        self.fixtures = fixtures
41
42    def _test(self, *args, **kwargs):
43        # Use a callback wrapper to make stack
44        # traces easier to understand.
45        self.test(*args, **kwargs)
46
47    def instantiate_test(self, name_pfx):
48        name = '-'.join([name_pfx, self.__class__.__name__])
49        return test.TestFunction(self._test,
50                name=name, fixtures=self.fixtures)
51
52    def failed(self, fixtures):
53        '''
54        Called if this verifier fails to cleanup (or not) as needed.
55        '''
56        try:
57            fixtures[constants.tempdir_fixture_name].skip_cleanup()
58        except KeyError:
59            pass # No need to do anything if the tempdir fixture doesn't exist
60
61
62class MatchGoldStandard(Verifier):
63    '''
64    Compares a standard output to the test output and passes if they match,
65    fails if they do not.
66    '''
67    def __init__(self, standard_filename, ignore_regex=None,
68                 test_filename='simout'):
69        '''
70        :param standard_filename: The path of the standard file to compare
71        output to.
72
73        :param ignore_regex: A string, compiled regex, or iterable containing
74        either which will be ignored in 'standard' and test output files when
75        diffing.
76        '''
77        super(MatchGoldStandard, self).__init__()
78        self.standard_filename = standard_filename
79        self.test_filename = test_filename
80
81        self.ignore_regex = _iterable_regex(ignore_regex)
82
83    def test(self, params):
84        # We need a tempdir fixture from our parent verifier suite.
85        fixtures = params.fixtures
86        # Get the file from the tempdir of the test.
87        tempdir = fixtures[constants.tempdir_fixture_name].path
88        self.test_filename = joinpath(tempdir, self.test_filename)
89
90        diff = diff_out_file(self.standard_filename,
91                            self.test_filename,
92                            ignore_regexes=self.ignore_regex,
93                            logger=params.log)
94        if diff is not None:
95            self.failed(fixtures)
96            test.fail('Stdout did not match:\n%s\nSee %s for full results'
97                      % (diff, tempdir))
98
99    def _generic_instance_warning(self, kwargs):
100        '''
101        Method for helper classes to tell users to use this more generic class
102        if they are going to manually override the test_filename param.
103        '''
104        if 'test_filename' in kwargs:
105            raise ValueError('If you are setting test_filename use the more'
106                             ' generic %s'
107                             ' instead' % MatchGoldStandard.__name__)
108
109class DerivedGoldStandard(MatchGoldStandard):
110    __ignore_regex_sentinel = object()
111    _file = None
112    _default_ignore_regex = []
113
114    def __init__(self, standard_filename,
115                 ignore_regex=__ignore_regex_sentinel, **kwargs):
116
117        if ignore_regex == self.__ignore_regex_sentinel:
118            ignore_regex = self._default_ignore_regex
119
120        self._generic_instance_warning(kwargs)
121
122        super(DerivedGoldStandard, self).__init__(
123            standard_filename,
124            test_filename=self._file,
125            ignore_regex=ignore_regex,
126            **kwargs)
127
128class MatchStdout(DerivedGoldStandard):
129    _file = constants.gem5_simulation_stdout
130    _default_ignore_regex = [
131            re.compile('^Redirecting (stdout|stderr) to'),
132            re.compile('^gem5 compiled '),
133            re.compile('^gem5 started '),
134            re.compile('^gem5 executing on '),
135            re.compile('^command line:'),
136            re.compile("^Couldn't import dot_parser,"),
137            re.compile("^info: kernel located at:"),
138            re.compile("^Couldn't unlink "),
139            re.compile("^Using GPU kernel code file\(s\) "),
140        ]
141
142class MatchStdoutNoPerf(MatchStdout):
143    _file = constants.gem5_simulation_stdout
144    _default_ignore_regex = MatchStdout._default_ignore_regex + [
145            re.compile('^Exiting @ tick'),
146        ]
147
148class MatchStderr(DerivedGoldStandard):
149    _file = constants.gem5_simulation_stderr
150    _default_ignore_regex = []
151
152class MatchStats(DerivedGoldStandard):
153    # TODO: Likely will want to change this verifier since we have the weird
154    # perl script right now. A simple diff probably isn't going to work.
155    _file = constants.gem5_simulation_stats
156    _default_ignore_regex = []
157
158class MatchConfigINI(DerivedGoldStandard):
159    _file = constants.gem5_simulation_config_ini
160    _default_ignore_regex = (
161            re.compile("^(executable|readfile|kernel|image_file)="),
162            re.compile("^(cwd|input|codefile)="),
163            )
164
165class MatchConfigJSON(DerivedGoldStandard):
166    _file = constants.gem5_simulation_config_json
167    _default_ignore_regex = (
168            re.compile(r'''^\s*"(executable|readfile|kernel|image_file)":'''),
169            re.compile(r'''^\s*"(cwd|input|codefile)":'''),
170            )
171
172class MatchRegex(Verifier):
173    def __init__(self, regex, match_stderr=True, match_stdout=True):
174        super(MatchRegex, self).__init__()
175        self.regex = _iterable_regex(regex)
176        self.match_stderr = match_stderr
177        self.match_stdout = match_stdout
178
179    def test(self, params):
180        fixtures = params.fixtures
181        # Get the file from the tempdir of the test.
182        tempdir = fixtures[constants.tempdir_fixture_name].path
183
184        def parse_file(fname):
185            with open(fname, 'r') as file_:
186                for line in file_:
187                    for regex in self.regex:
188                        if re.match(regex, line):
189                            return True
190        if self.match_stdout:
191            if parse_file(joinpath(tempdir,
192                                   constants.gem5_simulation_stdout)):
193                return # Success
194        if self.match_stderr:
195            if parse_file(joinpath(tempdir,
196                                   constants.gem5_simulation_stderr)):
197                return # Success
198        self.failed(fixtures)
199        test.fail('Could not match regex.')
200
201_re_type = type(re.compile(''))
202def _iterable_regex(regex):
203    if isinstance(regex, _re_type) or isinstance(regex, str):
204        regex = (regex,)
205    return regex
206