main.py revision 12882 
1# Copyright (c) 2017 Mark D. Hill and David A. Wood
2# All rights reserved.
3#
4# Redistribution and use in source and binary forms, with or without
5# modification, are permitted provided that the following conditions are
6# met: redistributions of source code must retain the above copyright
7# notice, this list of conditions and the following disclaimer;
8# redistributions in binary form must reproduce the above copyright
9# notice, this list of conditions and the following disclaimer in the
10# documentation and/or other materials provided with the distribution;
11# neither the name of the copyright holders nor the names of its
12# contributors may be used to endorse or promote products derived from
13# this software without specific prior written permission.
14#
15# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
16# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
17# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
18# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
19# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
20# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
21# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
22# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
23# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
24# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
25# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
26#
27# Authors: Sean Wilson
28
29import os
30import itertools
31
32import config
33import fixture as fixture_mod
34import handlers
35import loader as loader_mod
36import log
37import query
38import result
39import runner
40import terminal
41import uid
42
43def entry_message():
44    log.test_log.message("Running the new gem5 testing script.")
45    log.test_log.message("For more information see TESTING.md.")
46    log.test_log.message("To see details as the testing scripts are"
47                         " running, use the option"
48                         " -v, -vv, or -vvv")
49
50class RunLogHandler():
51    def __init__(self):
52        term_handler = handlers.TerminalHandler(
53            verbosity=config.config.verbose+log.LogLevel.Info
54        )
55        summary_handler = handlers.SummaryHandler()
56        self.mp_handler = handlers.MultiprocessingHandlerWrapper(
57                summary_handler, term_handler)
58        self.mp_handler.async_process()
59        log.test_log.log_obj.add_handler(self.mp_handler)
60        entry_message()
61
62    def schedule_finalized(self, test_schedule):
63        # Create the result handler object.
64        self.result_handler = handlers.ResultHandler(
65                test_schedule, config.config.result_path)
66        self.mp_handler.add_handler(self.result_handler)
67
68    def finish_testing(self):
69        self.result_handler.close()
70
71    def __enter__(self):
72        return self
73
74    def __exit__(self, *args):
75        self.close()
76        return False
77
78    def close(self):
79        self.mp_handler.close()
80
81def get_config_tags():
82    return getattr(config.config,
83            config.StorePositionalTagsAction.position_kword)
84
85def filter_with_config_tags(loaded_library):
86    tags = get_config_tags()
87    final_tags = []
88    regex_fmt = '^%s$'
89    cfg = config.config
90
91    def _append_inc_tag_filter(name):
92        if hasattr(cfg, name):
93            tag_opts = getattr(cfg, name)
94            for tag in tag_opts:
95                final_tags.append(config.TagRegex(True, regex_fmt % tag))
96
97    def _append_rem_tag_filter(name):
98        if hasattr(cfg, name):
99            tag_opts = getattr(cfg, name)
100            for tag in cfg.constants.supported_tags[name]:
101                if tag not in tag_opts:
102                    final_tags.append(config.TagRegex(False, regex_fmt % tag))
103
104    # Append additional tags for the isa, length, and variant options.
105    # They apply last (they take priority)
106    special_tags = (
107        cfg.constants.isa_tag_type,
108        cfg.constants.length_tag_type,
109        cfg.constants.variant_tag_type
110    )
111
112    for tagname in special_tags:
113        _append_inc_tag_filter(tagname)
114    for tagname in special_tags:
115        _append_rem_tag_filter(tagname)
116
117    if tags is None:
118        tags = tuple()
119
120    filters = list(itertools.chain(tags, final_tags))
121    string = 'Filtering suites with tags as follows:\n'
122    filter_string = '\t\n'.join((str(f) for f in filters))
123    log.test_log.trace(string + filter_string)
124
125    return filter_with_tags(loaded_library, filters)
126
127
128def filter_with_tags(loaded_library, filters):
129    '''
130    Filter logic supports two filter types:
131    --include-tags <regex>
132    --exclude-tags <regex>
133
134    The logic maintains a `set` of test suites.
135
136    If the regex provided with the `--include-tags` flag matches a tag of a
137    suite, that suite will added to the set.
138
139    If the regex provided with the `--exclude-tags` flag matches a tag of a
140    suite, that suite will removed to the set.
141
142    Suites can be added and removed multiple times.
143
144    First Flag Special Case Logic:
145    If include is the first flag, start with an empty set of suites.
146    If exclude is the first flag, start with the set of all collected suites.
147
148
149    Let's trace out the set as we go through the flags to clarify::
150
151        # Say our collection of suites looks like this: set(suite_ARM64,
152        # suite_X86, suite_Other).
153        #
154        # Additionally, we've passed the flags in the following order:
155        #  --include-tags "ARM64"  --exclude-tags ".*" --include-tags "X86"
156
157        # Process --include-tags "ARM64"
158        set(suite_ARM64)    # Suite begins empty, but adds the ARM64 suite
159        # Process --exclude-tags ".*"
160        set()               # Removed all suites which have tags
161        # Process --include-tags "X86"
162        set(suite_X86)
163    '''
164    if not filters:
165        return
166
167    query_runner = query.QueryRunner(loaded_library)
168    tags = query_runner.tags()
169
170    if not filters[0].include:
171        suites = set(query_runner.suites())
172    else:
173        suites = set()
174
175    def exclude(excludes):
176        return suites - excludes
177    def include(includes):
178        return suites | includes
179
180    for tag_regex in filters:
181        matched_tags = (tag for tag in tags if tag_regex.regex.search(tag))
182        for tag in matched_tags:
183            matched_suites = set(query_runner.suites_with_tag(tag))
184            suites = include(matched_suites) if tag_regex.include \
185                    else exclude(matched_suites)
186
187    # Set the library's suites to only those which where accepted by our filter
188    loaded_library.suites = [suite for suite in loaded_library.suites
189            if suite in suites]
190
191# TODO Add results command for listing previous results.
192
193def load_tests():
194    '''
195    Create a TestLoader and load tests for the directory given by the config.
196    '''
197    testloader = loader_mod.Loader()
198    log.test_log.message(terminal.separator())
199    log.test_log.message('Loading Tests', bold=True)
200    testloader.load_root(config.config.directory)
201    return testloader
202
203def do_list():
204    term_handler = handlers.TerminalHandler(
205        verbosity=config.config.verbose+log.LogLevel.Info,
206        machine_only=config.config.quiet
207    )
208    log.test_log.log_obj.add_handler(term_handler)
209
210    entry_message()
211
212    test_schedule = load_tests().schedule
213    filter_with_config_tags(test_schedule)
214
215    qrunner = query.QueryRunner(test_schedule)
216
217    if config.config.suites:
218        qrunner.list_suites()
219    elif config.config.tests:
220        qrunner.list_tests()
221    elif config.config.all_tags:
222        qrunner.list_tags()
223    else:
224        qrunner.list_suites()
225        qrunner.list_tests()
226        qrunner.list_tags()
227
228def run_schedule(test_schedule, log_handler):
229    '''
230    Test Phases
231    -----------
232    * Test Collection
233    * Fixture Parameterization
234    * Global Fixture Setup
235    * Iteratevely run suites:
236       * Suite Fixture Setup
237       * Iteratively run tests:
238          * Test Fixture Setup
239          * Run Test
240          * Test Fixture Teardown
241       * Suite Fixture Teardown
242    * Global Fixture Teardown
243    '''
244
245    log_handler.schedule_finalized(test_schedule)
246
247    # Iterate through all fixtures notifying them of the test schedule.
248    for suite in test_schedule:
249        copied_fixtures = []
250        for fixture in suite.fixtures:
251            copied_fixtures.append(fixture.schedule_finalized(test_schedule))
252        suite.fixtures = copied_fixtures
253
254        for test in suite:
255            copied_fixtures = []
256            for fixture in test.fixtures:
257                copied_fixtures.append(fixture.schedule_finalized(
258                        test_schedule))
259            test.fixtures = copied_fixtures
260
261    log.test_log.message(terminal.separator())
262    log.test_log.message('Running Tests from {} suites'
263            .format(len(test_schedule.suites)), bold=True)
264    log.test_log.message("Results will be stored in {}".format(
265                config.config.result_path))
266    log.test_log.message(terminal.separator())
267
268    # Build global fixtures and exectute scheduled test suites.
269    if config.config.test_threads > 1:
270        library_runner = runner.LibraryParallelRunner(test_schedule)
271        library_runner.set_threads(config.config.test_threads)
272    else:
273        library_runner = runner.LibraryRunner(test_schedule)
274    library_runner.run()
275
276    log_handler.finish_testing()
277
278def do_run():
279    # Initialize early parts of the log.
280    with RunLogHandler() as log_handler:
281        if config.config.uid:
282            uid_ = uid.UID.from_uid(config.config.uid)
283            if isinstance(uid_, uid.TestUID):
284                log.test_log.error('Unable to run a standalone test.\n'
285                        'Gem5 expects test suites to be the smallest unit '
286                        ' of test.\n\n'
287                        'Pass a SuiteUID instead.')
288                return
289            test_schedule = loader_mod.Loader().load_schedule_for_suites(uid_)
290            if get_config_tags():
291                log.test_log.warn(
292                    "The '--uid' flag was supplied,"
293                    " '--include-tags' and '--exclude-tags' will be ignored."
294                )
295        else:
296            test_schedule = load_tests().schedule
297            # Filter tests based on tags
298            filter_with_config_tags(test_schedule)
299        # Execute the tests
300        run_schedule(test_schedule, log_handler)
301
302
303def do_rerun():
304    # Init early parts of log
305    with RunLogHandler() as log_handler:
306        # Load previous results
307        results = result.InternalSavedResults.load(
308                os.path.join(config.config.result_path,
309                config.constants.pickle_filename))
310
311        rerun_suites = (suite.uid for suite in results if suite.unsucessful)
312
313        # Use loader to load suites
314        loader = loader_mod.Loader()
315        test_schedule = loader.load_schedule_for_suites(*rerun_suites)
316
317        # Execute the tests
318        run_schedule(test_schedule, log_handler)
319
320def main():
321    '''
322    Main entrypoint for the testlib test library.
323    '''
324    config.initialize_config()
325
326    # 'do' the given command.
327    globals()['do_'+config.config.command]()
328    log.test_log.close()
329