main.py revision 13789:d7b2be2c468b 
1# Copyright (c) 2017 Mark D. Hill and David A. Wood
2# All rights reserved.
3#
4# Redistribution and use in source and binary forms, with or without
5# modification, are permitted provided that the following conditions are
6# met: redistributions of source code must retain the above copyright
7# notice, this list of conditions and the following disclaimer;
8# redistributions in binary form must reproduce the above copyright
9# notice, this list of conditions and the following disclaimer in the
10# documentation and/or other materials provided with the distribution;
11# neither the name of the copyright holders nor the names of its
12# contributors may be used to endorse or promote products derived from
13# this software without specific prior written permission.
14#
15# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
16# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
17# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
18# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
19# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
20# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
21# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
22# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
23# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
24# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
25# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
26#
27# Authors: Sean Wilson
28
29import os
30import itertools
31
32import config
33import fixture as fixture_mod
34import handlers
35import loader as loader_mod
36import log
37import query
38import result
39import runner
40import terminal
41import uid
42
43def entry_message():
44    log.test_log.message("Running the new gem5 testing script.")
45    log.test_log.message("For more information see TESTING.md.")
46    log.test_log.message("To see details as the testing scripts are"
47                         " running, use the option"
48                         " -v, -vv, or -vvv")
49
50class RunLogHandler():
51    def __init__(self):
52        term_handler = handlers.TerminalHandler(
53            verbosity=config.config.verbose+log.LogLevel.Info
54        )
55        summary_handler = handlers.SummaryHandler()
56        self.mp_handler = handlers.MultiprocessingHandlerWrapper(
57                summary_handler, term_handler)
58        self.mp_handler.async_process()
59        log.test_log.log_obj.add_handler(self.mp_handler)
60        entry_message()
61
62    def schedule_finalized(self, test_schedule):
63        # Create the result handler object.
64        self.result_handler = handlers.ResultHandler(
65                test_schedule, config.config.result_path)
66        self.mp_handler.add_handler(self.result_handler)
67
68    def finish_testing(self):
69        self.result_handler.close()
70
71    def __enter__(self):
72        return self
73
74    def __exit__(self, *args):
75        self.close()
76        return False
77
78    def close(self):
79        self.mp_handler.close()
80
81    def unsuccessful(self):
82        '''
83        Performs an or reduce on all of the results.
84        Returns true if at least one test is unsuccessful, false when all tests
85        pass
86        '''
87        return self.result_handler.unsuccessful()
88
89def get_config_tags():
90    return getattr(config.config,
91            config.StorePositionalTagsAction.position_kword)
92
93def filter_with_config_tags(loaded_library):
94    tags = get_config_tags()
95    final_tags = []
96    regex_fmt = '^%s$'
97    cfg = config.config
98
99    def _append_inc_tag_filter(name):
100        if hasattr(cfg, name):
101            tag_opts = getattr(cfg, name)
102            for tag in tag_opts:
103                final_tags.append(config.TagRegex(True, regex_fmt % tag))
104
105    def _append_rem_tag_filter(name):
106        if hasattr(cfg, name):
107            tag_opts = getattr(cfg, name)
108            for tag in cfg.constants.supported_tags[name]:
109                if tag not in tag_opts:
110                    final_tags.append(config.TagRegex(False, regex_fmt % tag))
111
112    # Append additional tags for the isa, length, and variant options.
113    # They apply last (they take priority)
114    special_tags = (
115        cfg.constants.isa_tag_type,
116        cfg.constants.length_tag_type,
117        cfg.constants.variant_tag_type
118    )
119
120    for tagname in special_tags:
121        _append_inc_tag_filter(tagname)
122    for tagname in special_tags:
123        _append_rem_tag_filter(tagname)
124
125    if tags is None:
126        tags = tuple()
127
128    filters = list(itertools.chain(tags, final_tags))
129    string = 'Filtering suites with tags as follows:\n'
130    filter_string = '\t\n'.join((str(f) for f in filters))
131    log.test_log.trace(string + filter_string)
132
133    return filter_with_tags(loaded_library, filters)
134
135
136def filter_with_tags(loaded_library, filters):
137    '''
138    Filter logic supports two filter types:
139    --include-tags <regex>
140    --exclude-tags <regex>
141
142    The logic maintains a `set` of test suites.
143
144    If the regex provided with the `--include-tags` flag matches a tag of a
145    suite, that suite will added to the set.
146
147    If the regex provided with the `--exclude-tags` flag matches a tag of a
148    suite, that suite will removed to the set.
149
150    Suites can be added and removed multiple times.
151
152    First Flag Special Case Logic:
153    If include is the first flag, start with an empty set of suites.
154    If exclude is the first flag, start with the set of all collected suites.
155
156
157    Let's trace out the set as we go through the flags to clarify::
158
159        # Say our collection of suites looks like this: set(suite_ARM64,
160        # suite_X86, suite_Other).
161        #
162        # Additionally, we've passed the flags in the following order:
163        #  --include-tags "ARM64"  --exclude-tags ".*" --include-tags "X86"
164
165        # Process --include-tags "ARM64"
166        set(suite_ARM64)    # Suite begins empty, but adds the ARM64 suite
167        # Process --exclude-tags ".*"
168        set()               # Removed all suites which have tags
169        # Process --include-tags "X86"
170        set(suite_X86)
171    '''
172    if not filters:
173        return
174
175    query_runner = query.QueryRunner(loaded_library)
176    tags = query_runner.tags()
177
178    if not filters[0].include:
179        suites = set(query_runner.suites())
180    else:
181        suites = set()
182
183    def exclude(excludes):
184        return suites - excludes
185    def include(includes):
186        return suites | includes
187
188    for tag_regex in filters:
189        matched_tags = (tag for tag in tags if tag_regex.regex.search(tag))
190        for tag in matched_tags:
191            matched_suites = set(query_runner.suites_with_tag(tag))
192            suites = include(matched_suites) if tag_regex.include \
193                    else exclude(matched_suites)
194
195    # Set the library's suites to only those which where accepted by our filter
196    loaded_library.suites = [suite for suite in loaded_library.suites
197            if suite in suites]
198
199# TODO Add results command for listing previous results.
200
201def load_tests():
202    '''
203    Create a TestLoader and load tests for the directory given by the config.
204    '''
205    testloader = loader_mod.Loader()
206    log.test_log.message(terminal.separator())
207    log.test_log.message('Loading Tests', bold=True)
208    testloader.load_root(config.config.directory)
209    return testloader
210
211def do_list():
212    term_handler = handlers.TerminalHandler(
213        verbosity=config.config.verbose+log.LogLevel.Info,
214        machine_only=config.config.quiet
215    )
216    log.test_log.log_obj.add_handler(term_handler)
217
218    entry_message()
219
220    test_schedule = load_tests().schedule
221    filter_with_config_tags(test_schedule)
222
223    qrunner = query.QueryRunner(test_schedule)
224
225    if config.config.suites:
226        qrunner.list_suites()
227    elif config.config.tests:
228        qrunner.list_tests()
229    elif config.config.all_tags:
230        qrunner.list_tags()
231    else:
232        qrunner.list_suites()
233        qrunner.list_tests()
234        qrunner.list_tags()
235
236    return 0
237
238def run_schedule(test_schedule, log_handler):
239    '''
240    Test Phases
241    -----------
242    * Test Collection
243    * Fixture Parameterization
244    * Global Fixture Setup
245    * Iteratevely run suites:
246       * Suite Fixture Setup
247       * Iteratively run tests:
248          * Test Fixture Setup
249          * Run Test
250          * Test Fixture Teardown
251       * Suite Fixture Teardown
252    * Global Fixture Teardown
253    '''
254
255    log_handler.schedule_finalized(test_schedule)
256
257    # Iterate through all fixtures notifying them of the test schedule.
258    for suite in test_schedule:
259        copied_fixtures = []
260        for fixture in suite.fixtures:
261            copied_fixtures.append(fixture.schedule_finalized(test_schedule))
262        suite.fixtures = copied_fixtures
263
264        for test in suite:
265            copied_fixtures = []
266            for fixture in test.fixtures:
267                copied_fixtures.append(fixture.schedule_finalized(
268                        test_schedule))
269            test.fixtures = copied_fixtures
270
271    log.test_log.message(terminal.separator())
272    log.test_log.message('Running Tests from {} suites'
273            .format(len(test_schedule.suites)), bold=True)
274    log.test_log.message("Results will be stored in {}".format(
275                config.config.result_path))
276    log.test_log.message(terminal.separator())
277
278    # Build global fixtures and exectute scheduled test suites.
279    if config.config.test_threads > 1:
280        library_runner = runner.LibraryParallelRunner(test_schedule)
281        library_runner.set_threads(config.config.test_threads)
282    else:
283        library_runner = runner.LibraryRunner(test_schedule)
284    library_runner.run()
285
286    failed = log_handler.unsuccessful()
287
288    log_handler.finish_testing()
289
290    return 1 if failed else 0
291
292def do_run():
293    # Initialize early parts of the log.
294    with RunLogHandler() as log_handler:
295        if config.config.uid:
296            uid_ = uid.UID.from_uid(config.config.uid)
297            if isinstance(uid_, uid.TestUID):
298                log.test_log.error('Unable to run a standalone test.\n'
299                        'Gem5 expects test suites to be the smallest unit '
300                        ' of test.\n\n'
301                        'Pass a SuiteUID instead.')
302                return
303            test_schedule = loader_mod.Loader().load_schedule_for_suites(uid_)
304            if get_config_tags():
305                log.test_log.warn(
306                    "The '--uid' flag was supplied,"
307                    " '--include-tags' and '--exclude-tags' will be ignored."
308                )
309        else:
310            test_schedule = load_tests().schedule
311            # Filter tests based on tags
312            filter_with_config_tags(test_schedule)
313        # Execute the tests
314        return run_schedule(test_schedule, log_handler)
315
316def do_rerun():
317    # Init early parts of log
318    with RunLogHandler() as log_handler:
319        # Load previous results
320        results = result.InternalSavedResults.load(
321                os.path.join(config.config.result_path,
322                config.constants.pickle_filename))
323
324        rerun_suites = (suite.uid for suite in results if suite.unsuccessful)
325
326        # Use loader to load suites
327        loader = loader_mod.Loader()
328        test_schedule = loader.load_schedule_for_suites(*rerun_suites)
329
330        # Execute the tests
331        return run_schedule(test_schedule, log_handler)
332
333def main():
334    '''
335    Main entrypoint for the testlib test library.
336    Returns 0 on success and 1 otherwise so it can be used as a return code
337    for scripts.
338    '''
339    config.initialize_config()
340
341    # 'do' the given command.
342    result = globals()['do_'+config.config.command]()
343    log.test_log.close()
344
345    return result
346