1# Copyright (c) 2017 Mark D. Hill and David A. Wood
2# All rights reserved.
3#
4# Redistribution and use in source and binary forms, with or without
5# modification, are permitted provided that the following conditions are
6# met: redistributions of source code must retain the above copyright
7# notice, this list of conditions and the following disclaimer;
8# redistributions in binary form must reproduce the above copyright
9# notice, this list of conditions and the following disclaimer in the
10# documentation and/or other materials provided with the distribution;
11# neither the name of the copyright holders nor the names of its
12# contributors may be used to endorse or promote products derived from
13# this software without specific prior written permission.
14#
15# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
16# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
17# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
18# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
19# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
20# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
21# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
22# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
23# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
24# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
25# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
26#
27# Authors: Sean Wilson
28
29import os
30import itertools
31
32import config
33import fixture as fixture_mod
34import handlers
35import loader as loader_mod
36import log
37import query
38import result
39import runner
40import terminal
41import uid
42
43def entry_message():
44    log.test_log.message("Running the new gem5 testing script.")
45    log.test_log.message("For more information see TESTING.md.")
46    log.test_log.message("To see details as the testing scripts are"
47                         " running, use the option"
48                         " -v, -vv, or -vvv")
49
50class RunLogHandler():
51    def __init__(self):
52        term_handler = handlers.TerminalHandler(
53            verbosity=config.config.verbose+log.LogLevel.Info
54        )
55        summary_handler = handlers.SummaryHandler()
56        self.mp_handler = handlers.MultiprocessingHandlerWrapper(
57                summary_handler, term_handler)
58        self.mp_handler.async_process()
59        log.test_log.log_obj.add_handler(self.mp_handler)
60        entry_message()
61
62    def schedule_finalized(self, test_schedule):
63        # Create the result handler object.
64        self.result_handler = handlers.ResultHandler(
65                test_schedule, config.config.result_path)
66        self.mp_handler.add_handler(self.result_handler)
67
68    def finish_testing(self):
69        self.result_handler.close()
70
71    def __enter__(self):
72        return self
73
74    def __exit__(self, *args):
75        self.close()
76        return False
77
78    def close(self):
79        self.mp_handler.close()
80
81    def unsuccessful(self):
82        '''
83        Performs an or reduce on all of the results.
84        Returns true if at least one test is unsuccessful, false when all tests
85        pass
86        '''
87        return self.result_handler.unsuccessful()
88
89def get_config_tags():
90    return getattr(config.config,
91            config.StorePositionalTagsAction.position_kword)
92
93def filter_with_config_tags(loaded_library):
94    tags = get_config_tags()
95    final_tags = []
96    regex_fmt = '^%s$'
97    cfg = config.config
98
99    def _append_inc_tag_filter(name):
100        if hasattr(cfg, name):
101            tag_opts = getattr(cfg, name)
102            for tag in tag_opts:
103                final_tags.append(config.TagRegex(True, regex_fmt % tag))
104
105    def _append_rem_tag_filter(name):
106        if hasattr(cfg, name):
107            tag_opts = getattr(cfg, name)
108            for tag in cfg.constants.supported_tags[name]:
109                if tag not in tag_opts:
110                    final_tags.append(config.TagRegex(False, regex_fmt % tag))
111
112    # Append additional tags for the isa, length, and variant options.
113    # They apply last (they take priority)
114    special_tags = (
115        cfg.constants.isa_tag_type,
116        cfg.constants.length_tag_type,
117        cfg.constants.variant_tag_type
118    )
119
120    for tagname in special_tags:
121        _append_inc_tag_filter(tagname)
122    for tagname in special_tags:
123        _append_rem_tag_filter(tagname)
124
125    if tags is None:
126        tags = tuple()
127
128    filters = list(itertools.chain(tags, final_tags))
129    string = 'Filtering suites with tags as follows:\n'
130    filter_string = '\t\n'.join((str(f) for f in filters))
131    log.test_log.trace(string + filter_string)
132
133    return filter_with_tags(loaded_library, filters)
134
135
136def filter_with_tags(loaded_library, filters):
137    '''
138    Filter logic supports two filter types:
139    --include-tags <regex>
140    --exclude-tags <regex>
141
142    The logic maintains a `set` of test suites.
143
144    If the regex provided with the `--include-tags` flag matches a tag of a
145    suite, that suite will added to the set.
146
147    If the regex provided with the `--exclude-tags` flag matches a tag of a
148    suite, that suite will removed to the set.
149
150    Suites can be added and removed multiple times.
151
152    First Flag Special Case Logic:
153    If include is the first flag, start with an empty set of suites.
154    If exclude is the first flag, start with the set of all collected suites.
155
156
157    Let's trace out the set as we go through the flags to clarify::
158
159        # Say our collection of suites looks like this: set(suite_ARM64,
160        # suite_X86, suite_Other).
161        #
162        # Additionally, we've passed the flags in the following order:
163        #  --include-tags "ARM64"  --exclude-tags ".*" --include-tags "X86"
164
165        # Process --include-tags "ARM64"
166        set(suite_ARM64)    # Suite begins empty, but adds the ARM64 suite
167        # Process --exclude-tags ".*"
168        set()               # Removed all suites which have tags
169        # Process --include-tags "X86"
170        set(suite_X86)
171    '''
172    if not filters:
173        return
174
175    query_runner = query.QueryRunner(loaded_library)
176    tags = query_runner.tags()
177
178    if not filters[0].include:
179        suites = set(query_runner.suites())
180    else:
181        suites = set()
182
183    def exclude(excludes):
184        return suites - excludes
185    def include(includes):
186        return suites | includes
187
188    for tag_regex in filters:
189        matched_tags = (tag for tag in tags if tag_regex.regex.search(tag))
190        for tag in matched_tags:
191            matched_suites = set(query_runner.suites_with_tag(tag))
192            suites = include(matched_suites) if tag_regex.include \
193                    else exclude(matched_suites)
194
195    # Set the library's suites to only those which where accepted by our filter
196    loaded_library.suites = [suite for suite in loaded_library.suites
197            if suite in suites]
198
199# TODO Add results command for listing previous results.
200
201def load_tests():
202    '''
203    Create a TestLoader and load tests for the directory given by the config.
204    '''
205    testloader = loader_mod.Loader()
206    log.test_log.message(terminal.separator())
207    log.test_log.message('Loading Tests', bold=True)
208    testloader.load_root(config.config.directory)
209    return testloader
210
211def do_list():
212    term_handler = handlers.TerminalHandler(
213        verbosity=config.config.verbose+log.LogLevel.Info,
214        machine_only=config.config.quiet
215    )
216    log.test_log.log_obj.add_handler(term_handler)
217
218    entry_message()
219
220    test_schedule = load_tests().schedule
221    filter_with_config_tags(test_schedule)
222
223    qrunner = query.QueryRunner(test_schedule)
224
225    if config.config.suites:
226        qrunner.list_suites()
227    elif config.config.tests:
228        qrunner.list_tests()
229    elif config.config.all_tags:
230        qrunner.list_tags()
231    else:
232        qrunner.list_suites()
233        qrunner.list_tests()
234        qrunner.list_tags()
235
236    return 0
237
238def run_schedule(test_schedule, log_handler):
239    '''
240    Test Phases
241    -----------
242    * Test Collection
243    * Fixture Parameterization
244    * Global Fixture Setup
245    * Iteratevely run suites:
246       * Suite Fixture Setup
247       * Iteratively run tests:
248          * Test Fixture Setup
249          * Run Test
250          * Test Fixture Teardown
251       * Suite Fixture Teardown
252    * Global Fixture Teardown
253    '''
254
255    log_handler.schedule_finalized(test_schedule)
256
257    log.test_log.message(terminal.separator())
258    log.test_log.message('Running Tests from {} suites'
259            .format(len(test_schedule.suites)), bold=True)
260    log.test_log.message("Results will be stored in {}".format(
261                config.config.result_path))
262    log.test_log.message(terminal.separator())
263
264    # Build global fixtures and exectute scheduled test suites.
265    if config.config.test_threads > 1:
266        library_runner = runner.LibraryParallelRunner(test_schedule)
267        library_runner.set_threads(config.config.test_threads)
268    else:
269        library_runner = runner.LibraryRunner(test_schedule)
270    library_runner.run()
271
272    failed = log_handler.unsuccessful()
273
274    log_handler.finish_testing()
275
276    return 1 if failed else 0
277
278def do_run():
279    # Initialize early parts of the log.
280    with RunLogHandler() as log_handler:
281        if config.config.uid:
282            uid_ = uid.UID.from_uid(config.config.uid)
283            if isinstance(uid_, uid.TestUID):
284                log.test_log.error('Unable to run a standalone test.\n'
285                        'Gem5 expects test suites to be the smallest unit '
286                        ' of test.\n\n'
287                        'Pass a SuiteUID instead.')
288                return
289            test_schedule = loader_mod.Loader().load_schedule_for_suites(uid_)
290            if get_config_tags():
291                log.test_log.warn(
292                    "The '--uid' flag was supplied,"
293                    " '--include-tags' and '--exclude-tags' will be ignored."
294                )
295        else:
296            test_schedule = load_tests().schedule
297            # Filter tests based on tags
298            filter_with_config_tags(test_schedule)
299        # Execute the tests
300        return run_schedule(test_schedule, log_handler)
301
302def do_rerun():
303    # Init early parts of log
304    with RunLogHandler() as log_handler:
305        # Load previous results
306        results = result.InternalSavedResults.load(
307                os.path.join(config.config.result_path,
308                config.constants.pickle_filename))
309
310        rerun_suites = (suite.uid for suite in results if suite.unsuccessful)
311
312        # Use loader to load suites
313        loader = loader_mod.Loader()
314        test_schedule = loader.load_schedule_for_suites(*rerun_suites)
315
316        # Execute the tests
317        return run_schedule(test_schedule, log_handler)
318
319def main():
320    '''
321    Main entrypoint for the testlib test library.
322    Returns 0 on success and 1 otherwise so it can be used as a return code
323    for scripts.
324    '''
325    config.initialize_config()
326
327    # 'do' the given command.
328    result = globals()['do_'+config.config.command]()
329    log.test_log.close()
330
331    return result
332