main.py revision 13789
112882Sspwilson2@wisc.edu# Copyright (c) 2017 Mark D. Hill and David A. Wood 212882Sspwilson2@wisc.edu# All rights reserved. 312882Sspwilson2@wisc.edu# 412882Sspwilson2@wisc.edu# Redistribution and use in source and binary forms, with or without 512882Sspwilson2@wisc.edu# modification, are permitted provided that the following conditions are 612882Sspwilson2@wisc.edu# met: redistributions of source code must retain the above copyright 712882Sspwilson2@wisc.edu# notice, this list of conditions and the following disclaimer; 812882Sspwilson2@wisc.edu# redistributions in binary form must reproduce the above copyright 912882Sspwilson2@wisc.edu# notice, this list of conditions and the following disclaimer in the 1012882Sspwilson2@wisc.edu# documentation and/or other materials provided with the distribution; 1112882Sspwilson2@wisc.edu# neither the name of the copyright holders nor the names of its 1212882Sspwilson2@wisc.edu# contributors may be used to endorse or promote products derived from 1312882Sspwilson2@wisc.edu# this software without specific prior written permission. 1412882Sspwilson2@wisc.edu# 1512882Sspwilson2@wisc.edu# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS 1612882Sspwilson2@wisc.edu# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT 1712882Sspwilson2@wisc.edu# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR 1812882Sspwilson2@wisc.edu# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT 1912882Sspwilson2@wisc.edu# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, 2012882Sspwilson2@wisc.edu# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT 2112882Sspwilson2@wisc.edu# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, 2212882Sspwilson2@wisc.edu# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY 2312882Sspwilson2@wisc.edu# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT 2412882Sspwilson2@wisc.edu# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE 2512882Sspwilson2@wisc.edu# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 2612882Sspwilson2@wisc.edu# 2712882Sspwilson2@wisc.edu# Authors: Sean Wilson 2812882Sspwilson2@wisc.edu 2912882Sspwilson2@wisc.eduimport os 3012882Sspwilson2@wisc.eduimport itertools 3112882Sspwilson2@wisc.edu 3212882Sspwilson2@wisc.eduimport config 3312882Sspwilson2@wisc.eduimport fixture as fixture_mod 3412882Sspwilson2@wisc.eduimport handlers 3512882Sspwilson2@wisc.eduimport loader as loader_mod 3612882Sspwilson2@wisc.eduimport log 3712882Sspwilson2@wisc.eduimport query 3812882Sspwilson2@wisc.eduimport result 3912882Sspwilson2@wisc.eduimport runner 4012882Sspwilson2@wisc.eduimport terminal 4112882Sspwilson2@wisc.eduimport uid 4212882Sspwilson2@wisc.edu 4312882Sspwilson2@wisc.edudef entry_message(): 4412882Sspwilson2@wisc.edu log.test_log.message("Running the new gem5 testing script.") 4512882Sspwilson2@wisc.edu log.test_log.message("For more information see TESTING.md.") 4612882Sspwilson2@wisc.edu log.test_log.message("To see details as the testing scripts are" 4712882Sspwilson2@wisc.edu " running, use the option" 4812882Sspwilson2@wisc.edu " -v, -vv, or -vvv") 4912882Sspwilson2@wisc.edu 5012882Sspwilson2@wisc.educlass RunLogHandler(): 5112882Sspwilson2@wisc.edu def __init__(self): 5212882Sspwilson2@wisc.edu term_handler = handlers.TerminalHandler( 5312882Sspwilson2@wisc.edu verbosity=config.config.verbose+log.LogLevel.Info 5412882Sspwilson2@wisc.edu ) 5512882Sspwilson2@wisc.edu summary_handler = handlers.SummaryHandler() 5612882Sspwilson2@wisc.edu self.mp_handler = handlers.MultiprocessingHandlerWrapper( 5712882Sspwilson2@wisc.edu summary_handler, term_handler) 5812882Sspwilson2@wisc.edu self.mp_handler.async_process() 5912882Sspwilson2@wisc.edu log.test_log.log_obj.add_handler(self.mp_handler) 6012882Sspwilson2@wisc.edu entry_message() 6112882Sspwilson2@wisc.edu 6212882Sspwilson2@wisc.edu def schedule_finalized(self, test_schedule): 6312882Sspwilson2@wisc.edu # Create the result handler object. 6412882Sspwilson2@wisc.edu self.result_handler = handlers.ResultHandler( 6512882Sspwilson2@wisc.edu test_schedule, config.config.result_path) 6612882Sspwilson2@wisc.edu self.mp_handler.add_handler(self.result_handler) 6712882Sspwilson2@wisc.edu 6812882Sspwilson2@wisc.edu def finish_testing(self): 6912882Sspwilson2@wisc.edu self.result_handler.close() 7012882Sspwilson2@wisc.edu 7112882Sspwilson2@wisc.edu def __enter__(self): 7212882Sspwilson2@wisc.edu return self 7312882Sspwilson2@wisc.edu 7412882Sspwilson2@wisc.edu def __exit__(self, *args): 7512882Sspwilson2@wisc.edu self.close() 7612882Sspwilson2@wisc.edu return False 7712882Sspwilson2@wisc.edu 7812882Sspwilson2@wisc.edu def close(self): 7912882Sspwilson2@wisc.edu self.mp_handler.close() 8012882Sspwilson2@wisc.edu 8113789Sjason@lowepower.com def unsuccessful(self): 8213789Sjason@lowepower.com ''' 8313789Sjason@lowepower.com Performs an or reduce on all of the results. 8413789Sjason@lowepower.com Returns true if at least one test is unsuccessful, false when all tests 8513789Sjason@lowepower.com pass 8613789Sjason@lowepower.com ''' 8713789Sjason@lowepower.com return self.result_handler.unsuccessful() 8813789Sjason@lowepower.com 8912882Sspwilson2@wisc.edudef get_config_tags(): 9012882Sspwilson2@wisc.edu return getattr(config.config, 9112882Sspwilson2@wisc.edu config.StorePositionalTagsAction.position_kword) 9212882Sspwilson2@wisc.edu 9312882Sspwilson2@wisc.edudef filter_with_config_tags(loaded_library): 9412882Sspwilson2@wisc.edu tags = get_config_tags() 9512882Sspwilson2@wisc.edu final_tags = [] 9612882Sspwilson2@wisc.edu regex_fmt = '^%s$' 9712882Sspwilson2@wisc.edu cfg = config.config 9812882Sspwilson2@wisc.edu 9912882Sspwilson2@wisc.edu def _append_inc_tag_filter(name): 10012882Sspwilson2@wisc.edu if hasattr(cfg, name): 10112882Sspwilson2@wisc.edu tag_opts = getattr(cfg, name) 10212882Sspwilson2@wisc.edu for tag in tag_opts: 10312882Sspwilson2@wisc.edu final_tags.append(config.TagRegex(True, regex_fmt % tag)) 10412882Sspwilson2@wisc.edu 10512882Sspwilson2@wisc.edu def _append_rem_tag_filter(name): 10612882Sspwilson2@wisc.edu if hasattr(cfg, name): 10712882Sspwilson2@wisc.edu tag_opts = getattr(cfg, name) 10812882Sspwilson2@wisc.edu for tag in cfg.constants.supported_tags[name]: 10912882Sspwilson2@wisc.edu if tag not in tag_opts: 11012882Sspwilson2@wisc.edu final_tags.append(config.TagRegex(False, regex_fmt % tag)) 11112882Sspwilson2@wisc.edu 11212882Sspwilson2@wisc.edu # Append additional tags for the isa, length, and variant options. 11312882Sspwilson2@wisc.edu # They apply last (they take priority) 11412882Sspwilson2@wisc.edu special_tags = ( 11512882Sspwilson2@wisc.edu cfg.constants.isa_tag_type, 11612882Sspwilson2@wisc.edu cfg.constants.length_tag_type, 11712882Sspwilson2@wisc.edu cfg.constants.variant_tag_type 11812882Sspwilson2@wisc.edu ) 11912882Sspwilson2@wisc.edu 12012882Sspwilson2@wisc.edu for tagname in special_tags: 12112882Sspwilson2@wisc.edu _append_inc_tag_filter(tagname) 12212882Sspwilson2@wisc.edu for tagname in special_tags: 12312882Sspwilson2@wisc.edu _append_rem_tag_filter(tagname) 12412882Sspwilson2@wisc.edu 12512882Sspwilson2@wisc.edu if tags is None: 12612882Sspwilson2@wisc.edu tags = tuple() 12712882Sspwilson2@wisc.edu 12812882Sspwilson2@wisc.edu filters = list(itertools.chain(tags, final_tags)) 12912882Sspwilson2@wisc.edu string = 'Filtering suites with tags as follows:\n' 13012882Sspwilson2@wisc.edu filter_string = '\t\n'.join((str(f) for f in filters)) 13112882Sspwilson2@wisc.edu log.test_log.trace(string + filter_string) 13212882Sspwilson2@wisc.edu 13312882Sspwilson2@wisc.edu return filter_with_tags(loaded_library, filters) 13412882Sspwilson2@wisc.edu 13512882Sspwilson2@wisc.edu 13612882Sspwilson2@wisc.edudef filter_with_tags(loaded_library, filters): 13712882Sspwilson2@wisc.edu ''' 13812882Sspwilson2@wisc.edu Filter logic supports two filter types: 13912882Sspwilson2@wisc.edu --include-tags <regex> 14012882Sspwilson2@wisc.edu --exclude-tags <regex> 14112882Sspwilson2@wisc.edu 14212882Sspwilson2@wisc.edu The logic maintains a `set` of test suites. 14312882Sspwilson2@wisc.edu 14412882Sspwilson2@wisc.edu If the regex provided with the `--include-tags` flag matches a tag of a 14512882Sspwilson2@wisc.edu suite, that suite will added to the set. 14612882Sspwilson2@wisc.edu 14712882Sspwilson2@wisc.edu If the regex provided with the `--exclude-tags` flag matches a tag of a 14812882Sspwilson2@wisc.edu suite, that suite will removed to the set. 14912882Sspwilson2@wisc.edu 15012882Sspwilson2@wisc.edu Suites can be added and removed multiple times. 15112882Sspwilson2@wisc.edu 15212882Sspwilson2@wisc.edu First Flag Special Case Logic: 15312882Sspwilson2@wisc.edu If include is the first flag, start with an empty set of suites. 15412882Sspwilson2@wisc.edu If exclude is the first flag, start with the set of all collected suites. 15512882Sspwilson2@wisc.edu 15612882Sspwilson2@wisc.edu 15712882Sspwilson2@wisc.edu Let's trace out the set as we go through the flags to clarify:: 15812882Sspwilson2@wisc.edu 15912882Sspwilson2@wisc.edu # Say our collection of suites looks like this: set(suite_ARM64, 16012882Sspwilson2@wisc.edu # suite_X86, suite_Other). 16112882Sspwilson2@wisc.edu # 16212882Sspwilson2@wisc.edu # Additionally, we've passed the flags in the following order: 16312882Sspwilson2@wisc.edu # --include-tags "ARM64" --exclude-tags ".*" --include-tags "X86" 16412882Sspwilson2@wisc.edu 16512882Sspwilson2@wisc.edu # Process --include-tags "ARM64" 16612882Sspwilson2@wisc.edu set(suite_ARM64) # Suite begins empty, but adds the ARM64 suite 16712882Sspwilson2@wisc.edu # Process --exclude-tags ".*" 16812882Sspwilson2@wisc.edu set() # Removed all suites which have tags 16912882Sspwilson2@wisc.edu # Process --include-tags "X86" 17012882Sspwilson2@wisc.edu set(suite_X86) 17112882Sspwilson2@wisc.edu ''' 17212882Sspwilson2@wisc.edu if not filters: 17312882Sspwilson2@wisc.edu return 17412882Sspwilson2@wisc.edu 17512882Sspwilson2@wisc.edu query_runner = query.QueryRunner(loaded_library) 17612882Sspwilson2@wisc.edu tags = query_runner.tags() 17712882Sspwilson2@wisc.edu 17812882Sspwilson2@wisc.edu if not filters[0].include: 17912882Sspwilson2@wisc.edu suites = set(query_runner.suites()) 18012882Sspwilson2@wisc.edu else: 18112882Sspwilson2@wisc.edu suites = set() 18212882Sspwilson2@wisc.edu 18312882Sspwilson2@wisc.edu def exclude(excludes): 18412882Sspwilson2@wisc.edu return suites - excludes 18512882Sspwilson2@wisc.edu def include(includes): 18612882Sspwilson2@wisc.edu return suites | includes 18712882Sspwilson2@wisc.edu 18812882Sspwilson2@wisc.edu for tag_regex in filters: 18912882Sspwilson2@wisc.edu matched_tags = (tag for tag in tags if tag_regex.regex.search(tag)) 19012882Sspwilson2@wisc.edu for tag in matched_tags: 19112882Sspwilson2@wisc.edu matched_suites = set(query_runner.suites_with_tag(tag)) 19212882Sspwilson2@wisc.edu suites = include(matched_suites) if tag_regex.include \ 19312882Sspwilson2@wisc.edu else exclude(matched_suites) 19412882Sspwilson2@wisc.edu 19512882Sspwilson2@wisc.edu # Set the library's suites to only those which where accepted by our filter 19612882Sspwilson2@wisc.edu loaded_library.suites = [suite for suite in loaded_library.suites 19712882Sspwilson2@wisc.edu if suite in suites] 19812882Sspwilson2@wisc.edu 19912882Sspwilson2@wisc.edu# TODO Add results command for listing previous results. 20012882Sspwilson2@wisc.edu 20112882Sspwilson2@wisc.edudef load_tests(): 20212882Sspwilson2@wisc.edu ''' 20312882Sspwilson2@wisc.edu Create a TestLoader and load tests for the directory given by the config. 20412882Sspwilson2@wisc.edu ''' 20512882Sspwilson2@wisc.edu testloader = loader_mod.Loader() 20612882Sspwilson2@wisc.edu log.test_log.message(terminal.separator()) 20712882Sspwilson2@wisc.edu log.test_log.message('Loading Tests', bold=True) 20812882Sspwilson2@wisc.edu testloader.load_root(config.config.directory) 20912882Sspwilson2@wisc.edu return testloader 21012882Sspwilson2@wisc.edu 21112882Sspwilson2@wisc.edudef do_list(): 21212882Sspwilson2@wisc.edu term_handler = handlers.TerminalHandler( 21312882Sspwilson2@wisc.edu verbosity=config.config.verbose+log.LogLevel.Info, 21412882Sspwilson2@wisc.edu machine_only=config.config.quiet 21512882Sspwilson2@wisc.edu ) 21612882Sspwilson2@wisc.edu log.test_log.log_obj.add_handler(term_handler) 21712882Sspwilson2@wisc.edu 21812882Sspwilson2@wisc.edu entry_message() 21912882Sspwilson2@wisc.edu 22012882Sspwilson2@wisc.edu test_schedule = load_tests().schedule 22112882Sspwilson2@wisc.edu filter_with_config_tags(test_schedule) 22212882Sspwilson2@wisc.edu 22312882Sspwilson2@wisc.edu qrunner = query.QueryRunner(test_schedule) 22412882Sspwilson2@wisc.edu 22512882Sspwilson2@wisc.edu if config.config.suites: 22612882Sspwilson2@wisc.edu qrunner.list_suites() 22712882Sspwilson2@wisc.edu elif config.config.tests: 22812882Sspwilson2@wisc.edu qrunner.list_tests() 22912882Sspwilson2@wisc.edu elif config.config.all_tags: 23012882Sspwilson2@wisc.edu qrunner.list_tags() 23112882Sspwilson2@wisc.edu else: 23212882Sspwilson2@wisc.edu qrunner.list_suites() 23312882Sspwilson2@wisc.edu qrunner.list_tests() 23412882Sspwilson2@wisc.edu qrunner.list_tags() 23512882Sspwilson2@wisc.edu 23613789Sjason@lowepower.com return 0 23713789Sjason@lowepower.com 23812882Sspwilson2@wisc.edudef run_schedule(test_schedule, log_handler): 23912882Sspwilson2@wisc.edu ''' 24012882Sspwilson2@wisc.edu Test Phases 24112882Sspwilson2@wisc.edu ----------- 24212882Sspwilson2@wisc.edu * Test Collection 24312882Sspwilson2@wisc.edu * Fixture Parameterization 24412882Sspwilson2@wisc.edu * Global Fixture Setup 24512882Sspwilson2@wisc.edu * Iteratevely run suites: 24612882Sspwilson2@wisc.edu * Suite Fixture Setup 24712882Sspwilson2@wisc.edu * Iteratively run tests: 24812882Sspwilson2@wisc.edu * Test Fixture Setup 24912882Sspwilson2@wisc.edu * Run Test 25012882Sspwilson2@wisc.edu * Test Fixture Teardown 25112882Sspwilson2@wisc.edu * Suite Fixture Teardown 25212882Sspwilson2@wisc.edu * Global Fixture Teardown 25312882Sspwilson2@wisc.edu ''' 25412882Sspwilson2@wisc.edu 25512882Sspwilson2@wisc.edu log_handler.schedule_finalized(test_schedule) 25612882Sspwilson2@wisc.edu 25712882Sspwilson2@wisc.edu # Iterate through all fixtures notifying them of the test schedule. 25812882Sspwilson2@wisc.edu for suite in test_schedule: 25912882Sspwilson2@wisc.edu copied_fixtures = [] 26012882Sspwilson2@wisc.edu for fixture in suite.fixtures: 26112882Sspwilson2@wisc.edu copied_fixtures.append(fixture.schedule_finalized(test_schedule)) 26212882Sspwilson2@wisc.edu suite.fixtures = copied_fixtures 26312882Sspwilson2@wisc.edu 26412882Sspwilson2@wisc.edu for test in suite: 26512882Sspwilson2@wisc.edu copied_fixtures = [] 26612882Sspwilson2@wisc.edu for fixture in test.fixtures: 26712882Sspwilson2@wisc.edu copied_fixtures.append(fixture.schedule_finalized( 26812882Sspwilson2@wisc.edu test_schedule)) 26912882Sspwilson2@wisc.edu test.fixtures = copied_fixtures 27012882Sspwilson2@wisc.edu 27112882Sspwilson2@wisc.edu log.test_log.message(terminal.separator()) 27212882Sspwilson2@wisc.edu log.test_log.message('Running Tests from {} suites' 27312882Sspwilson2@wisc.edu .format(len(test_schedule.suites)), bold=True) 27412882Sspwilson2@wisc.edu log.test_log.message("Results will be stored in {}".format( 27512882Sspwilson2@wisc.edu config.config.result_path)) 27612882Sspwilson2@wisc.edu log.test_log.message(terminal.separator()) 27712882Sspwilson2@wisc.edu 27812882Sspwilson2@wisc.edu # Build global fixtures and exectute scheduled test suites. 27912882Sspwilson2@wisc.edu if config.config.test_threads > 1: 28012882Sspwilson2@wisc.edu library_runner = runner.LibraryParallelRunner(test_schedule) 28112882Sspwilson2@wisc.edu library_runner.set_threads(config.config.test_threads) 28212882Sspwilson2@wisc.edu else: 28312882Sspwilson2@wisc.edu library_runner = runner.LibraryRunner(test_schedule) 28412882Sspwilson2@wisc.edu library_runner.run() 28512882Sspwilson2@wisc.edu 28613789Sjason@lowepower.com failed = log_handler.unsuccessful() 28713789Sjason@lowepower.com 28812882Sspwilson2@wisc.edu log_handler.finish_testing() 28912882Sspwilson2@wisc.edu 29013789Sjason@lowepower.com return 1 if failed else 0 29113789Sjason@lowepower.com 29212882Sspwilson2@wisc.edudef do_run(): 29312882Sspwilson2@wisc.edu # Initialize early parts of the log. 29412882Sspwilson2@wisc.edu with RunLogHandler() as log_handler: 29512882Sspwilson2@wisc.edu if config.config.uid: 29612882Sspwilson2@wisc.edu uid_ = uid.UID.from_uid(config.config.uid) 29712882Sspwilson2@wisc.edu if isinstance(uid_, uid.TestUID): 29812882Sspwilson2@wisc.edu log.test_log.error('Unable to run a standalone test.\n' 29912882Sspwilson2@wisc.edu 'Gem5 expects test suites to be the smallest unit ' 30012882Sspwilson2@wisc.edu ' of test.\n\n' 30112882Sspwilson2@wisc.edu 'Pass a SuiteUID instead.') 30212882Sspwilson2@wisc.edu return 30312882Sspwilson2@wisc.edu test_schedule = loader_mod.Loader().load_schedule_for_suites(uid_) 30412882Sspwilson2@wisc.edu if get_config_tags(): 30512882Sspwilson2@wisc.edu log.test_log.warn( 30612882Sspwilson2@wisc.edu "The '--uid' flag was supplied," 30712882Sspwilson2@wisc.edu " '--include-tags' and '--exclude-tags' will be ignored." 30812882Sspwilson2@wisc.edu ) 30912882Sspwilson2@wisc.edu else: 31012882Sspwilson2@wisc.edu test_schedule = load_tests().schedule 31112882Sspwilson2@wisc.edu # Filter tests based on tags 31212882Sspwilson2@wisc.edu filter_with_config_tags(test_schedule) 31312882Sspwilson2@wisc.edu # Execute the tests 31413789Sjason@lowepower.com return run_schedule(test_schedule, log_handler) 31512882Sspwilson2@wisc.edu 31612882Sspwilson2@wisc.edudef do_rerun(): 31712882Sspwilson2@wisc.edu # Init early parts of log 31812882Sspwilson2@wisc.edu with RunLogHandler() as log_handler: 31912882Sspwilson2@wisc.edu # Load previous results 32012882Sspwilson2@wisc.edu results = result.InternalSavedResults.load( 32112882Sspwilson2@wisc.edu os.path.join(config.config.result_path, 32212882Sspwilson2@wisc.edu config.constants.pickle_filename)) 32312882Sspwilson2@wisc.edu 32413789Sjason@lowepower.com rerun_suites = (suite.uid for suite in results if suite.unsuccessful) 32512882Sspwilson2@wisc.edu 32612882Sspwilson2@wisc.edu # Use loader to load suites 32712882Sspwilson2@wisc.edu loader = loader_mod.Loader() 32812882Sspwilson2@wisc.edu test_schedule = loader.load_schedule_for_suites(*rerun_suites) 32912882Sspwilson2@wisc.edu 33012882Sspwilson2@wisc.edu # Execute the tests 33113789Sjason@lowepower.com return run_schedule(test_schedule, log_handler) 33212882Sspwilson2@wisc.edu 33312882Sspwilson2@wisc.edudef main(): 33412882Sspwilson2@wisc.edu ''' 33512882Sspwilson2@wisc.edu Main entrypoint for the testlib test library. 33613789Sjason@lowepower.com Returns 0 on success and 1 otherwise so it can be used as a return code 33713789Sjason@lowepower.com for scripts. 33812882Sspwilson2@wisc.edu ''' 33912882Sspwilson2@wisc.edu config.initialize_config() 34012882Sspwilson2@wisc.edu 34112882Sspwilson2@wisc.edu # 'do' the given command. 34213789Sjason@lowepower.com result = globals()['do_'+config.config.command]() 34312882Sspwilson2@wisc.edu log.test_log.close() 34413789Sjason@lowepower.com 34513789Sjason@lowepower.com return result 346