main.py revision 12882
112882Sspwilson2@wisc.edu# Copyright (c) 2017 Mark D. Hill and David A. Wood 212882Sspwilson2@wisc.edu# All rights reserved. 312882Sspwilson2@wisc.edu# 412882Sspwilson2@wisc.edu# Redistribution and use in source and binary forms, with or without 512882Sspwilson2@wisc.edu# modification, are permitted provided that the following conditions are 612882Sspwilson2@wisc.edu# met: redistributions of source code must retain the above copyright 712882Sspwilson2@wisc.edu# notice, this list of conditions and the following disclaimer; 812882Sspwilson2@wisc.edu# redistributions in binary form must reproduce the above copyright 912882Sspwilson2@wisc.edu# notice, this list of conditions and the following disclaimer in the 1012882Sspwilson2@wisc.edu# documentation and/or other materials provided with the distribution; 1112882Sspwilson2@wisc.edu# neither the name of the copyright holders nor the names of its 1212882Sspwilson2@wisc.edu# contributors may be used to endorse or promote products derived from 1312882Sspwilson2@wisc.edu# this software without specific prior written permission. 1412882Sspwilson2@wisc.edu# 1512882Sspwilson2@wisc.edu# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS 1612882Sspwilson2@wisc.edu# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT 1712882Sspwilson2@wisc.edu# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR 1812882Sspwilson2@wisc.edu# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT 1912882Sspwilson2@wisc.edu# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, 2012882Sspwilson2@wisc.edu# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT 2112882Sspwilson2@wisc.edu# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, 2212882Sspwilson2@wisc.edu# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY 2312882Sspwilson2@wisc.edu# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT 2412882Sspwilson2@wisc.edu# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE 2512882Sspwilson2@wisc.edu# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 2612882Sspwilson2@wisc.edu# 2712882Sspwilson2@wisc.edu# Authors: Sean Wilson 2812882Sspwilson2@wisc.edu 2912882Sspwilson2@wisc.eduimport os 3012882Sspwilson2@wisc.eduimport itertools 3112882Sspwilson2@wisc.edu 3212882Sspwilson2@wisc.eduimport config 3312882Sspwilson2@wisc.eduimport fixture as fixture_mod 3412882Sspwilson2@wisc.eduimport handlers 3512882Sspwilson2@wisc.eduimport loader as loader_mod 3612882Sspwilson2@wisc.eduimport log 3712882Sspwilson2@wisc.eduimport query 3812882Sspwilson2@wisc.eduimport result 3912882Sspwilson2@wisc.eduimport runner 4012882Sspwilson2@wisc.eduimport terminal 4112882Sspwilson2@wisc.eduimport uid 4212882Sspwilson2@wisc.edu 4312882Sspwilson2@wisc.edudef entry_message(): 4412882Sspwilson2@wisc.edu log.test_log.message("Running the new gem5 testing script.") 4512882Sspwilson2@wisc.edu log.test_log.message("For more information see TESTING.md.") 4612882Sspwilson2@wisc.edu log.test_log.message("To see details as the testing scripts are" 4712882Sspwilson2@wisc.edu " running, use the option" 4812882Sspwilson2@wisc.edu " -v, -vv, or -vvv") 4912882Sspwilson2@wisc.edu 5012882Sspwilson2@wisc.educlass RunLogHandler(): 5112882Sspwilson2@wisc.edu def __init__(self): 5212882Sspwilson2@wisc.edu term_handler = handlers.TerminalHandler( 5312882Sspwilson2@wisc.edu verbosity=config.config.verbose+log.LogLevel.Info 5412882Sspwilson2@wisc.edu ) 5512882Sspwilson2@wisc.edu summary_handler = handlers.SummaryHandler() 5612882Sspwilson2@wisc.edu self.mp_handler = handlers.MultiprocessingHandlerWrapper( 5712882Sspwilson2@wisc.edu summary_handler, term_handler) 5812882Sspwilson2@wisc.edu self.mp_handler.async_process() 5912882Sspwilson2@wisc.edu log.test_log.log_obj.add_handler(self.mp_handler) 6012882Sspwilson2@wisc.edu entry_message() 6112882Sspwilson2@wisc.edu 6212882Sspwilson2@wisc.edu def schedule_finalized(self, test_schedule): 6312882Sspwilson2@wisc.edu # Create the result handler object. 6412882Sspwilson2@wisc.edu self.result_handler = handlers.ResultHandler( 6512882Sspwilson2@wisc.edu test_schedule, config.config.result_path) 6612882Sspwilson2@wisc.edu self.mp_handler.add_handler(self.result_handler) 6712882Sspwilson2@wisc.edu 6812882Sspwilson2@wisc.edu def finish_testing(self): 6912882Sspwilson2@wisc.edu self.result_handler.close() 7012882Sspwilson2@wisc.edu 7112882Sspwilson2@wisc.edu def __enter__(self): 7212882Sspwilson2@wisc.edu return self 7312882Sspwilson2@wisc.edu 7412882Sspwilson2@wisc.edu def __exit__(self, *args): 7512882Sspwilson2@wisc.edu self.close() 7612882Sspwilson2@wisc.edu return False 7712882Sspwilson2@wisc.edu 7812882Sspwilson2@wisc.edu def close(self): 7912882Sspwilson2@wisc.edu self.mp_handler.close() 8012882Sspwilson2@wisc.edu 8112882Sspwilson2@wisc.edudef get_config_tags(): 8212882Sspwilson2@wisc.edu return getattr(config.config, 8312882Sspwilson2@wisc.edu config.StorePositionalTagsAction.position_kword) 8412882Sspwilson2@wisc.edu 8512882Sspwilson2@wisc.edudef filter_with_config_tags(loaded_library): 8612882Sspwilson2@wisc.edu tags = get_config_tags() 8712882Sspwilson2@wisc.edu final_tags = [] 8812882Sspwilson2@wisc.edu regex_fmt = '^%s$' 8912882Sspwilson2@wisc.edu cfg = config.config 9012882Sspwilson2@wisc.edu 9112882Sspwilson2@wisc.edu def _append_inc_tag_filter(name): 9212882Sspwilson2@wisc.edu if hasattr(cfg, name): 9312882Sspwilson2@wisc.edu tag_opts = getattr(cfg, name) 9412882Sspwilson2@wisc.edu for tag in tag_opts: 9512882Sspwilson2@wisc.edu final_tags.append(config.TagRegex(True, regex_fmt % tag)) 9612882Sspwilson2@wisc.edu 9712882Sspwilson2@wisc.edu def _append_rem_tag_filter(name): 9812882Sspwilson2@wisc.edu if hasattr(cfg, name): 9912882Sspwilson2@wisc.edu tag_opts = getattr(cfg, name) 10012882Sspwilson2@wisc.edu for tag in cfg.constants.supported_tags[name]: 10112882Sspwilson2@wisc.edu if tag not in tag_opts: 10212882Sspwilson2@wisc.edu final_tags.append(config.TagRegex(False, regex_fmt % tag)) 10312882Sspwilson2@wisc.edu 10412882Sspwilson2@wisc.edu # Append additional tags for the isa, length, and variant options. 10512882Sspwilson2@wisc.edu # They apply last (they take priority) 10612882Sspwilson2@wisc.edu special_tags = ( 10712882Sspwilson2@wisc.edu cfg.constants.isa_tag_type, 10812882Sspwilson2@wisc.edu cfg.constants.length_tag_type, 10912882Sspwilson2@wisc.edu cfg.constants.variant_tag_type 11012882Sspwilson2@wisc.edu ) 11112882Sspwilson2@wisc.edu 11212882Sspwilson2@wisc.edu for tagname in special_tags: 11312882Sspwilson2@wisc.edu _append_inc_tag_filter(tagname) 11412882Sspwilson2@wisc.edu for tagname in special_tags: 11512882Sspwilson2@wisc.edu _append_rem_tag_filter(tagname) 11612882Sspwilson2@wisc.edu 11712882Sspwilson2@wisc.edu if tags is None: 11812882Sspwilson2@wisc.edu tags = tuple() 11912882Sspwilson2@wisc.edu 12012882Sspwilson2@wisc.edu filters = list(itertools.chain(tags, final_tags)) 12112882Sspwilson2@wisc.edu string = 'Filtering suites with tags as follows:\n' 12212882Sspwilson2@wisc.edu filter_string = '\t\n'.join((str(f) for f in filters)) 12312882Sspwilson2@wisc.edu log.test_log.trace(string + filter_string) 12412882Sspwilson2@wisc.edu 12512882Sspwilson2@wisc.edu return filter_with_tags(loaded_library, filters) 12612882Sspwilson2@wisc.edu 12712882Sspwilson2@wisc.edu 12812882Sspwilson2@wisc.edudef filter_with_tags(loaded_library, filters): 12912882Sspwilson2@wisc.edu ''' 13012882Sspwilson2@wisc.edu Filter logic supports two filter types: 13112882Sspwilson2@wisc.edu --include-tags <regex> 13212882Sspwilson2@wisc.edu --exclude-tags <regex> 13312882Sspwilson2@wisc.edu 13412882Sspwilson2@wisc.edu The logic maintains a `set` of test suites. 13512882Sspwilson2@wisc.edu 13612882Sspwilson2@wisc.edu If the regex provided with the `--include-tags` flag matches a tag of a 13712882Sspwilson2@wisc.edu suite, that suite will added to the set. 13812882Sspwilson2@wisc.edu 13912882Sspwilson2@wisc.edu If the regex provided with the `--exclude-tags` flag matches a tag of a 14012882Sspwilson2@wisc.edu suite, that suite will removed to the set. 14112882Sspwilson2@wisc.edu 14212882Sspwilson2@wisc.edu Suites can be added and removed multiple times. 14312882Sspwilson2@wisc.edu 14412882Sspwilson2@wisc.edu First Flag Special Case Logic: 14512882Sspwilson2@wisc.edu If include is the first flag, start with an empty set of suites. 14612882Sspwilson2@wisc.edu If exclude is the first flag, start with the set of all collected suites. 14712882Sspwilson2@wisc.edu 14812882Sspwilson2@wisc.edu 14912882Sspwilson2@wisc.edu Let's trace out the set as we go through the flags to clarify:: 15012882Sspwilson2@wisc.edu 15112882Sspwilson2@wisc.edu # Say our collection of suites looks like this: set(suite_ARM64, 15212882Sspwilson2@wisc.edu # suite_X86, suite_Other). 15312882Sspwilson2@wisc.edu # 15412882Sspwilson2@wisc.edu # Additionally, we've passed the flags in the following order: 15512882Sspwilson2@wisc.edu # --include-tags "ARM64" --exclude-tags ".*" --include-tags "X86" 15612882Sspwilson2@wisc.edu 15712882Sspwilson2@wisc.edu # Process --include-tags "ARM64" 15812882Sspwilson2@wisc.edu set(suite_ARM64) # Suite begins empty, but adds the ARM64 suite 15912882Sspwilson2@wisc.edu # Process --exclude-tags ".*" 16012882Sspwilson2@wisc.edu set() # Removed all suites which have tags 16112882Sspwilson2@wisc.edu # Process --include-tags "X86" 16212882Sspwilson2@wisc.edu set(suite_X86) 16312882Sspwilson2@wisc.edu ''' 16412882Sspwilson2@wisc.edu if not filters: 16512882Sspwilson2@wisc.edu return 16612882Sspwilson2@wisc.edu 16712882Sspwilson2@wisc.edu query_runner = query.QueryRunner(loaded_library) 16812882Sspwilson2@wisc.edu tags = query_runner.tags() 16912882Sspwilson2@wisc.edu 17012882Sspwilson2@wisc.edu if not filters[0].include: 17112882Sspwilson2@wisc.edu suites = set(query_runner.suites()) 17212882Sspwilson2@wisc.edu else: 17312882Sspwilson2@wisc.edu suites = set() 17412882Sspwilson2@wisc.edu 17512882Sspwilson2@wisc.edu def exclude(excludes): 17612882Sspwilson2@wisc.edu return suites - excludes 17712882Sspwilson2@wisc.edu def include(includes): 17812882Sspwilson2@wisc.edu return suites | includes 17912882Sspwilson2@wisc.edu 18012882Sspwilson2@wisc.edu for tag_regex in filters: 18112882Sspwilson2@wisc.edu matched_tags = (tag for tag in tags if tag_regex.regex.search(tag)) 18212882Sspwilson2@wisc.edu for tag in matched_tags: 18312882Sspwilson2@wisc.edu matched_suites = set(query_runner.suites_with_tag(tag)) 18412882Sspwilson2@wisc.edu suites = include(matched_suites) if tag_regex.include \ 18512882Sspwilson2@wisc.edu else exclude(matched_suites) 18612882Sspwilson2@wisc.edu 18712882Sspwilson2@wisc.edu # Set the library's suites to only those which where accepted by our filter 18812882Sspwilson2@wisc.edu loaded_library.suites = [suite for suite in loaded_library.suites 18912882Sspwilson2@wisc.edu if suite in suites] 19012882Sspwilson2@wisc.edu 19112882Sspwilson2@wisc.edu# TODO Add results command for listing previous results. 19212882Sspwilson2@wisc.edu 19312882Sspwilson2@wisc.edudef load_tests(): 19412882Sspwilson2@wisc.edu ''' 19512882Sspwilson2@wisc.edu Create a TestLoader and load tests for the directory given by the config. 19612882Sspwilson2@wisc.edu ''' 19712882Sspwilson2@wisc.edu testloader = loader_mod.Loader() 19812882Sspwilson2@wisc.edu log.test_log.message(terminal.separator()) 19912882Sspwilson2@wisc.edu log.test_log.message('Loading Tests', bold=True) 20012882Sspwilson2@wisc.edu testloader.load_root(config.config.directory) 20112882Sspwilson2@wisc.edu return testloader 20212882Sspwilson2@wisc.edu 20312882Sspwilson2@wisc.edudef do_list(): 20412882Sspwilson2@wisc.edu term_handler = handlers.TerminalHandler( 20512882Sspwilson2@wisc.edu verbosity=config.config.verbose+log.LogLevel.Info, 20612882Sspwilson2@wisc.edu machine_only=config.config.quiet 20712882Sspwilson2@wisc.edu ) 20812882Sspwilson2@wisc.edu log.test_log.log_obj.add_handler(term_handler) 20912882Sspwilson2@wisc.edu 21012882Sspwilson2@wisc.edu entry_message() 21112882Sspwilson2@wisc.edu 21212882Sspwilson2@wisc.edu test_schedule = load_tests().schedule 21312882Sspwilson2@wisc.edu filter_with_config_tags(test_schedule) 21412882Sspwilson2@wisc.edu 21512882Sspwilson2@wisc.edu qrunner = query.QueryRunner(test_schedule) 21612882Sspwilson2@wisc.edu 21712882Sspwilson2@wisc.edu if config.config.suites: 21812882Sspwilson2@wisc.edu qrunner.list_suites() 21912882Sspwilson2@wisc.edu elif config.config.tests: 22012882Sspwilson2@wisc.edu qrunner.list_tests() 22112882Sspwilson2@wisc.edu elif config.config.all_tags: 22212882Sspwilson2@wisc.edu qrunner.list_tags() 22312882Sspwilson2@wisc.edu else: 22412882Sspwilson2@wisc.edu qrunner.list_suites() 22512882Sspwilson2@wisc.edu qrunner.list_tests() 22612882Sspwilson2@wisc.edu qrunner.list_tags() 22712882Sspwilson2@wisc.edu 22812882Sspwilson2@wisc.edudef run_schedule(test_schedule, log_handler): 22912882Sspwilson2@wisc.edu ''' 23012882Sspwilson2@wisc.edu Test Phases 23112882Sspwilson2@wisc.edu ----------- 23212882Sspwilson2@wisc.edu * Test Collection 23312882Sspwilson2@wisc.edu * Fixture Parameterization 23412882Sspwilson2@wisc.edu * Global Fixture Setup 23512882Sspwilson2@wisc.edu * Iteratevely run suites: 23612882Sspwilson2@wisc.edu * Suite Fixture Setup 23712882Sspwilson2@wisc.edu * Iteratively run tests: 23812882Sspwilson2@wisc.edu * Test Fixture Setup 23912882Sspwilson2@wisc.edu * Run Test 24012882Sspwilson2@wisc.edu * Test Fixture Teardown 24112882Sspwilson2@wisc.edu * Suite Fixture Teardown 24212882Sspwilson2@wisc.edu * Global Fixture Teardown 24312882Sspwilson2@wisc.edu ''' 24412882Sspwilson2@wisc.edu 24512882Sspwilson2@wisc.edu log_handler.schedule_finalized(test_schedule) 24612882Sspwilson2@wisc.edu 24712882Sspwilson2@wisc.edu # Iterate through all fixtures notifying them of the test schedule. 24812882Sspwilson2@wisc.edu for suite in test_schedule: 24912882Sspwilson2@wisc.edu copied_fixtures = [] 25012882Sspwilson2@wisc.edu for fixture in suite.fixtures: 25112882Sspwilson2@wisc.edu copied_fixtures.append(fixture.schedule_finalized(test_schedule)) 25212882Sspwilson2@wisc.edu suite.fixtures = copied_fixtures 25312882Sspwilson2@wisc.edu 25412882Sspwilson2@wisc.edu for test in suite: 25512882Sspwilson2@wisc.edu copied_fixtures = [] 25612882Sspwilson2@wisc.edu for fixture in test.fixtures: 25712882Sspwilson2@wisc.edu copied_fixtures.append(fixture.schedule_finalized( 25812882Sspwilson2@wisc.edu test_schedule)) 25912882Sspwilson2@wisc.edu test.fixtures = copied_fixtures 26012882Sspwilson2@wisc.edu 26112882Sspwilson2@wisc.edu log.test_log.message(terminal.separator()) 26212882Sspwilson2@wisc.edu log.test_log.message('Running Tests from {} suites' 26312882Sspwilson2@wisc.edu .format(len(test_schedule.suites)), bold=True) 26412882Sspwilson2@wisc.edu log.test_log.message("Results will be stored in {}".format( 26512882Sspwilson2@wisc.edu config.config.result_path)) 26612882Sspwilson2@wisc.edu log.test_log.message(terminal.separator()) 26712882Sspwilson2@wisc.edu 26812882Sspwilson2@wisc.edu # Build global fixtures and exectute scheduled test suites. 26912882Sspwilson2@wisc.edu if config.config.test_threads > 1: 27012882Sspwilson2@wisc.edu library_runner = runner.LibraryParallelRunner(test_schedule) 27112882Sspwilson2@wisc.edu library_runner.set_threads(config.config.test_threads) 27212882Sspwilson2@wisc.edu else: 27312882Sspwilson2@wisc.edu library_runner = runner.LibraryRunner(test_schedule) 27412882Sspwilson2@wisc.edu library_runner.run() 27512882Sspwilson2@wisc.edu 27612882Sspwilson2@wisc.edu log_handler.finish_testing() 27712882Sspwilson2@wisc.edu 27812882Sspwilson2@wisc.edudef do_run(): 27912882Sspwilson2@wisc.edu # Initialize early parts of the log. 28012882Sspwilson2@wisc.edu with RunLogHandler() as log_handler: 28112882Sspwilson2@wisc.edu if config.config.uid: 28212882Sspwilson2@wisc.edu uid_ = uid.UID.from_uid(config.config.uid) 28312882Sspwilson2@wisc.edu if isinstance(uid_, uid.TestUID): 28412882Sspwilson2@wisc.edu log.test_log.error('Unable to run a standalone test.\n' 28512882Sspwilson2@wisc.edu 'Gem5 expects test suites to be the smallest unit ' 28612882Sspwilson2@wisc.edu ' of test.\n\n' 28712882Sspwilson2@wisc.edu 'Pass a SuiteUID instead.') 28812882Sspwilson2@wisc.edu return 28912882Sspwilson2@wisc.edu test_schedule = loader_mod.Loader().load_schedule_for_suites(uid_) 29012882Sspwilson2@wisc.edu if get_config_tags(): 29112882Sspwilson2@wisc.edu log.test_log.warn( 29212882Sspwilson2@wisc.edu "The '--uid' flag was supplied," 29312882Sspwilson2@wisc.edu " '--include-tags' and '--exclude-tags' will be ignored." 29412882Sspwilson2@wisc.edu ) 29512882Sspwilson2@wisc.edu else: 29612882Sspwilson2@wisc.edu test_schedule = load_tests().schedule 29712882Sspwilson2@wisc.edu # Filter tests based on tags 29812882Sspwilson2@wisc.edu filter_with_config_tags(test_schedule) 29912882Sspwilson2@wisc.edu # Execute the tests 30012882Sspwilson2@wisc.edu run_schedule(test_schedule, log_handler) 30112882Sspwilson2@wisc.edu 30212882Sspwilson2@wisc.edu 30312882Sspwilson2@wisc.edudef do_rerun(): 30412882Sspwilson2@wisc.edu # Init early parts of log 30512882Sspwilson2@wisc.edu with RunLogHandler() as log_handler: 30612882Sspwilson2@wisc.edu # Load previous results 30712882Sspwilson2@wisc.edu results = result.InternalSavedResults.load( 30812882Sspwilson2@wisc.edu os.path.join(config.config.result_path, 30912882Sspwilson2@wisc.edu config.constants.pickle_filename)) 31012882Sspwilson2@wisc.edu 31112882Sspwilson2@wisc.edu rerun_suites = (suite.uid for suite in results if suite.unsucessful) 31212882Sspwilson2@wisc.edu 31312882Sspwilson2@wisc.edu # Use loader to load suites 31412882Sspwilson2@wisc.edu loader = loader_mod.Loader() 31512882Sspwilson2@wisc.edu test_schedule = loader.load_schedule_for_suites(*rerun_suites) 31612882Sspwilson2@wisc.edu 31712882Sspwilson2@wisc.edu # Execute the tests 31812882Sspwilson2@wisc.edu run_schedule(test_schedule, log_handler) 31912882Sspwilson2@wisc.edu 32012882Sspwilson2@wisc.edudef main(): 32112882Sspwilson2@wisc.edu ''' 32212882Sspwilson2@wisc.edu Main entrypoint for the testlib test library. 32312882Sspwilson2@wisc.edu ''' 32412882Sspwilson2@wisc.edu config.initialize_config() 32512882Sspwilson2@wisc.edu 32612882Sspwilson2@wisc.edu # 'do' the given command. 32712882Sspwilson2@wisc.edu globals()['do_'+config.config.command]() 32812882Sspwilson2@wisc.edu log.test_log.close() 329