main.py revision 12882:dd87d7f2f3e5
1# Copyright (c) 2017 Mark D. Hill and David A. Wood 2# All rights reserved. 3# 4# Redistribution and use in source and binary forms, with or without 5# modification, are permitted provided that the following conditions are 6# met: redistributions of source code must retain the above copyright 7# notice, this list of conditions and the following disclaimer; 8# redistributions in binary form must reproduce the above copyright 9# notice, this list of conditions and the following disclaimer in the 10# documentation and/or other materials provided with the distribution; 11# neither the name of the copyright holders nor the names of its 12# contributors may be used to endorse or promote products derived from 13# this software without specific prior written permission. 14# 15# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS 16# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT 17# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR 18# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT 19# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, 20# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT 21# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, 22# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY 23# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT 24# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE 25# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 26# 27# Authors: Sean Wilson 28 29import os 30import itertools 31 32import config 33import fixture as fixture_mod 34import handlers 35import loader as loader_mod 36import log 37import query 38import result 39import runner 40import terminal 41import uid 42 43def entry_message(): 44 log.test_log.message("Running the new gem5 testing script.") 45 log.test_log.message("For more information see TESTING.md.") 46 log.test_log.message("To see details as the testing scripts are" 47 " running, use the option" 48 " -v, -vv, or -vvv") 49 50class RunLogHandler(): 51 def __init__(self): 52 term_handler = handlers.TerminalHandler( 53 verbosity=config.config.verbose+log.LogLevel.Info 54 ) 55 summary_handler = handlers.SummaryHandler() 56 self.mp_handler = handlers.MultiprocessingHandlerWrapper( 57 summary_handler, term_handler) 58 self.mp_handler.async_process() 59 log.test_log.log_obj.add_handler(self.mp_handler) 60 entry_message() 61 62 def schedule_finalized(self, test_schedule): 63 # Create the result handler object. 64 self.result_handler = handlers.ResultHandler( 65 test_schedule, config.config.result_path) 66 self.mp_handler.add_handler(self.result_handler) 67 68 def finish_testing(self): 69 self.result_handler.close() 70 71 def __enter__(self): 72 return self 73 74 def __exit__(self, *args): 75 self.close() 76 return False 77 78 def close(self): 79 self.mp_handler.close() 80 81def get_config_tags(): 82 return getattr(config.config, 83 config.StorePositionalTagsAction.position_kword) 84 85def filter_with_config_tags(loaded_library): 86 tags = get_config_tags() 87 final_tags = [] 88 regex_fmt = '^%s$' 89 cfg = config.config 90 91 def _append_inc_tag_filter(name): 92 if hasattr(cfg, name): 93 tag_opts = getattr(cfg, name) 94 for tag in tag_opts: 95 final_tags.append(config.TagRegex(True, regex_fmt % tag)) 96 97 def _append_rem_tag_filter(name): 98 if hasattr(cfg, name): 99 tag_opts = getattr(cfg, name) 100 for tag in cfg.constants.supported_tags[name]: 101 if tag not in tag_opts: 102 final_tags.append(config.TagRegex(False, regex_fmt % tag)) 103 104 # Append additional tags for the isa, length, and variant options. 105 # They apply last (they take priority) 106 special_tags = ( 107 cfg.constants.isa_tag_type, 108 cfg.constants.length_tag_type, 109 cfg.constants.variant_tag_type 110 ) 111 112 for tagname in special_tags: 113 _append_inc_tag_filter(tagname) 114 for tagname in special_tags: 115 _append_rem_tag_filter(tagname) 116 117 if tags is None: 118 tags = tuple() 119 120 filters = list(itertools.chain(tags, final_tags)) 121 string = 'Filtering suites with tags as follows:\n' 122 filter_string = '\t\n'.join((str(f) for f in filters)) 123 log.test_log.trace(string + filter_string) 124 125 return filter_with_tags(loaded_library, filters) 126 127 128def filter_with_tags(loaded_library, filters): 129 ''' 130 Filter logic supports two filter types: 131 --include-tags <regex> 132 --exclude-tags <regex> 133 134 The logic maintains a `set` of test suites. 135 136 If the regex provided with the `--include-tags` flag matches a tag of a 137 suite, that suite will added to the set. 138 139 If the regex provided with the `--exclude-tags` flag matches a tag of a 140 suite, that suite will removed to the set. 141 142 Suites can be added and removed multiple times. 143 144 First Flag Special Case Logic: 145 If include is the first flag, start with an empty set of suites. 146 If exclude is the first flag, start with the set of all collected suites. 147 148 149 Let's trace out the set as we go through the flags to clarify:: 150 151 # Say our collection of suites looks like this: set(suite_ARM64, 152 # suite_X86, suite_Other). 153 # 154 # Additionally, we've passed the flags in the following order: 155 # --include-tags "ARM64" --exclude-tags ".*" --include-tags "X86" 156 157 # Process --include-tags "ARM64" 158 set(suite_ARM64) # Suite begins empty, but adds the ARM64 suite 159 # Process --exclude-tags ".*" 160 set() # Removed all suites which have tags 161 # Process --include-tags "X86" 162 set(suite_X86) 163 ''' 164 if not filters: 165 return 166 167 query_runner = query.QueryRunner(loaded_library) 168 tags = query_runner.tags() 169 170 if not filters[0].include: 171 suites = set(query_runner.suites()) 172 else: 173 suites = set() 174 175 def exclude(excludes): 176 return suites - excludes 177 def include(includes): 178 return suites | includes 179 180 for tag_regex in filters: 181 matched_tags = (tag for tag in tags if tag_regex.regex.search(tag)) 182 for tag in matched_tags: 183 matched_suites = set(query_runner.suites_with_tag(tag)) 184 suites = include(matched_suites) if tag_regex.include \ 185 else exclude(matched_suites) 186 187 # Set the library's suites to only those which where accepted by our filter 188 loaded_library.suites = [suite for suite in loaded_library.suites 189 if suite in suites] 190 191# TODO Add results command for listing previous results. 192 193def load_tests(): 194 ''' 195 Create a TestLoader and load tests for the directory given by the config. 196 ''' 197 testloader = loader_mod.Loader() 198 log.test_log.message(terminal.separator()) 199 log.test_log.message('Loading Tests', bold=True) 200 testloader.load_root(config.config.directory) 201 return testloader 202 203def do_list(): 204 term_handler = handlers.TerminalHandler( 205 verbosity=config.config.verbose+log.LogLevel.Info, 206 machine_only=config.config.quiet 207 ) 208 log.test_log.log_obj.add_handler(term_handler) 209 210 entry_message() 211 212 test_schedule = load_tests().schedule 213 filter_with_config_tags(test_schedule) 214 215 qrunner = query.QueryRunner(test_schedule) 216 217 if config.config.suites: 218 qrunner.list_suites() 219 elif config.config.tests: 220 qrunner.list_tests() 221 elif config.config.all_tags: 222 qrunner.list_tags() 223 else: 224 qrunner.list_suites() 225 qrunner.list_tests() 226 qrunner.list_tags() 227 228def run_schedule(test_schedule, log_handler): 229 ''' 230 Test Phases 231 ----------- 232 * Test Collection 233 * Fixture Parameterization 234 * Global Fixture Setup 235 * Iteratevely run suites: 236 * Suite Fixture Setup 237 * Iteratively run tests: 238 * Test Fixture Setup 239 * Run Test 240 * Test Fixture Teardown 241 * Suite Fixture Teardown 242 * Global Fixture Teardown 243 ''' 244 245 log_handler.schedule_finalized(test_schedule) 246 247 # Iterate through all fixtures notifying them of the test schedule. 248 for suite in test_schedule: 249 copied_fixtures = [] 250 for fixture in suite.fixtures: 251 copied_fixtures.append(fixture.schedule_finalized(test_schedule)) 252 suite.fixtures = copied_fixtures 253 254 for test in suite: 255 copied_fixtures = [] 256 for fixture in test.fixtures: 257 copied_fixtures.append(fixture.schedule_finalized( 258 test_schedule)) 259 test.fixtures = copied_fixtures 260 261 log.test_log.message(terminal.separator()) 262 log.test_log.message('Running Tests from {} suites' 263 .format(len(test_schedule.suites)), bold=True) 264 log.test_log.message("Results will be stored in {}".format( 265 config.config.result_path)) 266 log.test_log.message(terminal.separator()) 267 268 # Build global fixtures and exectute scheduled test suites. 269 if config.config.test_threads > 1: 270 library_runner = runner.LibraryParallelRunner(test_schedule) 271 library_runner.set_threads(config.config.test_threads) 272 else: 273 library_runner = runner.LibraryRunner(test_schedule) 274 library_runner.run() 275 276 log_handler.finish_testing() 277 278def do_run(): 279 # Initialize early parts of the log. 280 with RunLogHandler() as log_handler: 281 if config.config.uid: 282 uid_ = uid.UID.from_uid(config.config.uid) 283 if isinstance(uid_, uid.TestUID): 284 log.test_log.error('Unable to run a standalone test.\n' 285 'Gem5 expects test suites to be the smallest unit ' 286 ' of test.\n\n' 287 'Pass a SuiteUID instead.') 288 return 289 test_schedule = loader_mod.Loader().load_schedule_for_suites(uid_) 290 if get_config_tags(): 291 log.test_log.warn( 292 "The '--uid' flag was supplied," 293 " '--include-tags' and '--exclude-tags' will be ignored." 294 ) 295 else: 296 test_schedule = load_tests().schedule 297 # Filter tests based on tags 298 filter_with_config_tags(test_schedule) 299 # Execute the tests 300 run_schedule(test_schedule, log_handler) 301 302 303def do_rerun(): 304 # Init early parts of log 305 with RunLogHandler() as log_handler: 306 # Load previous results 307 results = result.InternalSavedResults.load( 308 os.path.join(config.config.result_path, 309 config.constants.pickle_filename)) 310 311 rerun_suites = (suite.uid for suite in results if suite.unsucessful) 312 313 # Use loader to load suites 314 loader = loader_mod.Loader() 315 test_schedule = loader.load_schedule_for_suites(*rerun_suites) 316 317 # Execute the tests 318 run_schedule(test_schedule, log_handler) 319 320def main(): 321 ''' 322 Main entrypoint for the testlib test library. 323 ''' 324 config.initialize_config() 325 326 # 'do' the given command. 327 globals()['do_'+config.config.command]() 328 log.test_log.close() 329