112337Sjason@lowepower.com# Copyright (c) 2017 Mark D. Hill and David A. Wood 212337Sjason@lowepower.com# All rights reserved. 312337Sjason@lowepower.com# 412337Sjason@lowepower.com# Redistribution and use in source and binary forms, with or without 512337Sjason@lowepower.com# modification, are permitted provided that the following conditions are 612337Sjason@lowepower.com# met: redistributions of source code must retain the above copyright 712337Sjason@lowepower.com# notice, this list of conditions and the following disclaimer; 812337Sjason@lowepower.com# redistributions in binary form must reproduce the above copyright 912337Sjason@lowepower.com# notice, this list of conditions and the following disclaimer in the 1012337Sjason@lowepower.com# documentation and/or other materials provided with the distribution; 1112337Sjason@lowepower.com# neither the name of the copyright holders nor the names of its 1212337Sjason@lowepower.com# contributors may be used to endorse or promote products derived from 1312337Sjason@lowepower.com# this software without specific prior written permission. 1412337Sjason@lowepower.com# 1512337Sjason@lowepower.com# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS 1612337Sjason@lowepower.com# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT 1712337Sjason@lowepower.com# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR 1812337Sjason@lowepower.com# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT 1912337Sjason@lowepower.com# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, 2012337Sjason@lowepower.com# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT 2112337Sjason@lowepower.com# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, 2212337Sjason@lowepower.com# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY 2312337Sjason@lowepower.com# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT 2412337Sjason@lowepower.com# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE 2512337Sjason@lowepower.com# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 2612337Sjason@lowepower.com# 2712337Sjason@lowepower.com# Authors: Sean Wilson 2812337Sjason@lowepower.com 2912337Sjason@lowepower.comimport multiprocessing.dummy 3012337Sjason@lowepower.comimport threading 3112337Sjason@lowepower.comimport traceback 3212337Sjason@lowepower.com 3312337Sjason@lowepower.comimport helper 3412337Sjason@lowepower.comimport state 3512337Sjason@lowepower.comimport log 3612337Sjason@lowepower.comimport sandbox 3712337Sjason@lowepower.com 3812337Sjason@lowepower.comfrom state import Status, Result 3912564Sgabeblack@google.comfrom fixture import SkipException 4013774Sandreas.sandberg@arm.com 4112564Sgabeblack@google.comdef compute_aggregate_result(iterable): 4212337Sjason@lowepower.com ''' 4312337Sjason@lowepower.com Status of the test suite by default is: 4412337Sjason@lowepower.com * Passed if all contained tests passed 4512337Sjason@lowepower.com * Errored if any contained tests errored 4612337Sjason@lowepower.com * Failed if no tests errored, but one or more failed. 4712337Sjason@lowepower.com * Skipped if all contained tests were skipped 4812337Sjason@lowepower.com ''' 4912337Sjason@lowepower.com failed = [] 5012337Sjason@lowepower.com skipped = [] 5112337Sjason@lowepower.com for testitem in iterable: 5212337Sjason@lowepower.com result = testitem.result 5312337Sjason@lowepower.com 5412337Sjason@lowepower.com if result.value == Result.Errored: 5512337Sjason@lowepower.com return Result(result.value, result.reason) 5612337Sjason@lowepower.com elif result.value == Result.Failed: 5712564Sgabeblack@google.com failed.append(result.reason) 5812337Sjason@lowepower.com elif result.value == result.Skipped: 5912564Sgabeblack@google.com skipped.append(result.reason) 60 if failed: 61 return Result(Result.Failed, failed) 62 elif skipped: 63 return Result(Result.Skipped, skipped) 64 else: 65 return Result(Result.Passed) 66 67class TestParameters(object): 68 def __init__(self, test, suite): 69 self.test = test 70 self.suite = suite 71 self.log = log.TestLogWrapper(log.test_log, test, suite) 72 73 @helper.cacheresult 74 def _fixtures(self): 75 fixtures = {fixture.name:fixture for fixture in self.suite.fixtures} 76 for fixture in self.test.fixtures: 77 fixtures[fixture.name] = fixture 78 return fixtures 79 80 @property 81 def fixtures(self): 82 return self._fixtures() 83 84 85class RunnerPattern: 86 def __init__(self, loaded_testable): 87 self.testable = loaded_testable 88 self.builder = FixtureBuilder(self.testable.fixtures) 89 90 def handle_error(self, trace): 91 self.testable.result = Result(Result.Errored, trace) 92 self.avoid_children(trace) 93 94 def handle_skip(self, trace): 95 self.testable.result = Result(Result.Skipped, trace) 96 self.avoid_children(trace) 97 98 def avoid_children(self, reason): 99 for testable in self.testable: 100 testable.result = Result(self.testable.result.value, reason) 101 testable.status = Status.Avoided 102 103 def test(self): 104 pass 105 106 def run(self): 107 avoided = False 108 try: 109 self.testable.status = Status.Building 110 self.builder.setup(self.testable) 111 except SkipException: 112 self.handle_skip(traceback.format_exc()) 113 avoided = True 114 except BrokenFixtureException: 115 self.handle_error(traceback.format_exc()) 116 avoided = True 117 else: 118 self.testable.status = Status.Running 119 self.test() 120 finally: 121 self.testable.status = Status.TearingDown 122 self.builder.teardown(self.testable) 123 124 if avoided: 125 self.testable.status = Status.Avoided 126 else: 127 self.testable.status = Status.Complete 128 129class TestRunner(RunnerPattern): 130 def test(self): 131 self.sandbox_test() 132 133 def sandbox_test(self): 134 try: 135 sandbox.Sandbox(TestParameters( 136 self.testable, 137 self.testable.parent_suite)) 138 except sandbox.SubprocessException: 139 self.testable.result = Result(Result.Failed, 140 traceback.format_exc()) 141 else: 142 self.testable.result = Result(Result.Passed) 143 144 145class SuiteRunner(RunnerPattern): 146 def test(self): 147 for test in self.testable: 148 test.runner(test).run() 149 self.testable.result = compute_aggregate_result( 150 iter(self.testable)) 151 152 153class LibraryRunner(SuiteRunner): 154 pass 155 156 157class LibraryParallelRunner(RunnerPattern): 158 def set_threads(self, threads): 159 self.threads = threads 160 161 def _entrypoint(self, suite): 162 suite.runner(suite).run() 163 164 def test(self): 165 pool = multiprocessing.dummy.Pool(self.threads) 166 pool.map(lambda suite : suite.runner(suite).run(), self.testable) 167 self.testable.result = compute_aggregate_result( 168 iter(self.testable)) 169 170 171class BrokenFixtureException(Exception): 172 def __init__(self, fixture, testitem, trace): 173 self.fixture = fixture 174 self.testitem = testitem 175 self.trace = trace 176 177 self.msg = ('%s\n' 178 'Exception raised building "%s" raised SkipException' 179 ' for "%s".' % 180 (trace, fixture.name, testitem.name) 181 ) 182 super(BrokenFixtureException, self).__init__(self.msg) 183 184class FixtureBuilder(object): 185 def __init__(self, fixtures): 186 self.fixtures = fixtures 187 self.built_fixtures = [] 188 189 def setup(self, testitem): 190 for fixture in self.fixtures: 191 # Mark as built before, so if the build fails 192 # we still try to tear it down. 193 self.built_fixtures.append(fixture) 194 try: 195 fixture.setup(testitem) 196 except SkipException: 197 raise 198 except Exception as e: 199 exc = traceback.format_exc() 200 msg = 'Exception raised while setting up fixture for %s' %\ 201 testitem.uid 202 log.test_log.warn('%s\n%s' % (exc, msg)) 203 204 raise BrokenFixtureException(fixture, testitem, 205 traceback.format_exc()) 206 207 def teardown(self, testitem): 208 for fixture in self.built_fixtures: 209 try: 210 fixture.teardown(testitem) 211 except Exception: 212 # Log exception but keep cleaning up. 213 exc = traceback.format_exc() 214 msg = 'Exception raised while tearing down fixture for %s' %\ 215 testitem.uid 216 log.test_log.warn('%s\n%s' % (exc, msg)) 217