runner.py revision 12882 
1# Copyright (c) 2017 Mark D. Hill and David A. Wood
2# All rights reserved.
3#
4# Redistribution and use in source and binary forms, with or without
5# modification, are permitted provided that the following conditions are
6# met: redistributions of source code must retain the above copyright
7# notice, this list of conditions and the following disclaimer;
8# redistributions in binary form must reproduce the above copyright
9# notice, this list of conditions and the following disclaimer in the
10# documentation and/or other materials provided with the distribution;
11# neither the name of the copyright holders nor the names of its
12# contributors may be used to endorse or promote products derived from
13# this software without specific prior written permission.
14#
15# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
16# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
17# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
18# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
19# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
20# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
21# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
22# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
23# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
24# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
25# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
26#
27# Authors: Sean Wilson
28
29import multiprocessing.dummy
30import threading
31import traceback
32
33import helper
34import state
35import log
36import sandbox
37
38from state import Status, Result
39from fixture import SkipException
40
41def compute_aggregate_result(iterable):
42    '''
43    Status of the test suite by default is:
44    * Passed if all contained tests passed
45    * Errored if any contained tests errored
46    * Failed if no tests errored, but one or more failed.
47    * Skipped if all contained tests were skipped
48    '''
49    failed = []
50    skipped = []
51    for testitem in iterable:
52        result = testitem.result
53
54        if result.value == Result.Errored:
55            return Result(result.value, result.reason)
56        elif result.value == Result.Failed:
57            failed.append(result.reason)
58        elif result.value == result.Skipped:
59            skipped.append(result.reason)
60    if failed:
61        return Result(Result.Failed, failed)
62    elif skipped:
63        return Result(Result.Skipped, skipped)
64    else:
65        return Result(Result.Passed)
66
67class TestParameters(object):
68    def __init__(self, test, suite):
69        self.test = test
70        self.suite = suite
71        self.log = log.TestLogWrapper(log.test_log, test, suite)
72
73    @helper.cacheresult
74    def _fixtures(self):
75        fixtures = {fixture.name:fixture for fixture in self.suite.fixtures}
76        for fixture in self.test.fixtures:
77            fixtures[fixture.name] = fixture
78        return fixtures
79
80    @property
81    def fixtures(self):
82        return self._fixtures()
83
84
85class RunnerPattern:
86    def __init__(self, loaded_testable):
87        self.testable = loaded_testable
88        self.builder = FixtureBuilder(self.testable.fixtures)
89
90    def handle_error(self, trace):
91        self.testable.result = Result(Result.Errored, trace)
92        self.avoid_children(trace)
93
94    def handle_skip(self, trace):
95        self.testable.result = Result(Result.Skipped, trace)
96        self.avoid_children(trace)
97
98    def avoid_children(self, reason):
99        for testable in self.testable:
100            testable.result = Result(self.testable.result.value, reason)
101            testable.status = Status.Avoided
102
103    def test(self):
104        pass
105
106    def run(self):
107        avoided = False
108        try:
109            self.testable.status = Status.Building
110            self.builder.setup(self.testable)
111        except SkipException:
112            self.handle_skip(traceback.format_exc())
113            avoided = True
114        except BrokenFixtureException:
115            self.handle_error(traceback.format_exc())
116            avoided = True
117        else:
118            self.testable.status = Status.Running
119            self.test()
120        finally:
121            self.testable.status = Status.TearingDown
122            self.builder.teardown(self.testable)
123
124        if avoided:
125            self.testable.status = Status.Avoided
126        else:
127            self.testable.status = Status.Complete
128
129class TestRunner(RunnerPattern):
130    def test(self):
131        self.sandbox_test()
132
133    def sandbox_test(self):
134        try:
135            sandbox.Sandbox(TestParameters(
136                    self.testable,
137                    self.testable.parent_suite))
138        except sandbox.SubprocessException:
139            self.testable.result = Result(Result.Failed,
140                    traceback.format_exc())
141        else:
142            self.testable.result = Result(Result.Passed)
143
144
145class SuiteRunner(RunnerPattern):
146    def test(self):
147        for test in self.testable:
148            test.runner(test).run()
149        self.testable.result = compute_aggregate_result(
150                iter(self.testable))
151
152
153class LibraryRunner(SuiteRunner):
154    pass
155
156
157class LibraryParallelRunner(RunnerPattern):
158    def set_threads(self, threads):
159        self.threads = threads
160
161    def _entrypoint(self, suite):
162        suite.runner(suite).run()
163
164    def test(self):
165        pool = multiprocessing.dummy.Pool(self.threads)
166        pool.map(lambda suite : suite.runner(suite).run(), self.testable)
167        self.testable.result = compute_aggregate_result(
168                iter(self.testable))
169
170
171class BrokenFixtureException(Exception):
172    def __init__(self, fixture, testitem, trace):
173        self.fixture = fixture
174        self.testitem = testitem
175        self.trace = trace
176
177        self.msg = ('%s\n'
178                   'Exception raised building "%s" raised SkipException'
179                   ' for "%s".' %
180                   (trace, fixture.name, testitem.name)
181        )
182        super(BrokenFixtureException, self).__init__(self.msg)
183
184class FixtureBuilder(object):
185    def __init__(self, fixtures):
186        self.fixtures = fixtures
187        self.built_fixtures = []
188
189    def setup(self, testitem):
190        for fixture in self.fixtures:
191            # Mark as built before, so if the build fails
192            # we still try to tear it down.
193            self.built_fixtures.append(fixture)
194            try:
195                fixture.setup(testitem)
196            except SkipException:
197                raise
198            except Exception as e:
199                exc = traceback.format_exc()
200                msg = 'Exception raised while setting up fixture for %s' %\
201                        testitem.uid
202                log.test_log.warn('%s\n%s' % (exc, msg))
203
204                raise BrokenFixtureException(fixture, testitem,
205                        traceback.format_exc())
206
207    def teardown(self, testitem):
208        for fixture in self.built_fixtures:
209            try:
210                fixture.teardown(testitem)
211            except Exception:
212                # Log exception but keep cleaning up.
213                exc = traceback.format_exc()
214                msg = 'Exception raised while tearing down fixture for %s' %\
215                        testitem.uid
216                log.test_log.warn('%s\n%s' % (exc, msg))
217