loader.py revision 12882:dd87d7f2f3e5 
1# Copyright (c) 2017 Mark D. Hill and David A. Wood
2# All rights reserved.
3#
4# Redistribution and use in source and binary forms, with or without
5# modification, are permitted provided that the following conditions are
6# met: redistributions of source code must retain the above copyright
7# notice, this list of conditions and the following disclaimer;
8# redistributions in binary form must reproduce the above copyright
9# notice, this list of conditions and the following disclaimer in the
10# documentation and/or other materials provided with the distribution;
11# neither the name of the copyright holders nor the names of its
12# contributors may be used to endorse or promote products derived from
13# this software without specific prior written permission.
14#
15# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
16# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
17# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
18# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
19# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
20# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
21# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
22# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
23# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
24# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
25# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
26#
27# Authors: Sean Wilson
28
29'''
30Contains the :class:`Loader` which is responsible for discovering and loading
31tests.
32
33Loading typically follows the following stages.
34
351. Recurse down a given directory looking for tests which match a given regex.
36
37    The default regex used will match any python file (ending in .py) that has
38    a name starting or ending in test(s). If there are any additional
39    components of the name they must be connected with '-' or '_'. Lastly,
40    file names that begin with '.' will be ignored.
41
42    The following names would match:
43
44    - `tests.py`
45    - `test.py`
46    - `test-this.py`
47    - `tests-that.py`
48    - `these-test.py`
49
50    These would not match:
51
52    - `.test.py`    - 'hidden' files are ignored.
53    - `test`        - Must end in '.py'
54    - `test-.py`    - Needs a character after the hypen.
55    - `testthis.py` - Needs a hypen or underscore to separate 'test' and 'this'
56
57
582. With all files discovered execute each file gathering its test items we
59   care about collecting. (`TestCase`, `TestSuite` and `Fixture` objects.)
60
61As a final note, :class:`TestCase` instances which are not put into
62a :class:`TestSuite` by the test writer will be placed into
63a :class:`TestSuite` named after the module.
64
65.. seealso:: :func:`load_file`
66'''
67
68import os
69import re
70import sys
71import traceback
72
73import config
74import log
75import suite as suite_mod
76import test as test_mod
77import fixture as fixture_mod
78import wrappers
79import uid
80
81class DuplicateTestItemException(Exception):
82    '''
83    Exception indicates multiple test items with the same UID
84    were discovered.
85    '''
86    pass
87
88
89# Match filenames that either begin or end with 'test' or tests and use
90# - or _ to separate additional name components.
91default_filepath_regex = re.compile(
92            r'(((.+[_])?tests?)|(tests?([-_].+)?))\.py$')
93
94def default_filepath_filter(filepath):
95    '''The default filter applied to filepaths to marks as test sources.'''
96    filepath = os.path.basename(filepath)
97    if default_filepath_regex.match(filepath):
98        # Make sure doesn't start with .
99        return not filepath.startswith('.')
100    return False
101
102def path_as_modulename(filepath):
103    '''Return the given filepath as a module name.'''
104    # Remove the file extention (.py)
105    return os.path.splitext(os.path.basename(filepath))[0]
106
107def path_as_suitename(filepath):
108    return os.path.split(os.path.dirname(os.path.abspath((filepath))))[-1]
109
110def _assert_files_in_same_dir(files):
111    if __debug__:
112        if files:
113            directory = os.path.dirname(files[0])
114            for f in files:
115                assert os.path.dirname(f) == directory
116
117class Loader(object):
118    '''
119    Class for discovering tests.
120
121    Discovered :class:`TestCase` and :class:`TestSuite` objects are wrapped by
122    :class:`LoadedTest` and :class:`LoadedSuite` objects respectively.
123    These objects provided additional methods and metadata about the loaded
124    objects and are the internal representation used by testlib.
125
126    To simply discover and load all tests using the default filter create an
127    instance and `load_root`.
128
129    >>> import os
130    >>> tl = Loader()
131    >>> tl.load_root(os.getcwd())
132
133    .. note:: If tests are not contained in a TestSuite, they will
134        automatically be placed into one for the module.
135
136    .. warn:: This class is extremely thread-unsafe.
137       It modifies the sys path and global config.
138       Use with care.
139    '''
140    def __init__(self):
141        self.suites = []
142        self.suite_uids = {}
143        self.filepath_filter = default_filepath_filter
144
145        # filepath -> Successful | Failed to load
146        self._files = {}
147
148    @property
149    def schedule(self):
150        return wrappers.LoadedLibrary(self.suites, fixture_mod.global_fixtures)
151
152    def load_schedule_for_suites(self, *uids):
153        files = {uid.UID.uid_to_path(id_) for id_ in uids}
154        for file_ in files:
155            self.load_file(file_)
156
157        return wrappers.LoadedLibrary(
158                [self.suite_uids[id_] for id_ in uids],
159                fixture_mod.global_fixtures)
160
161    def _verify_no_duplicate_suites(self, new_suites):
162        new_suite_uids = self.suite_uids.copy()
163        for suite in new_suites:
164            if suite.uid in new_suite_uids:
165                raise DuplicateTestItemException(
166                        "More than one suite with UID '%s' was defined" %\
167                                suite.uid)
168            new_suite_uids[suite.uid] = suite
169
170    def _verify_no_duplicate_tests_in_suites(self, new_suites):
171        for suite in new_suites:
172            test_uids = set()
173            for test in suite:
174                if test.uid in test_uids:
175                     raise DuplicateTestItemException(
176                            "More than one test with UID '%s' was defined"
177                            " in suite '%s'"
178                            % (test.uid, suite.uid))
179                test_uids.add(test.uid)
180
181    def load_root(self, root):
182        '''
183        Load files from the given root directory which match
184        `self.filepath_filter`.
185        '''
186        if __debug__:
187            self._loaded_a_file = True
188
189        for directory in self._discover_files(root):
190            if directory:
191                _assert_files_in_same_dir(directory)
192                for f in directory:
193                    self.load_file(f)
194
195    def load_dir(self, directory):
196        for dir_ in self._discover_files(directory):
197            _assert_files_in_same_dir(dir_)
198            for f in dir_:
199                self.load_file(f)
200
201    def load_file(self, path):
202        path = os.path.abspath(path)
203
204        if path in self._files:
205            if not self._files[path]:
206                raise Exception('Attempted to load a file which already'
207                        ' failed to load')
208            else:
209                log.test_log.debug('Tried to reload: %s' % path)
210                return
211
212        # Create a custom dictionary for the loaded module.
213        newdict = {
214            '__builtins__':__builtins__,
215            '__name__': path_as_modulename(path),
216            '__file__': path,
217        }
218
219        # Add the file's containing directory to the system path. So it can do
220        # relative imports naturally.
221        old_path = sys.path[:]
222        sys.path.insert(0, os.path.dirname(path))
223        cwd = os.getcwd()
224        os.chdir(os.path.dirname(path))
225        config.config.file_under_load = path
226
227        new_tests = test_mod.TestCase.collector.create()
228        new_suites = suite_mod.TestSuite.collector.create()
229        new_fixtures = fixture_mod.Fixture.collector.create()
230
231        def cleanup():
232            config.config.file_under_load = None
233            sys.path[:] = old_path
234            os.chdir(cwd)
235            test_mod.TestCase.collector.remove(new_tests)
236            suite_mod.TestSuite.collector.remove(new_suites)
237            fixture_mod.Fixture.collector.remove(new_fixtures)
238
239        try:
240            execfile(path, newdict, newdict)
241        except Exception as e:
242            log.test_log.debug(traceback.format_exc())
243            log.test_log.warn(
244                              'Exception thrown while loading "%s"\n'
245                              'Ignoring all tests in this file.'
246                               % (path))
247            cleanup()
248            return
249
250        # Create a module test suite for those not contained in a suite.
251        orphan_tests = set(new_tests)
252        for suite in new_suites:
253            for test in suite:
254                # Remove the test if it wasn't already removed.
255                # (Suites may contain copies of tests.)
256                if test in orphan_tests:
257                    orphan_tests.remove(test)
258        if orphan_tests:
259            orphan_tests = sorted(orphan_tests, key=new_tests.index)
260            # FIXME Use the config based default to group all uncollected
261            # tests.
262            # NOTE: This is automatically collected (we still have the
263            # collector active.)
264            suite_mod.TestSuite(tests=orphan_tests,
265                    name=path_as_suitename(path))
266
267        try:
268            loaded_suites = [wrappers.LoadedSuite(suite, path)
269                    for suite in new_suites]
270
271            self._verify_no_duplicate_suites(loaded_suites)
272            self._verify_no_duplicate_tests_in_suites(loaded_suites)
273        except Exception as e:
274            log.test_log.warn('%s\n'
275                    'Exception thrown while loading "%s"\n'
276                    'Ignoring all tests in this file.'
277                    % (traceback.format_exc(), path))
278        else:
279            log.test_log.info('Discovered %d tests and %d suites in %s'
280                    '' % (len(new_tests), len(loaded_suites), path))
281
282            self.suites.extend(loaded_suites)
283            self.suite_uids.update({suite.uid: suite
284                    for suite in loaded_suites})
285        cleanup()
286
287    def _discover_files(self, root):
288        '''
289        Recurse down from the given root directory returning a list of
290        directories which contain a list of files matching
291        `self.filepath_filter`.
292        '''
293        # Will probably want to order this traversal.
294        for root, dirnames, filenames in os.walk(root):
295            dirnames.sort()
296            if filenames:
297                filenames.sort()
298                filepaths = [os.path.join(root, filename) \
299                             for filename in filenames]
300                filepaths = filter(self.filepath_filter, filepaths)
301                if filepaths:
302                    yield filepaths
303