1# Copyright (c) 2017 Mark D. Hill and David A. Wood
2# All rights reserved.
3#
4# Redistribution and use in source and binary forms, with or without
5# modification, are permitted provided that the following conditions are
6# met: redistributions of source code must retain the above copyright
7# notice, this list of conditions and the following disclaimer;
8# redistributions in binary form must reproduce the above copyright
9# notice, this list of conditions and the following disclaimer in the
10# documentation and/or other materials provided with the distribution;
11# neither the name of the copyright holders nor the names of its
12# contributors may be used to endorse or promote products derived from
13# this software without specific prior written permission.
14#
15# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
16# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
17# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
18# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
19# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
20# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
21# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
22# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
23# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
24# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
25# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
26#
27# Authors: Sean Wilson
28
29'''
30Contains the :class:`Loader` which is responsible for discovering and loading
31tests.
32
33Loading typically follows the following stages.
34
351. Recurse down a given directory looking for tests which match a given regex.
36
37 The default regex used will match any python file (ending in .py) that has
38 a name starting or ending in test(s). If there are any additional
39 components of the name they must be connected with '-' or '_'. Lastly,
40 file names that begin with '.' will be ignored.
41
42 The following names would match:
43
44 - `tests.py`
45 - `test.py`
46 - `test-this.py`
47 - `tests-that.py`
48 - `these-test.py`
49
50 These would not match:
51
52 - `.test.py` - 'hidden' files are ignored.
53 - `test` - Must end in '.py'
54 - `test-.py` - Needs a character after the hypen.
55 - `testthis.py` - Needs a hypen or underscore to separate 'test' and 'this'
56
57
582. With all files discovered execute each file gathering its test items we
59 care about collecting. (`TestCase`, `TestSuite` and `Fixture` objects.)
60
61As a final note, :class:`TestCase` instances which are not put into
62a :class:`TestSuite` by the test writer will be placed into
63a :class:`TestSuite` named after the module.
64
65.. seealso:: :func:`load_file`
66'''
67
68import os
69import re
70import sys
71import traceback
72
73import config
74import log
75import suite as suite_mod
76import test as test_mod
77import fixture as fixture_mod
78import wrappers
79import uid
80
81class DuplicateTestItemException(Exception):
82 '''
83 Exception indicates multiple test items with the same UID
84 were discovered.
85 '''
86 pass
87
88
89# Match filenames that either begin or end with 'test' or tests and use
90# - or _ to separate additional name components.
91default_filepath_regex = re.compile(
92 r'(((.+[_])?tests?)|(tests?([-_].+)?))\.py$')
93
94def default_filepath_filter(filepath):
95 '''The default filter applied to filepaths to marks as test sources.'''
96 filepath = os.path.basename(filepath)
97 if default_filepath_regex.match(filepath):
98 # Make sure doesn't start with .
99 return not filepath.startswith('.')
100 return False
101
102def path_as_modulename(filepath):
103 '''Return the given filepath as a module name.'''
104 # Remove the file extention (.py)
105 return os.path.splitext(os.path.basename(filepath))[0]
106
107def path_as_suitename(filepath):
108 return os.path.split(os.path.dirname(os.path.abspath((filepath))))[-1]
109
110def _assert_files_in_same_dir(files):
111 if __debug__:
112 if files:
113 directory = os.path.dirname(files[0])
114 for f in files:
115 assert os.path.dirname(f) == directory
116
117class Loader(object):
118 '''
119 Class for discovering tests.
120
121 Discovered :class:`TestCase` and :class:`TestSuite` objects are wrapped by
122 :class:`LoadedTest` and :class:`LoadedSuite` objects respectively.
123 These objects provided additional methods and metadata about the loaded
124 objects and are the internal representation used by testlib.
125
126 To simply discover and load all tests using the default filter create an
127 instance and `load_root`.
128
129 >>> import os
130 >>> tl = Loader()
131 >>> tl.load_root(os.getcwd())
132
133 .. note:: If tests are not contained in a TestSuite, they will
134 automatically be placed into one for the module.
135
136 .. warn:: This class is extremely thread-unsafe.
137 It modifies the sys path and global config.
138 Use with care.
139 '''
140 def __init__(self):
141 self.suites = []
142 self.suite_uids = {}
143 self.filepath_filter = default_filepath_filter
144
145 # filepath -> Successful | Failed to load
146 self._files = {}
147
148 @property
149 def schedule(self):
| 1# Copyright (c) 2017 Mark D. Hill and David A. Wood
2# All rights reserved.
3#
4# Redistribution and use in source and binary forms, with or without
5# modification, are permitted provided that the following conditions are
6# met: redistributions of source code must retain the above copyright
7# notice, this list of conditions and the following disclaimer;
8# redistributions in binary form must reproduce the above copyright
9# notice, this list of conditions and the following disclaimer in the
10# documentation and/or other materials provided with the distribution;
11# neither the name of the copyright holders nor the names of its
12# contributors may be used to endorse or promote products derived from
13# this software without specific prior written permission.
14#
15# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
16# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
17# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
18# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
19# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
20# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
21# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
22# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
23# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
24# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
25# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
26#
27# Authors: Sean Wilson
28
29'''
30Contains the :class:`Loader` which is responsible for discovering and loading
31tests.
32
33Loading typically follows the following stages.
34
351. Recurse down a given directory looking for tests which match a given regex.
36
37 The default regex used will match any python file (ending in .py) that has
38 a name starting or ending in test(s). If there are any additional
39 components of the name they must be connected with '-' or '_'. Lastly,
40 file names that begin with '.' will be ignored.
41
42 The following names would match:
43
44 - `tests.py`
45 - `test.py`
46 - `test-this.py`
47 - `tests-that.py`
48 - `these-test.py`
49
50 These would not match:
51
52 - `.test.py` - 'hidden' files are ignored.
53 - `test` - Must end in '.py'
54 - `test-.py` - Needs a character after the hypen.
55 - `testthis.py` - Needs a hypen or underscore to separate 'test' and 'this'
56
57
582. With all files discovered execute each file gathering its test items we
59 care about collecting. (`TestCase`, `TestSuite` and `Fixture` objects.)
60
61As a final note, :class:`TestCase` instances which are not put into
62a :class:`TestSuite` by the test writer will be placed into
63a :class:`TestSuite` named after the module.
64
65.. seealso:: :func:`load_file`
66'''
67
68import os
69import re
70import sys
71import traceback
72
73import config
74import log
75import suite as suite_mod
76import test as test_mod
77import fixture as fixture_mod
78import wrappers
79import uid
80
81class DuplicateTestItemException(Exception):
82 '''
83 Exception indicates multiple test items with the same UID
84 were discovered.
85 '''
86 pass
87
88
89# Match filenames that either begin or end with 'test' or tests and use
90# - or _ to separate additional name components.
91default_filepath_regex = re.compile(
92 r'(((.+[_])?tests?)|(tests?([-_].+)?))\.py$')
93
94def default_filepath_filter(filepath):
95 '''The default filter applied to filepaths to marks as test sources.'''
96 filepath = os.path.basename(filepath)
97 if default_filepath_regex.match(filepath):
98 # Make sure doesn't start with .
99 return not filepath.startswith('.')
100 return False
101
102def path_as_modulename(filepath):
103 '''Return the given filepath as a module name.'''
104 # Remove the file extention (.py)
105 return os.path.splitext(os.path.basename(filepath))[0]
106
107def path_as_suitename(filepath):
108 return os.path.split(os.path.dirname(os.path.abspath((filepath))))[-1]
109
110def _assert_files_in_same_dir(files):
111 if __debug__:
112 if files:
113 directory = os.path.dirname(files[0])
114 for f in files:
115 assert os.path.dirname(f) == directory
116
117class Loader(object):
118 '''
119 Class for discovering tests.
120
121 Discovered :class:`TestCase` and :class:`TestSuite` objects are wrapped by
122 :class:`LoadedTest` and :class:`LoadedSuite` objects respectively.
123 These objects provided additional methods and metadata about the loaded
124 objects and are the internal representation used by testlib.
125
126 To simply discover and load all tests using the default filter create an
127 instance and `load_root`.
128
129 >>> import os
130 >>> tl = Loader()
131 >>> tl.load_root(os.getcwd())
132
133 .. note:: If tests are not contained in a TestSuite, they will
134 automatically be placed into one for the module.
135
136 .. warn:: This class is extremely thread-unsafe.
137 It modifies the sys path and global config.
138 Use with care.
139 '''
140 def __init__(self):
141 self.suites = []
142 self.suite_uids = {}
143 self.filepath_filter = default_filepath_filter
144
145 # filepath -> Successful | Failed to load
146 self._files = {}
147
148 @property
149 def schedule(self):
|
150 return wrappers.LoadedLibrary(self.suites, fixture_mod.global_fixtures)
| 150 return wrappers.LoadedLibrary(self.suites)
|
151
152 def load_schedule_for_suites(self, *uids):
153 files = {uid.UID.uid_to_path(id_) for id_ in uids}
154 for file_ in files:
155 self.load_file(file_)
156
157 return wrappers.LoadedLibrary(
| 151
152 def load_schedule_for_suites(self, *uids):
153 files = {uid.UID.uid_to_path(id_) for id_ in uids}
154 for file_ in files:
155 self.load_file(file_)
156
157 return wrappers.LoadedLibrary(
|
158 [self.suite_uids[id_] for id_ in uids],
159 fixture_mod.global_fixtures)
| 158 [self.suite_uids[id_] for id_ in uids])
|
160
161 def _verify_no_duplicate_suites(self, new_suites):
162 new_suite_uids = self.suite_uids.copy()
163 for suite in new_suites:
164 if suite.uid in new_suite_uids:
165 raise DuplicateTestItemException(
166 "More than one suite with UID '%s' was defined" %\
167 suite.uid)
168 new_suite_uids[suite.uid] = suite
169
170 def _verify_no_duplicate_tests_in_suites(self, new_suites):
171 for suite in new_suites:
172 test_uids = set()
173 for test in suite:
174 if test.uid in test_uids:
175 raise DuplicateTestItemException(
176 "More than one test with UID '%s' was defined"
177 " in suite '%s'"
178 % (test.uid, suite.uid))
179 test_uids.add(test.uid)
180
181 def load_root(self, root):
182 '''
183 Load files from the given root directory which match
184 `self.filepath_filter`.
185 '''
186 if __debug__:
187 self._loaded_a_file = True
188
189 for directory in self._discover_files(root):
190 if directory:
191 _assert_files_in_same_dir(directory)
192 for f in directory:
193 self.load_file(f)
194
195 def load_dir(self, directory):
196 for dir_ in self._discover_files(directory):
197 _assert_files_in_same_dir(dir_)
198 for f in dir_:
199 self.load_file(f)
200
201 def load_file(self, path):
202 path = os.path.abspath(path)
203
204 if path in self._files:
205 if not self._files[path]:
206 raise Exception('Attempted to load a file which already'
207 ' failed to load')
208 else:
209 log.test_log.debug('Tried to reload: %s' % path)
210 return
211
212 # Create a custom dictionary for the loaded module.
213 newdict = {
214 '__builtins__':__builtins__,
215 '__name__': path_as_modulename(path),
216 '__file__': path,
217 }
218
219 # Add the file's containing directory to the system path. So it can do
220 # relative imports naturally.
221 old_path = sys.path[:]
222 sys.path.insert(0, os.path.dirname(path))
223 cwd = os.getcwd()
224 os.chdir(os.path.dirname(path))
225 config.config.file_under_load = path
226
227 new_tests = test_mod.TestCase.collector.create()
228 new_suites = suite_mod.TestSuite.collector.create()
229 new_fixtures = fixture_mod.Fixture.collector.create()
230
231 def cleanup():
232 config.config.file_under_load = None
233 sys.path[:] = old_path
234 os.chdir(cwd)
235 test_mod.TestCase.collector.remove(new_tests)
236 suite_mod.TestSuite.collector.remove(new_suites)
237 fixture_mod.Fixture.collector.remove(new_fixtures)
238
239 try:
240 execfile(path, newdict, newdict)
241 except Exception as e:
242 log.test_log.debug(traceback.format_exc())
243 log.test_log.warn(
244 'Exception thrown while loading "%s"\n'
245 'Ignoring all tests in this file.'
246 % (path))
247 cleanup()
248 return
249
250 # Create a module test suite for those not contained in a suite.
251 orphan_tests = set(new_tests)
252 for suite in new_suites:
253 for test in suite:
254 # Remove the test if it wasn't already removed.
255 # (Suites may contain copies of tests.)
256 if test in orphan_tests:
257 orphan_tests.remove(test)
258 if orphan_tests:
259 orphan_tests = sorted(orphan_tests, key=new_tests.index)
260 # FIXME Use the config based default to group all uncollected
261 # tests.
262 # NOTE: This is automatically collected (we still have the
263 # collector active.)
264 suite_mod.TestSuite(tests=orphan_tests,
265 name=path_as_suitename(path))
266
267 try:
268 loaded_suites = [wrappers.LoadedSuite(suite, path)
269 for suite in new_suites]
270
271 self._verify_no_duplicate_suites(loaded_suites)
272 self._verify_no_duplicate_tests_in_suites(loaded_suites)
273 except Exception as e:
274 log.test_log.warn('%s\n'
275 'Exception thrown while loading "%s"\n'
276 'Ignoring all tests in this file.'
277 % (traceback.format_exc(), path))
278 else:
279 log.test_log.info('Discovered %d tests and %d suites in %s'
280 '' % (len(new_tests), len(loaded_suites), path))
281
282 self.suites.extend(loaded_suites)
283 self.suite_uids.update({suite.uid: suite
284 for suite in loaded_suites})
285 cleanup()
286
287 def _discover_files(self, root):
288 '''
289 Recurse down from the given root directory returning a list of
290 directories which contain a list of files matching
291 `self.filepath_filter`.
292 '''
293 # Will probably want to order this traversal.
294 for root, dirnames, filenames in os.walk(root):
295 dirnames.sort()
296 if filenames:
297 filenames.sort()
298 filepaths = [os.path.join(root, filename) \
299 for filename in filenames]
300 filepaths = filter(self.filepath_filter, filepaths)
301 if filepaths:
302 yield filepaths
| 159
160 def _verify_no_duplicate_suites(self, new_suites):
161 new_suite_uids = self.suite_uids.copy()
162 for suite in new_suites:
163 if suite.uid in new_suite_uids:
164 raise DuplicateTestItemException(
165 "More than one suite with UID '%s' was defined" %\
166 suite.uid)
167 new_suite_uids[suite.uid] = suite
168
169 def _verify_no_duplicate_tests_in_suites(self, new_suites):
170 for suite in new_suites:
171 test_uids = set()
172 for test in suite:
173 if test.uid in test_uids:
174 raise DuplicateTestItemException(
175 "More than one test with UID '%s' was defined"
176 " in suite '%s'"
177 % (test.uid, suite.uid))
178 test_uids.add(test.uid)
179
180 def load_root(self, root):
181 '''
182 Load files from the given root directory which match
183 `self.filepath_filter`.
184 '''
185 if __debug__:
186 self._loaded_a_file = True
187
188 for directory in self._discover_files(root):
189 if directory:
190 _assert_files_in_same_dir(directory)
191 for f in directory:
192 self.load_file(f)
193
194 def load_dir(self, directory):
195 for dir_ in self._discover_files(directory):
196 _assert_files_in_same_dir(dir_)
197 for f in dir_:
198 self.load_file(f)
199
200 def load_file(self, path):
201 path = os.path.abspath(path)
202
203 if path in self._files:
204 if not self._files[path]:
205 raise Exception('Attempted to load a file which already'
206 ' failed to load')
207 else:
208 log.test_log.debug('Tried to reload: %s' % path)
209 return
210
211 # Create a custom dictionary for the loaded module.
212 newdict = {
213 '__builtins__':__builtins__,
214 '__name__': path_as_modulename(path),
215 '__file__': path,
216 }
217
218 # Add the file's containing directory to the system path. So it can do
219 # relative imports naturally.
220 old_path = sys.path[:]
221 sys.path.insert(0, os.path.dirname(path))
222 cwd = os.getcwd()
223 os.chdir(os.path.dirname(path))
224 config.config.file_under_load = path
225
226 new_tests = test_mod.TestCase.collector.create()
227 new_suites = suite_mod.TestSuite.collector.create()
228 new_fixtures = fixture_mod.Fixture.collector.create()
229
230 def cleanup():
231 config.config.file_under_load = None
232 sys.path[:] = old_path
233 os.chdir(cwd)
234 test_mod.TestCase.collector.remove(new_tests)
235 suite_mod.TestSuite.collector.remove(new_suites)
236 fixture_mod.Fixture.collector.remove(new_fixtures)
237
238 try:
239 execfile(path, newdict, newdict)
240 except Exception as e:
241 log.test_log.debug(traceback.format_exc())
242 log.test_log.warn(
243 'Exception thrown while loading "%s"\n'
244 'Ignoring all tests in this file.'
245 % (path))
246 cleanup()
247 return
248
249 # Create a module test suite for those not contained in a suite.
250 orphan_tests = set(new_tests)
251 for suite in new_suites:
252 for test in suite:
253 # Remove the test if it wasn't already removed.
254 # (Suites may contain copies of tests.)
255 if test in orphan_tests:
256 orphan_tests.remove(test)
257 if orphan_tests:
258 orphan_tests = sorted(orphan_tests, key=new_tests.index)
259 # FIXME Use the config based default to group all uncollected
260 # tests.
261 # NOTE: This is automatically collected (we still have the
262 # collector active.)
263 suite_mod.TestSuite(tests=orphan_tests,
264 name=path_as_suitename(path))
265
266 try:
267 loaded_suites = [wrappers.LoadedSuite(suite, path)
268 for suite in new_suites]
269
270 self._verify_no_duplicate_suites(loaded_suites)
271 self._verify_no_duplicate_tests_in_suites(loaded_suites)
272 except Exception as e:
273 log.test_log.warn('%s\n'
274 'Exception thrown while loading "%s"\n'
275 'Ignoring all tests in this file.'
276 % (traceback.format_exc(), path))
277 else:
278 log.test_log.info('Discovered %d tests and %d suites in %s'
279 '' % (len(new_tests), len(loaded_suites), path))
280
281 self.suites.extend(loaded_suites)
282 self.suite_uids.update({suite.uid: suite
283 for suite in loaded_suites})
284 cleanup()
285
286 def _discover_files(self, root):
287 '''
288 Recurse down from the given root directory returning a list of
289 directories which contain a list of files matching
290 `self.filepath_filter`.
291 '''
292 # Will probably want to order this traversal.
293 for root, dirnames, filenames in os.walk(root):
294 dirnames.sort()
295 if filenames:
296 filenames.sort()
297 filepaths = [os.path.join(root, filename) \
298 for filename in filenames]
299 filepaths = filter(self.filepath_filter, filepaths)
300 if filepaths:
301 yield filepaths
|