1# Copyright (c) 2017 Mark D. Hill and David A. Wood
2# All rights reserved.
3#
4# Redistribution and use in source and binary forms, with or without
5# modification, are permitted provided that the following conditions are
6# met: redistributions of source code must retain the above copyright
7# notice, this list of conditions and the following disclaimer;
8# redistributions in binary form must reproduce the above copyright
9# notice, this list of conditions and the following disclaimer in the
10# documentation and/or other materials provided with the distribution;
11# neither the name of the copyright holders nor the names of its
12# contributors may be used to endorse or promote products derived from
13# this software without specific prior written permission.
14#
15# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
16# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
17# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
18# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
19# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
20# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
21# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
22# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
23# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
24# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
25# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
26#
27# Authors: Sean Wilson
28
29'''
30Helper classes for writing tests with this test library.
31'''
32from collections import MutableSet, OrderedDict
33
34import difflib
35import errno
36import os
37import Queue
38import re
39import shutil
40import stat
41import subprocess
42import tempfile
43import threading
44import time
45import traceback
46
47#TODO Tear out duplicate logic from the sandbox IOManager
48def log_call(logger, command, *popenargs, **kwargs):
49    '''
50    Calls the given process and automatically logs the command and output.
51
52    If stdout or stderr are provided output will also be piped into those
53    streams as well.
54
55    :params stdout: Iterable of items to write to as we read from the
56        subprocess.
57
58    :params stderr: Iterable of items to write to as we read from the
59        subprocess.
60    '''
61    if isinstance(command, str):
62        cmdstr = command
63    else:
64        cmdstr = ' '.join(command)
65
66    logger_callback = logger.trace
67    logger.trace('Logging call to command: %s' % cmdstr)
68
69    stdout_redirect = kwargs.get('stdout', tuple())
70    stderr_redirect = kwargs.get('stderr', tuple())
71
72    if hasattr(stdout_redirect, 'write'):
73        stdout_redirect = (stdout_redirect,)
74    if hasattr(stderr_redirect, 'write'):
75        stderr_redirect = (stderr_redirect,)
76
77    kwargs['stdout'] = subprocess.PIPE
78    kwargs['stderr'] = subprocess.PIPE
79    p = subprocess.Popen(command, *popenargs, **kwargs)
80
81    def log_output(log_callback, pipe, redirects=tuple()):
82        # Read iteractively, don't allow input to fill the pipe.
83        for line in iter(pipe.readline, ''):
84            for r in redirects:
85                r.write(line)
86            log_callback(line.rstrip())
87
88    stdout_thread = threading.Thread(target=log_output,
89                           args=(logger_callback, p.stdout, stdout_redirect))
90    stdout_thread.setDaemon(True)
91    stderr_thread = threading.Thread(target=log_output,
92                           args=(logger_callback, p.stderr, stderr_redirect))
93    stderr_thread.setDaemon(True)
94
95    stdout_thread.start()
96    stderr_thread.start()
97
98    retval = p.wait()
99    stdout_thread.join()
100    stderr_thread.join()
101    # Return the return exit code of the process.
102    if retval != 0:
103        raise subprocess.CalledProcessError(retval, cmdstr)
104
105# lru_cache stuff (Introduced in python 3.2+)
106# Renamed and modified to cacheresult
107class _HashedSeq(list):
108    '''
109    This class guarantees that hash() will be called no more than once per
110    element. This is important because the cacheresult() will hash the key
111    multiple times on a cache miss.
112
113    .. note:: From cpython 3.7
114    '''
115
116    __slots__ = 'hashvalue'
117
118    def __init__(self, tup, hash=hash):
119        self[:] = tup
120        self.hashvalue = hash(tup)
121
122    def __hash__(self):
123        return self.hashvalue
124
125def _make_key(args, kwds, typed,
126             kwd_mark = (object(),),
127             fasttypes = {int, str, frozenset, type(None)},
128             tuple=tuple, type=type, len=len):
129    '''
130    Make a cache key from optionally typed positional and keyword arguments.
131    The key is constructed in a way that is flat as possible rather than as
132    a nested structure that would take more memory.  If there is only a single
133    argument and its data type is known to cache its hash value, then that
134    argument is returned without a wrapper. This saves space and improves
135    lookup speed.
136
137    .. note:: From cpython 3.7
138    '''
139    key = args
140    if kwds:
141        key += kwd_mark
142        for item in kwds.items():
143            key += item
144    if typed:
145        key += tuple(type(v) for v in args)
146        if kwds:
147            key += tuple(type(v) for v in kwds.values())
148    elif len(key) == 1 and type(key[0]) in fasttypes:
149        return key[0]
150    return _HashedSeq(key)
151
152
153def cacheresult(function, typed=False):
154    '''
155    :param typed: If typed is True, arguments of different types will be
156        cached separately. I.e. f(3.0) and f(3) will be treated as distinct
157        calls with distinct results.
158
159    .. note:: From cpython 3.7
160    '''
161    sentinel = object()          # unique object used to signal cache misses
162    make_key = _make_key         # build a key from the function arguments
163    cache = {}
164    def wrapper(*args, **kwds):
165        # Simple caching without ordering or size limit
166        key = _make_key(args, kwds, typed)
167        result = cache.get(key, sentinel)
168        if result is not sentinel:
169            return result
170        result = function(*args, **kwds)
171        cache[key] = result
172        return result
173    return wrapper
174
175class OrderedSet(MutableSet):
176    '''
177    Maintain ordering of insertion in items to the set with quick iteration.
178
179    http://code.activestate.com/recipes/576694/
180    '''
181
182    def __init__(self, iterable=None):
183        self.end = end = []
184        end += [None, end, end]         # sentinel node for doubly linked list
185        self.map = {}                   # key --> [key, prev, next]
186        if iterable is not None:
187            self |= iterable
188
189    def __len__(self):
190        return len(self.map)
191
192    def __contains__(self, key):
193        return key in self.map
194
195    def add(self, key):
196        if key not in self.map:
197            end = self.end
198            curr = end[1]
199            curr[2] = end[1] = self.map[key] = [key, curr, end]
200
201    def update(self, keys):
202        for key in keys:
203            self.add(key)
204
205    def discard(self, key):
206        if key in self.map:
207            key, prev, next = self.map.pop(key)
208            prev[2] = next
209            next[1] = prev
210
211    def __iter__(self):
212        end = self.end
213        curr = end[2]
214        while curr is not end:
215            yield curr[0]
216            curr = curr[2]
217
218    def __reversed__(self):
219        end = self.end
220        curr = end[1]
221        while curr is not end:
222            yield curr[0]
223            curr = curr[1]
224
225    def pop(self, last=True):
226        if not self:
227            raise KeyError('set is empty')
228        key = self.end[1][0] if last else self.end[2][0]
229        self.discard(key)
230        return key
231
232    def __repr__(self):
233        if not self:
234            return '%s()' % (self.__class__.__name__,)
235        return '%s(%r)' % (self.__class__.__name__, list(self))
236
237    def __eq__(self, other):
238        if isinstance(other, OrderedSet):
239            return len(self) == len(other) and list(self) == list(other)
240        return set(self) == set(other)
241
242def absdirpath(path):
243    '''
244    Return the directory component of the absolute path of the given path.
245    '''
246    return os.path.dirname(os.path.abspath(path))
247
248joinpath = os.path.join
249
250def mkdir_p(path):
251    '''
252    Same thing as mkdir -p
253
254    https://stackoverflow.com/a/600612
255    '''
256    try:
257        os.makedirs(path)
258    except OSError as exc:  # Python >2.5
259        if exc.errno == errno.EEXIST and os.path.isdir(path):
260            pass
261        else:
262            raise
263
264
265class FrozenSetException(Exception):
266    '''Signals one tried to set a value in a 'frozen' object.'''
267    pass
268
269
270class AttrDict(object):
271    '''Object which exposes its own internal dictionary through attributes.'''
272    def __init__(self, dict_={}):
273        self.update(dict_)
274
275    def __getattr__(self, attr):
276        dict_ = self.__dict__
277        if attr in dict_:
278            return dict_[attr]
279        raise AttributeError('Could not find %s attribute' % attr)
280
281    def __setattr__(self, attr, val):
282        self.__dict__[attr] = val
283
284    def __iter__(self):
285        return iter(self.__dict__)
286
287    def __getitem__(self, item):
288        return self.__dict__[item]
289
290    def update(self, items):
291        self.__dict__.update(items)
292
293
294class FrozenAttrDict(AttrDict):
295    '''An AttrDict whose attributes cannot be modified directly.'''
296    __initialized = False
297    def __init__(self, dict_={}):
298        super(FrozenAttrDict, self).__init__(dict_)
299        self.__initialized = True
300
301    def __setattr__(self, attr, val):
302        if self.__initialized:
303            raise FrozenSetException(
304                        'Cannot modify an attribute in a FozenAttrDict')
305        else:
306            super(FrozenAttrDict, self).__setattr__(attr, val)
307
308    def update(self, items):
309        if self.__initialized:
310            raise FrozenSetException(
311                        'Cannot modify an attribute in a FozenAttrDict')
312        else:
313            super(FrozenAttrDict, self).update(items)
314
315
316class InstanceCollector(object):
317    '''
318    A class used to simplify collecting of Classes.
319
320    >> instance_list = collector.create()
321    >> # Create a bunch of classes which call collector.collect(self)
322    >> # instance_list contains all instances created since
323    >> # collector.create was called
324    >> collector.remove(instance_list)
325    '''
326    def __init__(self):
327        self.collectors = []
328
329    def create(self):
330        collection = []
331        self.collectors.append(collection)
332        return collection
333
334    def remove(self, collector):
335        self.collectors.remove(collector)
336
337    def collect(self, instance):
338        for col in self.collectors:
339            col.append(instance)
340
341
342def append_dictlist(dict_, key, value):
343    '''
344    Append the `value` to a list associated with `key` in `dict_`.
345    If `key` doesn't exist, create a new list in the `dict_` with value in it.
346    '''
347    list_ = dict_.get(key, [])
348    list_.append(value)
349    dict_[key] = list_
350
351
352class ExceptionThread(threading.Thread):
353    '''
354    Wrapper around a python :class:`Thread` which will raise an
355    exception on join if the child threw an unhandled exception.
356    '''
357    def __init__(self, *args, **kwargs):
358        threading.Thread.__init__(self, *args, **kwargs)
359        self._eq = Queue.Queue()
360
361    def run(self, *args, **kwargs):
362        try:
363            threading.Thread.run(self, *args, **kwargs)
364            self._eq.put(None)
365        except:
366            tb = traceback.format_exc()
367            self._eq.put(tb)
368
369    def join(self, *args, **kwargs):
370        threading.Thread.join(*args, **kwargs)
371        exception = self._eq.get()
372        if exception:
373            raise Exception(exception)
374
375
376def _filter_file(fname, filters):
377    with open(fname, "r") as file_:
378        for line in file_:
379            for regex in filters:
380                if re.match(regex, line):
381                    break
382            else:
383                yield line
384
385
386def _copy_file_keep_perms(source, target):
387    '''Copy a file keeping the original permisions of the target.'''
388    st = os.stat(target)
389    shutil.copy2(source, target)
390    os.chown(target, st[stat.ST_UID], st[stat.ST_GID])
391
392
393def _filter_file_inplace(fname, filters):
394    '''
395    Filter the given file writing filtered lines out to a temporary file, then
396    copy that tempfile back into the original file.
397    '''
398    reenter = False
399    (_, tfname) = tempfile.mkstemp(text=True)
400    with open(tfname, 'w') as tempfile_:
401        for line in _filter_file(fname, filters):
402            tempfile_.write(line)
403
404    # Now filtered output is into tempfile_
405    _copy_file_keep_perms(tfname, fname)
406
407
408def diff_out_file(ref_file, out_file, logger, ignore_regexes=tuple()):
409    '''Diff two files returning the diff as a string.'''
410
411    if not os.path.exists(ref_file):
412        raise OSError("%s doesn't exist in reference directory"\
413                                     % ref_file)
414    if not os.path.exists(out_file):
415        raise OSError("%s doesn't exist in output directory" % out_file)
416
417    _filter_file_inplace(out_file, ignore_regexes)
418    _filter_file_inplace(ref_file, ignore_regexes)
419
420    #try :
421    (_, tfname) = tempfile.mkstemp(text=True)
422    with open(tfname, 'r+') as tempfile_:
423        try:
424            log_call(logger, ['diff', out_file, ref_file], stdout=tempfile_)
425        except OSError:
426            # Likely signals that diff does not exist on this system. fallback
427            # to difflib
428            with open(out_file, 'r') as outf, open(ref_file, 'r') as reff:
429                diff = difflib.unified_diff(iter(reff.readline, ''),
430                                            iter(outf.readline, ''),
431                                            fromfile=ref_file,
432                                            tofile=out_file)
433                return ''.join(diff)
434        except subprocess.CalledProcessError:
435            tempfile_.seek(0)
436            return ''.join(tempfile_.readlines())
437        else:
438            return None
439
440class Timer():
441    def __init__(self):
442        self.restart()
443
444    def restart(self):
445        self._start = self.timestamp()
446        self._stop = None
447
448    def stop(self):
449        self._stop = self.timestamp()
450        return self._stop - self._start
451
452    def runtime(self):
453        return self._stop - self._start
454
455    def active_time(self):
456        return self.timestamp() - self._start
457
458    @staticmethod
459    def timestamp():
460        return time.time()