112882Sspwilson2@wisc.edu# Copyright (c) 2017 Mark D. Hill and David A. Wood 212882Sspwilson2@wisc.edu# All rights reserved. 312882Sspwilson2@wisc.edu# 412882Sspwilson2@wisc.edu# Redistribution and use in source and binary forms, with or without 512882Sspwilson2@wisc.edu# modification, are permitted provided that the following conditions are 612882Sspwilson2@wisc.edu# met: redistributions of source code must retain the above copyright 712882Sspwilson2@wisc.edu# notice, this list of conditions and the following disclaimer; 812882Sspwilson2@wisc.edu# redistributions in binary form must reproduce the above copyright 912882Sspwilson2@wisc.edu# notice, this list of conditions and the following disclaimer in the 1012882Sspwilson2@wisc.edu# documentation and/or other materials provided with the distribution; 1112882Sspwilson2@wisc.edu# neither the name of the copyright holders nor the names of its 1212882Sspwilson2@wisc.edu# contributors may be used to endorse or promote products derived from 1312882Sspwilson2@wisc.edu# this software without specific prior written permission. 1412882Sspwilson2@wisc.edu# 1512882Sspwilson2@wisc.edu# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS 1612882Sspwilson2@wisc.edu# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT 1712882Sspwilson2@wisc.edu# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR 1812882Sspwilson2@wisc.edu# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT 1912882Sspwilson2@wisc.edu# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, 2012882Sspwilson2@wisc.edu# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT 2112882Sspwilson2@wisc.edu# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, 2212882Sspwilson2@wisc.edu# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY 2312882Sspwilson2@wisc.edu# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT 2412882Sspwilson2@wisc.edu# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE 2512882Sspwilson2@wisc.edu# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 2612882Sspwilson2@wisc.edu# 2712882Sspwilson2@wisc.edu# Authors: Sean Wilson 2812882Sspwilson2@wisc.edu 2912882Sspwilson2@wisc.edu''' 3012882Sspwilson2@wisc.eduHelper classes for writing tests with this test library. 3112882Sspwilson2@wisc.edu''' 3212882Sspwilson2@wisc.edufrom collections import MutableSet, OrderedDict 3312882Sspwilson2@wisc.edu 3412882Sspwilson2@wisc.eduimport difflib 3512882Sspwilson2@wisc.eduimport errno 3612882Sspwilson2@wisc.eduimport os 3712882Sspwilson2@wisc.eduimport Queue 3812882Sspwilson2@wisc.eduimport re 3912882Sspwilson2@wisc.eduimport shutil 4012882Sspwilson2@wisc.eduimport stat 4112882Sspwilson2@wisc.eduimport subprocess 4212882Sspwilson2@wisc.eduimport tempfile 4312882Sspwilson2@wisc.eduimport threading 4412882Sspwilson2@wisc.eduimport time 4512882Sspwilson2@wisc.eduimport traceback 4612882Sspwilson2@wisc.edu 4712882Sspwilson2@wisc.edu#TODO Tear out duplicate logic from the sandbox IOManager 4812882Sspwilson2@wisc.edudef log_call(logger, command, *popenargs, **kwargs): 4912882Sspwilson2@wisc.edu ''' 5012882Sspwilson2@wisc.edu Calls the given process and automatically logs the command and output. 5112882Sspwilson2@wisc.edu 5212882Sspwilson2@wisc.edu If stdout or stderr are provided output will also be piped into those 5312882Sspwilson2@wisc.edu streams as well. 5412882Sspwilson2@wisc.edu 5512882Sspwilson2@wisc.edu :params stdout: Iterable of items to write to as we read from the 5612882Sspwilson2@wisc.edu subprocess. 5712882Sspwilson2@wisc.edu 5812882Sspwilson2@wisc.edu :params stderr: Iterable of items to write to as we read from the 5912882Sspwilson2@wisc.edu subprocess. 6012882Sspwilson2@wisc.edu ''' 6112882Sspwilson2@wisc.edu if isinstance(command, str): 6212882Sspwilson2@wisc.edu cmdstr = command 6312882Sspwilson2@wisc.edu else: 6412882Sspwilson2@wisc.edu cmdstr = ' '.join(command) 6512882Sspwilson2@wisc.edu 6612882Sspwilson2@wisc.edu logger_callback = logger.trace 6712882Sspwilson2@wisc.edu logger.trace('Logging call to command: %s' % cmdstr) 6812882Sspwilson2@wisc.edu 6912882Sspwilson2@wisc.edu stdout_redirect = kwargs.get('stdout', tuple()) 7012882Sspwilson2@wisc.edu stderr_redirect = kwargs.get('stderr', tuple()) 7112882Sspwilson2@wisc.edu 7212882Sspwilson2@wisc.edu if hasattr(stdout_redirect, 'write'): 7312882Sspwilson2@wisc.edu stdout_redirect = (stdout_redirect,) 7412882Sspwilson2@wisc.edu if hasattr(stderr_redirect, 'write'): 7512882Sspwilson2@wisc.edu stderr_redirect = (stderr_redirect,) 7612882Sspwilson2@wisc.edu 7712882Sspwilson2@wisc.edu kwargs['stdout'] = subprocess.PIPE 7812882Sspwilson2@wisc.edu kwargs['stderr'] = subprocess.PIPE 7912882Sspwilson2@wisc.edu p = subprocess.Popen(command, *popenargs, **kwargs) 8012882Sspwilson2@wisc.edu 8112882Sspwilson2@wisc.edu def log_output(log_callback, pipe, redirects=tuple()): 8212882Sspwilson2@wisc.edu # Read iteractively, don't allow input to fill the pipe. 8312882Sspwilson2@wisc.edu for line in iter(pipe.readline, ''): 8412882Sspwilson2@wisc.edu for r in redirects: 8512882Sspwilson2@wisc.edu r.write(line) 8612882Sspwilson2@wisc.edu log_callback(line.rstrip()) 8712882Sspwilson2@wisc.edu 8812882Sspwilson2@wisc.edu stdout_thread = threading.Thread(target=log_output, 8912882Sspwilson2@wisc.edu args=(logger_callback, p.stdout, stdout_redirect)) 9012882Sspwilson2@wisc.edu stdout_thread.setDaemon(True) 9112882Sspwilson2@wisc.edu stderr_thread = threading.Thread(target=log_output, 9212882Sspwilson2@wisc.edu args=(logger_callback, p.stderr, stderr_redirect)) 9312882Sspwilson2@wisc.edu stderr_thread.setDaemon(True) 9412882Sspwilson2@wisc.edu 9512882Sspwilson2@wisc.edu stdout_thread.start() 9612882Sspwilson2@wisc.edu stderr_thread.start() 9712882Sspwilson2@wisc.edu 9812882Sspwilson2@wisc.edu retval = p.wait() 9912882Sspwilson2@wisc.edu stdout_thread.join() 10012882Sspwilson2@wisc.edu stderr_thread.join() 10112882Sspwilson2@wisc.edu # Return the return exit code of the process. 10212882Sspwilson2@wisc.edu if retval != 0: 10312882Sspwilson2@wisc.edu raise subprocess.CalledProcessError(retval, cmdstr) 10412882Sspwilson2@wisc.edu 10512882Sspwilson2@wisc.edu# lru_cache stuff (Introduced in python 3.2+) 10612882Sspwilson2@wisc.edu# Renamed and modified to cacheresult 10712882Sspwilson2@wisc.educlass _HashedSeq(list): 10812882Sspwilson2@wisc.edu ''' 10912882Sspwilson2@wisc.edu This class guarantees that hash() will be called no more than once per 11012882Sspwilson2@wisc.edu element. This is important because the cacheresult() will hash the key 11112882Sspwilson2@wisc.edu multiple times on a cache miss. 11212882Sspwilson2@wisc.edu 11312882Sspwilson2@wisc.edu .. note:: From cpython 3.7 11412882Sspwilson2@wisc.edu ''' 11512882Sspwilson2@wisc.edu 11612882Sspwilson2@wisc.edu __slots__ = 'hashvalue' 11712882Sspwilson2@wisc.edu 11812882Sspwilson2@wisc.edu def __init__(self, tup, hash=hash): 11912882Sspwilson2@wisc.edu self[:] = tup 12012882Sspwilson2@wisc.edu self.hashvalue = hash(tup) 12112882Sspwilson2@wisc.edu 12212882Sspwilson2@wisc.edu def __hash__(self): 12312882Sspwilson2@wisc.edu return self.hashvalue 12412882Sspwilson2@wisc.edu 12512882Sspwilson2@wisc.edudef _make_key(args, kwds, typed, 12612882Sspwilson2@wisc.edu kwd_mark = (object(),), 12712882Sspwilson2@wisc.edu fasttypes = {int, str, frozenset, type(None)}, 12812882Sspwilson2@wisc.edu tuple=tuple, type=type, len=len): 12912882Sspwilson2@wisc.edu ''' 13012882Sspwilson2@wisc.edu Make a cache key from optionally typed positional and keyword arguments. 13112882Sspwilson2@wisc.edu The key is constructed in a way that is flat as possible rather than as 13212882Sspwilson2@wisc.edu a nested structure that would take more memory. If there is only a single 13312882Sspwilson2@wisc.edu argument and its data type is known to cache its hash value, then that 13412882Sspwilson2@wisc.edu argument is returned without a wrapper. This saves space and improves 13512882Sspwilson2@wisc.edu lookup speed. 13612882Sspwilson2@wisc.edu 13712882Sspwilson2@wisc.edu .. note:: From cpython 3.7 13812882Sspwilson2@wisc.edu ''' 13912882Sspwilson2@wisc.edu key = args 14012882Sspwilson2@wisc.edu if kwds: 14112882Sspwilson2@wisc.edu key += kwd_mark 14212882Sspwilson2@wisc.edu for item in kwds.items(): 14312882Sspwilson2@wisc.edu key += item 14412882Sspwilson2@wisc.edu if typed: 14512882Sspwilson2@wisc.edu key += tuple(type(v) for v in args) 14612882Sspwilson2@wisc.edu if kwds: 14712882Sspwilson2@wisc.edu key += tuple(type(v) for v in kwds.values()) 14812882Sspwilson2@wisc.edu elif len(key) == 1 and type(key[0]) in fasttypes: 14912882Sspwilson2@wisc.edu return key[0] 15012882Sspwilson2@wisc.edu return _HashedSeq(key) 15112882Sspwilson2@wisc.edu 15212882Sspwilson2@wisc.edu 15312882Sspwilson2@wisc.edudef cacheresult(function, typed=False): 15412882Sspwilson2@wisc.edu ''' 15512882Sspwilson2@wisc.edu :param typed: If typed is True, arguments of different types will be 15612882Sspwilson2@wisc.edu cached separately. I.e. f(3.0) and f(3) will be treated as distinct 15712882Sspwilson2@wisc.edu calls with distinct results. 15812882Sspwilson2@wisc.edu 15912882Sspwilson2@wisc.edu .. note:: From cpython 3.7 16012882Sspwilson2@wisc.edu ''' 16112882Sspwilson2@wisc.edu sentinel = object() # unique object used to signal cache misses 16212882Sspwilson2@wisc.edu make_key = _make_key # build a key from the function arguments 16312882Sspwilson2@wisc.edu cache = {} 16412882Sspwilson2@wisc.edu def wrapper(*args, **kwds): 16512882Sspwilson2@wisc.edu # Simple caching without ordering or size limit 16612882Sspwilson2@wisc.edu key = _make_key(args, kwds, typed) 16712882Sspwilson2@wisc.edu result = cache.get(key, sentinel) 16812882Sspwilson2@wisc.edu if result is not sentinel: 16912882Sspwilson2@wisc.edu return result 17012882Sspwilson2@wisc.edu result = function(*args, **kwds) 17112882Sspwilson2@wisc.edu cache[key] = result 17212882Sspwilson2@wisc.edu return result 17312882Sspwilson2@wisc.edu return wrapper 17412882Sspwilson2@wisc.edu 17512882Sspwilson2@wisc.educlass OrderedSet(MutableSet): 17612882Sspwilson2@wisc.edu ''' 17712882Sspwilson2@wisc.edu Maintain ordering of insertion in items to the set with quick iteration. 17812882Sspwilson2@wisc.edu 17912882Sspwilson2@wisc.edu http://code.activestate.com/recipes/576694/ 18012882Sspwilson2@wisc.edu ''' 18112882Sspwilson2@wisc.edu 18212882Sspwilson2@wisc.edu def __init__(self, iterable=None): 18312882Sspwilson2@wisc.edu self.end = end = [] 18412882Sspwilson2@wisc.edu end += [None, end, end] # sentinel node for doubly linked list 18512882Sspwilson2@wisc.edu self.map = {} # key --> [key, prev, next] 18612882Sspwilson2@wisc.edu if iterable is not None: 18712882Sspwilson2@wisc.edu self |= iterable 18812882Sspwilson2@wisc.edu 18912882Sspwilson2@wisc.edu def __len__(self): 19012882Sspwilson2@wisc.edu return len(self.map) 19112882Sspwilson2@wisc.edu 19212882Sspwilson2@wisc.edu def __contains__(self, key): 19312882Sspwilson2@wisc.edu return key in self.map 19412882Sspwilson2@wisc.edu 19512882Sspwilson2@wisc.edu def add(self, key): 19612882Sspwilson2@wisc.edu if key not in self.map: 19712882Sspwilson2@wisc.edu end = self.end 19812882Sspwilson2@wisc.edu curr = end[1] 19912882Sspwilson2@wisc.edu curr[2] = end[1] = self.map[key] = [key, curr, end] 20012882Sspwilson2@wisc.edu 20112882Sspwilson2@wisc.edu def update(self, keys): 20212882Sspwilson2@wisc.edu for key in keys: 20312882Sspwilson2@wisc.edu self.add(key) 20412882Sspwilson2@wisc.edu 20512882Sspwilson2@wisc.edu def discard(self, key): 20612882Sspwilson2@wisc.edu if key in self.map: 20712882Sspwilson2@wisc.edu key, prev, next = self.map.pop(key) 20812882Sspwilson2@wisc.edu prev[2] = next 20912882Sspwilson2@wisc.edu next[1] = prev 21012882Sspwilson2@wisc.edu 21112882Sspwilson2@wisc.edu def __iter__(self): 21212882Sspwilson2@wisc.edu end = self.end 21312882Sspwilson2@wisc.edu curr = end[2] 21412882Sspwilson2@wisc.edu while curr is not end: 21512882Sspwilson2@wisc.edu yield curr[0] 21612882Sspwilson2@wisc.edu curr = curr[2] 21712882Sspwilson2@wisc.edu 21812882Sspwilson2@wisc.edu def __reversed__(self): 21912882Sspwilson2@wisc.edu end = self.end 22012882Sspwilson2@wisc.edu curr = end[1] 22112882Sspwilson2@wisc.edu while curr is not end: 22212882Sspwilson2@wisc.edu yield curr[0] 22312882Sspwilson2@wisc.edu curr = curr[1] 22412882Sspwilson2@wisc.edu 22512882Sspwilson2@wisc.edu def pop(self, last=True): 22612882Sspwilson2@wisc.edu if not self: 22712882Sspwilson2@wisc.edu raise KeyError('set is empty') 22812882Sspwilson2@wisc.edu key = self.end[1][0] if last else self.end[2][0] 22912882Sspwilson2@wisc.edu self.discard(key) 23012882Sspwilson2@wisc.edu return key 23112882Sspwilson2@wisc.edu 23212882Sspwilson2@wisc.edu def __repr__(self): 23312882Sspwilson2@wisc.edu if not self: 23412882Sspwilson2@wisc.edu return '%s()' % (self.__class__.__name__,) 23512882Sspwilson2@wisc.edu return '%s(%r)' % (self.__class__.__name__, list(self)) 23612882Sspwilson2@wisc.edu 23712882Sspwilson2@wisc.edu def __eq__(self, other): 23812882Sspwilson2@wisc.edu if isinstance(other, OrderedSet): 23912882Sspwilson2@wisc.edu return len(self) == len(other) and list(self) == list(other) 24012882Sspwilson2@wisc.edu return set(self) == set(other) 24112882Sspwilson2@wisc.edu 24212882Sspwilson2@wisc.edudef absdirpath(path): 24312882Sspwilson2@wisc.edu ''' 24412882Sspwilson2@wisc.edu Return the directory component of the absolute path of the given path. 24512882Sspwilson2@wisc.edu ''' 24612882Sspwilson2@wisc.edu return os.path.dirname(os.path.abspath(path)) 24712882Sspwilson2@wisc.edu 24812882Sspwilson2@wisc.edujoinpath = os.path.join 24912882Sspwilson2@wisc.edu 25012882Sspwilson2@wisc.edudef mkdir_p(path): 25112882Sspwilson2@wisc.edu ''' 25212882Sspwilson2@wisc.edu Same thing as mkdir -p 25312882Sspwilson2@wisc.edu 25412882Sspwilson2@wisc.edu https://stackoverflow.com/a/600612 25512882Sspwilson2@wisc.edu ''' 25612882Sspwilson2@wisc.edu try: 25712882Sspwilson2@wisc.edu os.makedirs(path) 25812882Sspwilson2@wisc.edu except OSError as exc: # Python >2.5 25912882Sspwilson2@wisc.edu if exc.errno == errno.EEXIST and os.path.isdir(path): 26012882Sspwilson2@wisc.edu pass 26112882Sspwilson2@wisc.edu else: 26212882Sspwilson2@wisc.edu raise 26312882Sspwilson2@wisc.edu 26412882Sspwilson2@wisc.edu 26512882Sspwilson2@wisc.educlass FrozenSetException(Exception): 26612882Sspwilson2@wisc.edu '''Signals one tried to set a value in a 'frozen' object.''' 26712882Sspwilson2@wisc.edu pass 26812882Sspwilson2@wisc.edu 26912882Sspwilson2@wisc.edu 27012882Sspwilson2@wisc.educlass AttrDict(object): 27112882Sspwilson2@wisc.edu '''Object which exposes its own internal dictionary through attributes.''' 27212882Sspwilson2@wisc.edu def __init__(self, dict_={}): 27312882Sspwilson2@wisc.edu self.update(dict_) 27412882Sspwilson2@wisc.edu 27512882Sspwilson2@wisc.edu def __getattr__(self, attr): 27612882Sspwilson2@wisc.edu dict_ = self.__dict__ 27712882Sspwilson2@wisc.edu if attr in dict_: 27812882Sspwilson2@wisc.edu return dict_[attr] 27912882Sspwilson2@wisc.edu raise AttributeError('Could not find %s attribute' % attr) 28012882Sspwilson2@wisc.edu 28112882Sspwilson2@wisc.edu def __setattr__(self, attr, val): 28212882Sspwilson2@wisc.edu self.__dict__[attr] = val 28312882Sspwilson2@wisc.edu 28412882Sspwilson2@wisc.edu def __iter__(self): 28512882Sspwilson2@wisc.edu return iter(self.__dict__) 28612882Sspwilson2@wisc.edu 28712882Sspwilson2@wisc.edu def __getitem__(self, item): 28812882Sspwilson2@wisc.edu return self.__dict__[item] 28912882Sspwilson2@wisc.edu 29012882Sspwilson2@wisc.edu def update(self, items): 29112882Sspwilson2@wisc.edu self.__dict__.update(items) 29212882Sspwilson2@wisc.edu 29312882Sspwilson2@wisc.edu 29412882Sspwilson2@wisc.educlass FrozenAttrDict(AttrDict): 29512882Sspwilson2@wisc.edu '''An AttrDict whose attributes cannot be modified directly.''' 29612882Sspwilson2@wisc.edu __initialized = False 29712882Sspwilson2@wisc.edu def __init__(self, dict_={}): 29812882Sspwilson2@wisc.edu super(FrozenAttrDict, self).__init__(dict_) 29912882Sspwilson2@wisc.edu self.__initialized = True 30012882Sspwilson2@wisc.edu 30112882Sspwilson2@wisc.edu def __setattr__(self, attr, val): 30212882Sspwilson2@wisc.edu if self.__initialized: 30312882Sspwilson2@wisc.edu raise FrozenSetException( 30412882Sspwilson2@wisc.edu 'Cannot modify an attribute in a FozenAttrDict') 30512882Sspwilson2@wisc.edu else: 30612882Sspwilson2@wisc.edu super(FrozenAttrDict, self).__setattr__(attr, val) 30712882Sspwilson2@wisc.edu 30812882Sspwilson2@wisc.edu def update(self, items): 30912882Sspwilson2@wisc.edu if self.__initialized: 31012882Sspwilson2@wisc.edu raise FrozenSetException( 31112882Sspwilson2@wisc.edu 'Cannot modify an attribute in a FozenAttrDict') 31212882Sspwilson2@wisc.edu else: 31312882Sspwilson2@wisc.edu super(FrozenAttrDict, self).update(items) 31412882Sspwilson2@wisc.edu 31512882Sspwilson2@wisc.edu 31612882Sspwilson2@wisc.educlass InstanceCollector(object): 31712882Sspwilson2@wisc.edu ''' 31812882Sspwilson2@wisc.edu A class used to simplify collecting of Classes. 31912882Sspwilson2@wisc.edu 32012882Sspwilson2@wisc.edu >> instance_list = collector.create() 32112882Sspwilson2@wisc.edu >> # Create a bunch of classes which call collector.collect(self) 32212882Sspwilson2@wisc.edu >> # instance_list contains all instances created since 32312882Sspwilson2@wisc.edu >> # collector.create was called 32412882Sspwilson2@wisc.edu >> collector.remove(instance_list) 32512882Sspwilson2@wisc.edu ''' 32612882Sspwilson2@wisc.edu def __init__(self): 32712882Sspwilson2@wisc.edu self.collectors = [] 32812882Sspwilson2@wisc.edu 32912882Sspwilson2@wisc.edu def create(self): 33012882Sspwilson2@wisc.edu collection = [] 33112882Sspwilson2@wisc.edu self.collectors.append(collection) 33212882Sspwilson2@wisc.edu return collection 33312882Sspwilson2@wisc.edu 33412882Sspwilson2@wisc.edu def remove(self, collector): 33512882Sspwilson2@wisc.edu self.collectors.remove(collector) 33612882Sspwilson2@wisc.edu 33712882Sspwilson2@wisc.edu def collect(self, instance): 33812882Sspwilson2@wisc.edu for col in self.collectors: 33912882Sspwilson2@wisc.edu col.append(instance) 34012882Sspwilson2@wisc.edu 34112882Sspwilson2@wisc.edu 34212882Sspwilson2@wisc.edudef append_dictlist(dict_, key, value): 34312882Sspwilson2@wisc.edu ''' 34412882Sspwilson2@wisc.edu Append the `value` to a list associated with `key` in `dict_`. 34512882Sspwilson2@wisc.edu If `key` doesn't exist, create a new list in the `dict_` with value in it. 34612882Sspwilson2@wisc.edu ''' 34712882Sspwilson2@wisc.edu list_ = dict_.get(key, []) 34812882Sspwilson2@wisc.edu list_.append(value) 34912882Sspwilson2@wisc.edu dict_[key] = list_ 35012882Sspwilson2@wisc.edu 35112882Sspwilson2@wisc.edu 35212882Sspwilson2@wisc.educlass ExceptionThread(threading.Thread): 35312882Sspwilson2@wisc.edu ''' 35412882Sspwilson2@wisc.edu Wrapper around a python :class:`Thread` which will raise an 35512882Sspwilson2@wisc.edu exception on join if the child threw an unhandled exception. 35612882Sspwilson2@wisc.edu ''' 35712882Sspwilson2@wisc.edu def __init__(self, *args, **kwargs): 35812882Sspwilson2@wisc.edu threading.Thread.__init__(self, *args, **kwargs) 35912882Sspwilson2@wisc.edu self._eq = Queue.Queue() 36012882Sspwilson2@wisc.edu 36112882Sspwilson2@wisc.edu def run(self, *args, **kwargs): 36212882Sspwilson2@wisc.edu try: 36312882Sspwilson2@wisc.edu threading.Thread.run(self, *args, **kwargs) 36412882Sspwilson2@wisc.edu self._eq.put(None) 36512882Sspwilson2@wisc.edu except: 36612882Sspwilson2@wisc.edu tb = traceback.format_exc() 36712882Sspwilson2@wisc.edu self._eq.put(tb) 36812882Sspwilson2@wisc.edu 36912882Sspwilson2@wisc.edu def join(self, *args, **kwargs): 37012882Sspwilson2@wisc.edu threading.Thread.join(*args, **kwargs) 37112882Sspwilson2@wisc.edu exception = self._eq.get() 37212882Sspwilson2@wisc.edu if exception: 37312882Sspwilson2@wisc.edu raise Exception(exception) 37412882Sspwilson2@wisc.edu 37512882Sspwilson2@wisc.edu 37612882Sspwilson2@wisc.edudef _filter_file(fname, filters): 37712882Sspwilson2@wisc.edu with open(fname, "r") as file_: 37812882Sspwilson2@wisc.edu for line in file_: 37912882Sspwilson2@wisc.edu for regex in filters: 38012882Sspwilson2@wisc.edu if re.match(regex, line): 38112882Sspwilson2@wisc.edu break 38212882Sspwilson2@wisc.edu else: 38312882Sspwilson2@wisc.edu yield line 38412882Sspwilson2@wisc.edu 38512882Sspwilson2@wisc.edu 38612882Sspwilson2@wisc.edudef _copy_file_keep_perms(source, target): 38712882Sspwilson2@wisc.edu '''Copy a file keeping the original permisions of the target.''' 38812882Sspwilson2@wisc.edu st = os.stat(target) 38912882Sspwilson2@wisc.edu shutil.copy2(source, target) 39012882Sspwilson2@wisc.edu os.chown(target, st[stat.ST_UID], st[stat.ST_GID]) 39112882Sspwilson2@wisc.edu 39212882Sspwilson2@wisc.edu 39312882Sspwilson2@wisc.edudef _filter_file_inplace(fname, filters): 39412882Sspwilson2@wisc.edu ''' 39512882Sspwilson2@wisc.edu Filter the given file writing filtered lines out to a temporary file, then 39612882Sspwilson2@wisc.edu copy that tempfile back into the original file. 39712882Sspwilson2@wisc.edu ''' 39812882Sspwilson2@wisc.edu reenter = False 39912882Sspwilson2@wisc.edu (_, tfname) = tempfile.mkstemp(text=True) 40012882Sspwilson2@wisc.edu with open(tfname, 'w') as tempfile_: 40112882Sspwilson2@wisc.edu for line in _filter_file(fname, filters): 40212882Sspwilson2@wisc.edu tempfile_.write(line) 40312882Sspwilson2@wisc.edu 40412882Sspwilson2@wisc.edu # Now filtered output is into tempfile_ 40512882Sspwilson2@wisc.edu _copy_file_keep_perms(tfname, fname) 40612882Sspwilson2@wisc.edu 40712882Sspwilson2@wisc.edu 40812882Sspwilson2@wisc.edudef diff_out_file(ref_file, out_file, logger, ignore_regexes=tuple()): 40912882Sspwilson2@wisc.edu '''Diff two files returning the diff as a string.''' 41012882Sspwilson2@wisc.edu 41112882Sspwilson2@wisc.edu if not os.path.exists(ref_file): 41212882Sspwilson2@wisc.edu raise OSError("%s doesn't exist in reference directory"\ 41312882Sspwilson2@wisc.edu % ref_file) 41412882Sspwilson2@wisc.edu if not os.path.exists(out_file): 41512882Sspwilson2@wisc.edu raise OSError("%s doesn't exist in output directory" % out_file) 41612882Sspwilson2@wisc.edu 41712882Sspwilson2@wisc.edu _filter_file_inplace(out_file, ignore_regexes) 41812882Sspwilson2@wisc.edu _filter_file_inplace(ref_file, ignore_regexes) 41912882Sspwilson2@wisc.edu 42012882Sspwilson2@wisc.edu #try : 42112882Sspwilson2@wisc.edu (_, tfname) = tempfile.mkstemp(text=True) 42212882Sspwilson2@wisc.edu with open(tfname, 'r+') as tempfile_: 42312882Sspwilson2@wisc.edu try: 42412882Sspwilson2@wisc.edu log_call(logger, ['diff', out_file, ref_file], stdout=tempfile_) 42512882Sspwilson2@wisc.edu except OSError: 42612882Sspwilson2@wisc.edu # Likely signals that diff does not exist on this system. fallback 42712882Sspwilson2@wisc.edu # to difflib 42812882Sspwilson2@wisc.edu with open(out_file, 'r') as outf, open(ref_file, 'r') as reff: 42912882Sspwilson2@wisc.edu diff = difflib.unified_diff(iter(reff.readline, ''), 43012882Sspwilson2@wisc.edu iter(outf.readline, ''), 43112882Sspwilson2@wisc.edu fromfile=ref_file, 43212882Sspwilson2@wisc.edu tofile=out_file) 43312882Sspwilson2@wisc.edu return ''.join(diff) 43412882Sspwilson2@wisc.edu except subprocess.CalledProcessError: 43512882Sspwilson2@wisc.edu tempfile_.seek(0) 43612882Sspwilson2@wisc.edu return ''.join(tempfile_.readlines()) 43712882Sspwilson2@wisc.edu else: 43812882Sspwilson2@wisc.edu return None 43912882Sspwilson2@wisc.edu 44012882Sspwilson2@wisc.educlass Timer(): 44112882Sspwilson2@wisc.edu def __init__(self): 44212882Sspwilson2@wisc.edu self.restart() 44312882Sspwilson2@wisc.edu 44412882Sspwilson2@wisc.edu def restart(self): 44512882Sspwilson2@wisc.edu self._start = self.timestamp() 44612882Sspwilson2@wisc.edu self._stop = None 44712882Sspwilson2@wisc.edu 44812882Sspwilson2@wisc.edu def stop(self): 44912882Sspwilson2@wisc.edu self._stop = self.timestamp() 45012882Sspwilson2@wisc.edu return self._stop - self._start 45112882Sspwilson2@wisc.edu 45212882Sspwilson2@wisc.edu def runtime(self): 45312882Sspwilson2@wisc.edu return self._stop - self._start 45412882Sspwilson2@wisc.edu 45512882Sspwilson2@wisc.edu def active_time(self): 45612882Sspwilson2@wisc.edu return self.timestamp() - self._start 45712882Sspwilson2@wisc.edu 45812882Sspwilson2@wisc.edu @staticmethod 45912882Sspwilson2@wisc.edu def timestamp(): 46012882Sspwilson2@wisc.edu return time.time()