112882Sspwilson2@wisc.edu# Copyright (c) 2017 Mark D. Hill and David A. Wood 212882Sspwilson2@wisc.edu# All rights reserved. 312882Sspwilson2@wisc.edu# 412882Sspwilson2@wisc.edu# Redistribution and use in source and binary forms, with or without 512882Sspwilson2@wisc.edu# modification, are permitted provided that the following conditions are 612882Sspwilson2@wisc.edu# met: redistributions of source code must retain the above copyright 712882Sspwilson2@wisc.edu# notice, this list of conditions and the following disclaimer; 812882Sspwilson2@wisc.edu# redistributions in binary form must reproduce the above copyright 912882Sspwilson2@wisc.edu# notice, this list of conditions and the following disclaimer in the 1012882Sspwilson2@wisc.edu# documentation and/or other materials provided with the distribution; 1112882Sspwilson2@wisc.edu# neither the name of the copyright holders nor the names of its 1212882Sspwilson2@wisc.edu# contributors may be used to endorse or promote products derived from 1312882Sspwilson2@wisc.edu# this software without specific prior written permission. 1412882Sspwilson2@wisc.edu# 1512882Sspwilson2@wisc.edu# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS 1612882Sspwilson2@wisc.edu# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT 1712882Sspwilson2@wisc.edu# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR 1812882Sspwilson2@wisc.edu# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT 1912882Sspwilson2@wisc.edu# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, 2012882Sspwilson2@wisc.edu# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT 2112882Sspwilson2@wisc.edu# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, 2212882Sspwilson2@wisc.edu# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY 2312882Sspwilson2@wisc.edu# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT 2412882Sspwilson2@wisc.edu# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE 2512882Sspwilson2@wisc.edu# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 2612882Sspwilson2@wisc.edu# 2712882Sspwilson2@wisc.edu# Authors: Sean Wilson 2812882Sspwilson2@wisc.edu 2912882Sspwilson2@wisc.eduimport os 3012882Sspwilson2@wisc.eduimport pickle 3112882Sspwilson2@wisc.eduimport xml.sax.saxutils 3212882Sspwilson2@wisc.edu 3312882Sspwilson2@wisc.edufrom config import config 3412882Sspwilson2@wisc.eduimport helper 3512882Sspwilson2@wisc.eduimport state 3612882Sspwilson2@wisc.eduimport log 3712882Sspwilson2@wisc.edu 3812882Sspwilson2@wisc.edudef _create_uid_index(iterable): 3912882Sspwilson2@wisc.edu index = {} 4012882Sspwilson2@wisc.edu for item in iterable: 4112882Sspwilson2@wisc.edu assert item.uid not in index 4212882Sspwilson2@wisc.edu index[item.uid] = item 4312882Sspwilson2@wisc.edu return index 4412882Sspwilson2@wisc.edu 4512882Sspwilson2@wisc.edu 4612882Sspwilson2@wisc.educlass _CommonMetadataMixin: 4712882Sspwilson2@wisc.edu @property 4812882Sspwilson2@wisc.edu def name(self): 4912882Sspwilson2@wisc.edu return self._metadata.name 5012882Sspwilson2@wisc.edu @property 5112882Sspwilson2@wisc.edu def uid(self): 5212882Sspwilson2@wisc.edu return self._metadata.uid 5312882Sspwilson2@wisc.edu @property 5412882Sspwilson2@wisc.edu def result(self): 5512882Sspwilson2@wisc.edu return self._metadata.result 5612882Sspwilson2@wisc.edu @result.setter 5712882Sspwilson2@wisc.edu def result(self, result): 5812882Sspwilson2@wisc.edu self._metadata.result = result 5912882Sspwilson2@wisc.edu 6012882Sspwilson2@wisc.edu @property 6113789Sjason@lowepower.com def unsuccessful(self): 6212882Sspwilson2@wisc.edu return self._metadata.result.value != state.Result.Passed 6312882Sspwilson2@wisc.edu 6412882Sspwilson2@wisc.edu 6512882Sspwilson2@wisc.educlass InternalTestResult(object, _CommonMetadataMixin): 6612882Sspwilson2@wisc.edu def __init__(self, obj, suite, directory): 6712882Sspwilson2@wisc.edu self._metadata = obj.metadata 6812882Sspwilson2@wisc.edu self.suite = suite 6912882Sspwilson2@wisc.edu 7012882Sspwilson2@wisc.edu self.stderr = os.path.join( 7112882Sspwilson2@wisc.edu InternalSavedResults.output_path(self.uid, suite.uid), 7212882Sspwilson2@wisc.edu 'stderr' 7312882Sspwilson2@wisc.edu ) 7412882Sspwilson2@wisc.edu self.stdout = os.path.join( 7512882Sspwilson2@wisc.edu InternalSavedResults.output_path(self.uid, suite.uid), 7612882Sspwilson2@wisc.edu 'stdout' 7712882Sspwilson2@wisc.edu ) 7812882Sspwilson2@wisc.edu 7912882Sspwilson2@wisc.edu 8012882Sspwilson2@wisc.educlass InternalSuiteResult(object, _CommonMetadataMixin): 8112882Sspwilson2@wisc.edu def __init__(self, obj, directory): 8212882Sspwilson2@wisc.edu self._metadata = obj.metadata 8312882Sspwilson2@wisc.edu self.directory = directory 8412882Sspwilson2@wisc.edu self._wrap_tests(obj) 8512882Sspwilson2@wisc.edu 8612882Sspwilson2@wisc.edu def _wrap_tests(self, obj): 8712882Sspwilson2@wisc.edu self._tests = [InternalTestResult(test, self, self.directory) 8812882Sspwilson2@wisc.edu for test in obj] 8912882Sspwilson2@wisc.edu self._tests_index = _create_uid_index(self._tests) 9012882Sspwilson2@wisc.edu 9112882Sspwilson2@wisc.edu def get_test(self, uid): 9212882Sspwilson2@wisc.edu return self._tests_index[uid] 9312882Sspwilson2@wisc.edu 9412882Sspwilson2@wisc.edu def __iter__(self): 9512882Sspwilson2@wisc.edu return iter(self._tests) 9612882Sspwilson2@wisc.edu 9712882Sspwilson2@wisc.edu def get_test_result(self, uid): 9812882Sspwilson2@wisc.edu return self.get_test(uid) 9912882Sspwilson2@wisc.edu 10012882Sspwilson2@wisc.edu def aggregate_test_results(self): 10112882Sspwilson2@wisc.edu results = {} 10212882Sspwilson2@wisc.edu for test in self: 10312882Sspwilson2@wisc.edu helper.append_dictlist(results, test.result.value, test) 10412882Sspwilson2@wisc.edu return results 10512882Sspwilson2@wisc.edu 10612882Sspwilson2@wisc.edu 10712882Sspwilson2@wisc.educlass InternalLibraryResults(object, _CommonMetadataMixin): 10812882Sspwilson2@wisc.edu def __init__(self, obj, directory): 10912882Sspwilson2@wisc.edu self.directory = directory 11012882Sspwilson2@wisc.edu self._metadata = obj.metadata 11112882Sspwilson2@wisc.edu self._wrap_suites(obj) 11212882Sspwilson2@wisc.edu 11312882Sspwilson2@wisc.edu def __iter__(self): 11412882Sspwilson2@wisc.edu return iter(self._suites) 11512882Sspwilson2@wisc.edu 11612882Sspwilson2@wisc.edu def _wrap_suites(self, obj): 11712882Sspwilson2@wisc.edu self._suites = [InternalSuiteResult(suite, self.directory) 11812882Sspwilson2@wisc.edu for suite in obj] 11912882Sspwilson2@wisc.edu self._suites_index = _create_uid_index(self._suites) 12012882Sspwilson2@wisc.edu 12112882Sspwilson2@wisc.edu def add_suite(self, suite): 12212882Sspwilson2@wisc.edu if suite.uid in self._suites: 12312882Sspwilson2@wisc.edu raise ValueError('Cannot have duplicate suite UIDs.') 12412882Sspwilson2@wisc.edu self._suites[suite.uid] = suite 12512882Sspwilson2@wisc.edu 12612882Sspwilson2@wisc.edu def get_suite_result(self, suite_uid): 12712882Sspwilson2@wisc.edu return self._suites_index[suite_uid] 12812882Sspwilson2@wisc.edu 12912882Sspwilson2@wisc.edu def get_test_result(self, test_uid, suite_uid): 13012882Sspwilson2@wisc.edu return self.get_suite_result(suite_uid).get_test_result(test_uid) 13112882Sspwilson2@wisc.edu 13212882Sspwilson2@wisc.edu def aggregate_test_results(self): 13312882Sspwilson2@wisc.edu results = {} 13412882Sspwilson2@wisc.edu for suite in self._suites: 13512882Sspwilson2@wisc.edu for test in suite: 13612882Sspwilson2@wisc.edu helper.append_dictlist(results, test.result.value, test) 13712882Sspwilson2@wisc.edu return results 13812882Sspwilson2@wisc.edu 13912882Sspwilson2@wisc.educlass InternalSavedResults: 14012882Sspwilson2@wisc.edu @staticmethod 14112882Sspwilson2@wisc.edu def output_path(test_uid, suite_uid, base=None): 14212882Sspwilson2@wisc.edu ''' 14312882Sspwilson2@wisc.edu Return the path which results for a specific test case should be 14412882Sspwilson2@wisc.edu stored. 14512882Sspwilson2@wisc.edu ''' 14612882Sspwilson2@wisc.edu if base is None: 14712882Sspwilson2@wisc.edu base = config.result_path 14812882Sspwilson2@wisc.edu return os.path.join( 14912882Sspwilson2@wisc.edu base, 15012882Sspwilson2@wisc.edu str(suite_uid).replace(os.path.sep, '-'), 15112882Sspwilson2@wisc.edu str(test_uid).replace(os.path.sep, '-')) 15212882Sspwilson2@wisc.edu 15312882Sspwilson2@wisc.edu @staticmethod 15412882Sspwilson2@wisc.edu def save(results, path, protocol=pickle.HIGHEST_PROTOCOL): 15512882Sspwilson2@wisc.edu if not os.path.exists(os.path.dirname(path)): 15612882Sspwilson2@wisc.edu try: 15712882Sspwilson2@wisc.edu os.makedirs(os.path.dirname(path)) 15812882Sspwilson2@wisc.edu except OSError as exc: # Guard against race condition 15912882Sspwilson2@wisc.edu if exc.errno != errno.EEXIST: 16012882Sspwilson2@wisc.edu raise 16112882Sspwilson2@wisc.edu 16212882Sspwilson2@wisc.edu with open(path, 'w') as f: 16312882Sspwilson2@wisc.edu pickle.dump(results, f, protocol) 16412882Sspwilson2@wisc.edu 16512882Sspwilson2@wisc.edu @staticmethod 16612882Sspwilson2@wisc.edu def load(path): 16712882Sspwilson2@wisc.edu with open(path, 'r') as f: 16812882Sspwilson2@wisc.edu return pickle.load(f) 16912882Sspwilson2@wisc.edu 17012882Sspwilson2@wisc.edu 17112882Sspwilson2@wisc.educlass XMLElement(object): 17212882Sspwilson2@wisc.edu def write(self, file_): 17312882Sspwilson2@wisc.edu self.begin(file_) 17412882Sspwilson2@wisc.edu self.end(file_) 17512882Sspwilson2@wisc.edu 17612882Sspwilson2@wisc.edu def begin(self, file_): 17712882Sspwilson2@wisc.edu file_.write('<') 17812882Sspwilson2@wisc.edu file_.write(self.name) 17912882Sspwilson2@wisc.edu for attr in self.attributes: 18012882Sspwilson2@wisc.edu file_.write(' ') 18112882Sspwilson2@wisc.edu attr.write(file_) 18212882Sspwilson2@wisc.edu file_.write('>') 18312882Sspwilson2@wisc.edu 18412882Sspwilson2@wisc.edu self.body(file_) 18512882Sspwilson2@wisc.edu 18612882Sspwilson2@wisc.edu def body(self, file_): 18712882Sspwilson2@wisc.edu for elem in self.elements: 18812882Sspwilson2@wisc.edu file_.write('\n') 18912882Sspwilson2@wisc.edu elem.write(file_) 19012882Sspwilson2@wisc.edu file_.write('\n') 19112882Sspwilson2@wisc.edu 19212882Sspwilson2@wisc.edu def end(self, file_): 19312882Sspwilson2@wisc.edu file_.write('</%s>' % self.name) 19412882Sspwilson2@wisc.edu 19512882Sspwilson2@wisc.educlass XMLAttribute(object): 19612882Sspwilson2@wisc.edu def __init__(self, name, value): 19712882Sspwilson2@wisc.edu self.name = name 19812882Sspwilson2@wisc.edu self.value = value 19912882Sspwilson2@wisc.edu 20012882Sspwilson2@wisc.edu def write(self, file_): 20112882Sspwilson2@wisc.edu file_.write('%s=%s' % (self.name, 20212882Sspwilson2@wisc.edu xml.sax.saxutils.quoteattr(self.value))) 20312882Sspwilson2@wisc.edu 20412882Sspwilson2@wisc.edu 20512882Sspwilson2@wisc.educlass JUnitTestSuites(XMLElement): 20612882Sspwilson2@wisc.edu name = 'testsuites' 20712882Sspwilson2@wisc.edu result_map = { 20812882Sspwilson2@wisc.edu state.Result.Errored: 'errors', 20912882Sspwilson2@wisc.edu state.Result.Failed: 'failures', 21012882Sspwilson2@wisc.edu state.Result.Passed: 'tests' 21112882Sspwilson2@wisc.edu } 21212882Sspwilson2@wisc.edu 21312882Sspwilson2@wisc.edu def __init__(self, internal_results): 21412882Sspwilson2@wisc.edu results = internal_results.aggregate_test_results() 21512882Sspwilson2@wisc.edu 21612882Sspwilson2@wisc.edu self.attributes = [] 21712882Sspwilson2@wisc.edu for result, tests in results.items(): 21812882Sspwilson2@wisc.edu self.attributes.append(self.result_attribute(result, 21912882Sspwilson2@wisc.edu str(len(tests)))) 22012882Sspwilson2@wisc.edu 22112882Sspwilson2@wisc.edu self.elements = [] 22212882Sspwilson2@wisc.edu for suite in internal_results: 22312882Sspwilson2@wisc.edu self.elements.append(JUnitTestSuite(suite)) 22412882Sspwilson2@wisc.edu 22512882Sspwilson2@wisc.edu def result_attribute(self, result, count): 22612882Sspwilson2@wisc.edu return XMLAttribute(self.result_map[result], count) 22712882Sspwilson2@wisc.edu 22812882Sspwilson2@wisc.educlass JUnitTestSuite(JUnitTestSuites): 22912882Sspwilson2@wisc.edu name = 'testsuite' 23012882Sspwilson2@wisc.edu result_map = { 23112882Sspwilson2@wisc.edu state.Result.Errored: 'errors', 23212882Sspwilson2@wisc.edu state.Result.Failed: 'failures', 23312882Sspwilson2@wisc.edu state.Result.Passed: 'tests', 23412882Sspwilson2@wisc.edu state.Result.Skipped: 'skipped' 23512882Sspwilson2@wisc.edu } 23612882Sspwilson2@wisc.edu 23712882Sspwilson2@wisc.edu def __init__(self, suite_result): 23812882Sspwilson2@wisc.edu results = suite_result.aggregate_test_results() 23912882Sspwilson2@wisc.edu 24012882Sspwilson2@wisc.edu self.attributes = [ 24112882Sspwilson2@wisc.edu XMLAttribute('name', suite_result.name) 24212882Sspwilson2@wisc.edu ] 24312882Sspwilson2@wisc.edu for result, tests in results.items(): 24412882Sspwilson2@wisc.edu self.attributes.append(self.result_attribute(result, 24512882Sspwilson2@wisc.edu str(len(tests)))) 24612882Sspwilson2@wisc.edu 24712882Sspwilson2@wisc.edu self.elements = [] 24812882Sspwilson2@wisc.edu for test in suite_result: 24912882Sspwilson2@wisc.edu self.elements.append(JUnitTestCase(test)) 25012882Sspwilson2@wisc.edu 25112882Sspwilson2@wisc.edu def result_attribute(self, result, count): 25212882Sspwilson2@wisc.edu return XMLAttribute(self.result_map[result], count) 25312882Sspwilson2@wisc.edu 25412882Sspwilson2@wisc.educlass JUnitTestCase(XMLElement): 25512882Sspwilson2@wisc.edu name = 'testcase' 25612882Sspwilson2@wisc.edu def __init__(self, test_result): 25712882Sspwilson2@wisc.edu self.attributes = [ 25812882Sspwilson2@wisc.edu XMLAttribute('name', test_result.name), 25912882Sspwilson2@wisc.edu # TODO JUnit expects class of test.. add as test metadata. 26012882Sspwilson2@wisc.edu XMLAttribute('classname', str(test_result.uid)), 26112882Sspwilson2@wisc.edu XMLAttribute('status', str(test_result.result)), 26212882Sspwilson2@wisc.edu ] 26312882Sspwilson2@wisc.edu 26412882Sspwilson2@wisc.edu # TODO JUnit expects a message for the reason a test was 26512882Sspwilson2@wisc.edu # skipped or errored, save this with the test metadata. 26612882Sspwilson2@wisc.edu # http://llg.cubic.org/docs/junit/ 26712882Sspwilson2@wisc.edu self.elements = [ 26812882Sspwilson2@wisc.edu LargeFileElement('system-err', test_result.stderr), 26912882Sspwilson2@wisc.edu LargeFileElement('system-out', test_result.stdout), 27012882Sspwilson2@wisc.edu ] 27112882Sspwilson2@wisc.edu 27212882Sspwilson2@wisc.educlass LargeFileElement(XMLElement): 27312882Sspwilson2@wisc.edu def __init__(self, name, filename): 27412882Sspwilson2@wisc.edu self.name = name 27512882Sspwilson2@wisc.edu self.filename = filename 27612882Sspwilson2@wisc.edu self.attributes = [] 27712882Sspwilson2@wisc.edu 27812882Sspwilson2@wisc.edu def body(self, file_): 27912882Sspwilson2@wisc.edu try: 28012882Sspwilson2@wisc.edu with open(self.filename, 'r') as f: 28112882Sspwilson2@wisc.edu for line in f: 28212882Sspwilson2@wisc.edu file_.write(xml.sax.saxutils.escape(line)) 28312882Sspwilson2@wisc.edu except IOError: 28412882Sspwilson2@wisc.edu # TODO Better error logic, this is sometimes O.K. 28512882Sspwilson2@wisc.edu # if there was no stdout/stderr captured for the test 28612882Sspwilson2@wisc.edu # 28712882Sspwilson2@wisc.edu # TODO If that was the case, the file should still be made and it 28812882Sspwilson2@wisc.edu # should just be empty instead of not existing. 28912882Sspwilson2@wisc.edu pass 29012882Sspwilson2@wisc.edu 29112882Sspwilson2@wisc.edu 29212882Sspwilson2@wisc.edu 29312882Sspwilson2@wisc.educlass JUnitSavedResults: 29412882Sspwilson2@wisc.edu @staticmethod 29512882Sspwilson2@wisc.edu def save(results, path): 29612882Sspwilson2@wisc.edu ''' 29712882Sspwilson2@wisc.edu Compile the internal results into JUnit format writting it to the 29812882Sspwilson2@wisc.edu given file. 29912882Sspwilson2@wisc.edu ''' 30012882Sspwilson2@wisc.edu results = JUnitTestSuites(results) 30112882Sspwilson2@wisc.edu with open(path, 'w') as f: 30212882Sspwilson2@wisc.edu results.write(f) 30312882Sspwilson2@wisc.edu 304