result.py revision 13789:d7b2be2c468b
14826Ssaidi@eecs.umich.edu# Copyright (c) 2017 Mark D. Hill and David A. Wood 24826Ssaidi@eecs.umich.edu# All rights reserved. 34826Ssaidi@eecs.umich.edu# 44826Ssaidi@eecs.umich.edu# Redistribution and use in source and binary forms, with or without 54826Ssaidi@eecs.umich.edu# modification, are permitted provided that the following conditions are 64826Ssaidi@eecs.umich.edu# met: redistributions of source code must retain the above copyright 74826Ssaidi@eecs.umich.edu# notice, this list of conditions and the following disclaimer; 84826Ssaidi@eecs.umich.edu# redistributions in binary form must reproduce the above copyright 94826Ssaidi@eecs.umich.edu# notice, this list of conditions and the following disclaimer in the 104826Ssaidi@eecs.umich.edu# documentation and/or other materials provided with the distribution; 114826Ssaidi@eecs.umich.edu# neither the name of the copyright holders nor the names of its 124826Ssaidi@eecs.umich.edu# contributors may be used to endorse or promote products derived from 134826Ssaidi@eecs.umich.edu# this software without specific prior written permission. 144826Ssaidi@eecs.umich.edu# 154826Ssaidi@eecs.umich.edu# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS 164826Ssaidi@eecs.umich.edu# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT 174826Ssaidi@eecs.umich.edu# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR 184826Ssaidi@eecs.umich.edu# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT 194826Ssaidi@eecs.umich.edu# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, 204826Ssaidi@eecs.umich.edu# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT 214826Ssaidi@eecs.umich.edu# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, 224826Ssaidi@eecs.umich.edu# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY 234826Ssaidi@eecs.umich.edu# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT 244826Ssaidi@eecs.umich.edu# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE 254826Ssaidi@eecs.umich.edu# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 264826Ssaidi@eecs.umich.edu# 274826Ssaidi@eecs.umich.edu# Authors: Sean Wilson 284826Ssaidi@eecs.umich.edu 294826Ssaidi@eecs.umich.eduimport os 304826Ssaidi@eecs.umich.eduimport pickle 314826Ssaidi@eecs.umich.eduimport xml.sax.saxutils 327678Sgblack@eecs.umich.edu 334826Ssaidi@eecs.umich.edufrom config import config 344826Ssaidi@eecs.umich.eduimport helper 354826Ssaidi@eecs.umich.eduimport state 368706Sandreas.hansson@arm.comimport log 374826Ssaidi@eecs.umich.edu 384826Ssaidi@eecs.umich.edudef _create_uid_index(iterable): 394826Ssaidi@eecs.umich.edu index = {} 404826Ssaidi@eecs.umich.edu for item in iterable: 414826Ssaidi@eecs.umich.edu assert item.uid not in index 427741Sgblack@eecs.umich.edu index[item.uid] = item 437741Sgblack@eecs.umich.edu return index 447741Sgblack@eecs.umich.edu 457741Sgblack@eecs.umich.edu 467741Sgblack@eecs.umich.educlass _CommonMetadataMixin: 477741Sgblack@eecs.umich.edu @property 487707Sgblack@eecs.umich.edu def name(self): 497707Sgblack@eecs.umich.edu return self._metadata.name 507707Sgblack@eecs.umich.edu @property 514826Ssaidi@eecs.umich.edu def uid(self): 525958Sgblack@eecs.umich.edu return self._metadata.uid 534826Ssaidi@eecs.umich.edu @property 545958Sgblack@eecs.umich.edu def result(self): 554826Ssaidi@eecs.umich.edu return self._metadata.result 564826Ssaidi@eecs.umich.edu @result.setter 578706Sandreas.hansson@arm.com def result(self, result): 584826Ssaidi@eecs.umich.edu self._metadata.result = result 594826Ssaidi@eecs.umich.edu 604826Ssaidi@eecs.umich.edu @property 614826Ssaidi@eecs.umich.edu def unsuccessful(self): 624826Ssaidi@eecs.umich.edu return self._metadata.result.value != state.Result.Passed 634826Ssaidi@eecs.umich.edu 644826Ssaidi@eecs.umich.edu 654826Ssaidi@eecs.umich.educlass InternalTestResult(object, _CommonMetadataMixin): 664826Ssaidi@eecs.umich.edu def __init__(self, obj, suite, directory): 676329Sgblack@eecs.umich.edu self._metadata = obj.metadata 686329Sgblack@eecs.umich.edu self.suite = suite 696329Sgblack@eecs.umich.edu 706329Sgblack@eecs.umich.edu self.stderr = os.path.join( 716329Sgblack@eecs.umich.edu InternalSavedResults.output_path(self.uid, suite.uid), 726329Sgblack@eecs.umich.edu 'stderr' 736329Sgblack@eecs.umich.edu ) 746329Sgblack@eecs.umich.edu self.stdout = os.path.join( 757741Sgblack@eecs.umich.edu InternalSavedResults.output_path(self.uid, suite.uid), 766329Sgblack@eecs.umich.edu 'stdout' 776329Sgblack@eecs.umich.edu ) 786329Sgblack@eecs.umich.edu 797741Sgblack@eecs.umich.edu 807741Sgblack@eecs.umich.educlass InternalSuiteResult(object, _CommonMetadataMixin): 817741Sgblack@eecs.umich.edu def __init__(self, obj, directory): 827741Sgblack@eecs.umich.edu self._metadata = obj.metadata 837741Sgblack@eecs.umich.edu self.directory = directory 847741Sgblack@eecs.umich.edu self._wrap_tests(obj) 857741Sgblack@eecs.umich.edu 867741Sgblack@eecs.umich.edu def _wrap_tests(self, obj): 876329Sgblack@eecs.umich.edu self._tests = [InternalTestResult(test, self, self.directory) 886329Sgblack@eecs.umich.edu for test in obj] 896329Sgblack@eecs.umich.edu self._tests_index = _create_uid_index(self._tests) 906329Sgblack@eecs.umich.edu 916329Sgblack@eecs.umich.edu def get_test(self, uid): 926329Sgblack@eecs.umich.edu return self._tests_index[uid] 936329Sgblack@eecs.umich.edu 946329Sgblack@eecs.umich.edu def __iter__(self): 957741Sgblack@eecs.umich.edu return iter(self._tests) 967741Sgblack@eecs.umich.edu 977741Sgblack@eecs.umich.edu def get_test_result(self, uid): 987741Sgblack@eecs.umich.edu return self.get_test(uid) 997741Sgblack@eecs.umich.edu 1007741Sgblack@eecs.umich.edu def aggregate_test_results(self): 1017741Sgblack@eecs.umich.edu results = {} 1027741Sgblack@eecs.umich.edu for test in self: 1037741Sgblack@eecs.umich.edu helper.append_dictlist(results, test.result.value, test) 1047741Sgblack@eecs.umich.edu return results 1057741Sgblack@eecs.umich.edu 1067741Sgblack@eecs.umich.edu 1077741Sgblack@eecs.umich.educlass InternalLibraryResults(object, _CommonMetadataMixin): 1087741Sgblack@eecs.umich.edu def __init__(self, obj, directory): 1097741Sgblack@eecs.umich.edu self.directory = directory 1107741Sgblack@eecs.umich.edu self._metadata = obj.metadata 1117741Sgblack@eecs.umich.edu self._wrap_suites(obj) 1127741Sgblack@eecs.umich.edu 1136329Sgblack@eecs.umich.edu def __iter__(self): 1146329Sgblack@eecs.umich.edu return iter(self._suites) 1157741Sgblack@eecs.umich.edu 1167741Sgblack@eecs.umich.edu def _wrap_suites(self, obj): 1177741Sgblack@eecs.umich.edu self._suites = [InternalSuiteResult(suite, self.directory) 1187741Sgblack@eecs.umich.edu for suite in obj] 1197741Sgblack@eecs.umich.edu self._suites_index = _create_uid_index(self._suites) 1207741Sgblack@eecs.umich.edu 1217741Sgblack@eecs.umich.edu def add_suite(self, suite): 1227741Sgblack@eecs.umich.edu if suite.uid in self._suites: 1237741Sgblack@eecs.umich.edu raise ValueError('Cannot have duplicate suite UIDs.') 1247741Sgblack@eecs.umich.edu self._suites[suite.uid] = suite 1257741Sgblack@eecs.umich.edu 1267741Sgblack@eecs.umich.edu def get_suite_result(self, suite_uid): 1277741Sgblack@eecs.umich.edu return self._suites_index[suite_uid] 1287741Sgblack@eecs.umich.edu 1297741Sgblack@eecs.umich.edu def get_test_result(self, test_uid, suite_uid): 1307741Sgblack@eecs.umich.edu return self.get_suite_result(suite_uid).get_test_result(test_uid) 1317741Sgblack@eecs.umich.edu 1327741Sgblack@eecs.umich.edu def aggregate_test_results(self): 1337741Sgblack@eecs.umich.edu results = {} 1347741Sgblack@eecs.umich.edu for suite in self._suites: 1356337Sgblack@eecs.umich.edu for test in suite: 1366329Sgblack@eecs.umich.edu helper.append_dictlist(results, test.result.value, test) 1376329Sgblack@eecs.umich.edu return results 1387741Sgblack@eecs.umich.edu 1397741Sgblack@eecs.umich.educlass InternalSavedResults: 1407741Sgblack@eecs.umich.edu @staticmethod 1417741Sgblack@eecs.umich.edu def output_path(test_uid, suite_uid, base=None): 1427741Sgblack@eecs.umich.edu ''' 1437741Sgblack@eecs.umich.edu Return the path which results for a specific test case should be 1446329Sgblack@eecs.umich.edu stored. 1456329Sgblack@eecs.umich.edu ''' 1466329Sgblack@eecs.umich.edu if base is None: 1476329Sgblack@eecs.umich.edu base = config.result_path 1486329Sgblack@eecs.umich.edu return os.path.join( 1496329Sgblack@eecs.umich.edu base, 1507741Sgblack@eecs.umich.edu str(suite_uid).replace(os.path.sep, '-'), 1517741Sgblack@eecs.umich.edu str(test_uid).replace(os.path.sep, '-')) 1526329Sgblack@eecs.umich.edu 1537741Sgblack@eecs.umich.edu @staticmethod 1546329Sgblack@eecs.umich.edu def save(results, path, protocol=pickle.HIGHEST_PROTOCOL): 1556329Sgblack@eecs.umich.edu if not os.path.exists(os.path.dirname(path)): 1566329Sgblack@eecs.umich.edu try: 1576329Sgblack@eecs.umich.edu os.makedirs(os.path.dirname(path)) 1586329Sgblack@eecs.umich.edu except OSError as exc: # Guard against race condition 1596329Sgblack@eecs.umich.edu if exc.errno != errno.EEXIST: 1606329Sgblack@eecs.umich.edu raise 1616329Sgblack@eecs.umich.edu 1626329Sgblack@eecs.umich.edu with open(path, 'w') as f: 1636329Sgblack@eecs.umich.edu pickle.dump(results, f, protocol) 1646329Sgblack@eecs.umich.edu 1656329Sgblack@eecs.umich.edu @staticmethod 1666329Sgblack@eecs.umich.edu def load(path): 1676329Sgblack@eecs.umich.edu with open(path, 'r') as f: 1686329Sgblack@eecs.umich.edu return pickle.load(f) 1696329Sgblack@eecs.umich.edu 1706329Sgblack@eecs.umich.edu 1716329Sgblack@eecs.umich.educlass XMLElement(object): 1726329Sgblack@eecs.umich.edu def write(self, file_): 1736329Sgblack@eecs.umich.edu self.begin(file_) 1746329Sgblack@eecs.umich.edu self.end(file_) 1756329Sgblack@eecs.umich.edu 1766329Sgblack@eecs.umich.edu def begin(self, file_): 1776329Sgblack@eecs.umich.edu file_.write('<') 1786329Sgblack@eecs.umich.edu file_.write(self.name) 1796329Sgblack@eecs.umich.edu for attr in self.attributes: 1806329Sgblack@eecs.umich.edu file_.write(' ') 1816329Sgblack@eecs.umich.edu attr.write(file_) 1826329Sgblack@eecs.umich.edu file_.write('>') 1836329Sgblack@eecs.umich.edu 1846329Sgblack@eecs.umich.edu self.body(file_) 1856329Sgblack@eecs.umich.edu 1866329Sgblack@eecs.umich.edu def body(self, file_): 1876329Sgblack@eecs.umich.edu for elem in self.elements: 1886329Sgblack@eecs.umich.edu file_.write('\n') 1896329Sgblack@eecs.umich.edu elem.write(file_) 1906329Sgblack@eecs.umich.edu file_.write('\n') 1916329Sgblack@eecs.umich.edu 1926329Sgblack@eecs.umich.edu def end(self, file_): 1936329Sgblack@eecs.umich.edu file_.write('</%s>' % self.name) 1946329Sgblack@eecs.umich.edu 1956329Sgblack@eecs.umich.educlass XMLAttribute(object): 1966329Sgblack@eecs.umich.edu def __init__(self, name, value): 1976329Sgblack@eecs.umich.edu self.name = name 1986329Sgblack@eecs.umich.edu self.value = value 1996329Sgblack@eecs.umich.edu 2006329Sgblack@eecs.umich.edu def write(self, file_): 2016329Sgblack@eecs.umich.edu file_.write('%s=%s' % (self.name, 2026329Sgblack@eecs.umich.edu xml.sax.saxutils.quoteattr(self.value))) 2036329Sgblack@eecs.umich.edu 2046329Sgblack@eecs.umich.edu 2056329Sgblack@eecs.umich.educlass JUnitTestSuites(XMLElement): 2066329Sgblack@eecs.umich.edu name = 'testsuites' 2077741Sgblack@eecs.umich.edu result_map = { 2086329Sgblack@eecs.umich.edu state.Result.Errored: 'errors', 2096329Sgblack@eecs.umich.edu state.Result.Failed: 'failures', 2107741Sgblack@eecs.umich.edu state.Result.Passed: 'tests' 2116329Sgblack@eecs.umich.edu } 2126337Sgblack@eecs.umich.edu 2136337Sgblack@eecs.umich.edu def __init__(self, internal_results): 2146329Sgblack@eecs.umich.edu results = internal_results.aggregate_test_results() 2156329Sgblack@eecs.umich.edu 2166329Sgblack@eecs.umich.edu self.attributes = [] 2176329Sgblack@eecs.umich.edu for result, tests in results.items(): 2187741Sgblack@eecs.umich.edu self.attributes.append(self.result_attribute(result, 2196329Sgblack@eecs.umich.edu str(len(tests)))) 2206337Sgblack@eecs.umich.edu 2216337Sgblack@eecs.umich.edu self.elements = [] 2226329Sgblack@eecs.umich.edu for suite in internal_results: 2236329Sgblack@eecs.umich.edu self.elements.append(JUnitTestSuite(suite)) 2246329Sgblack@eecs.umich.edu 2257741Sgblack@eecs.umich.edu def result_attribute(self, result, count): 2266329Sgblack@eecs.umich.edu return XMLAttribute(self.result_map[result], count) 2276329Sgblack@eecs.umich.edu 2286329Sgblack@eecs.umich.educlass JUnitTestSuite(JUnitTestSuites): 2297741Sgblack@eecs.umich.edu name = 'testsuite' 2306337Sgblack@eecs.umich.edu result_map = { 2316337Sgblack@eecs.umich.edu state.Result.Errored: 'errors', 2326329Sgblack@eecs.umich.edu state.Result.Failed: 'failures', 2336329Sgblack@eecs.umich.edu state.Result.Passed: 'tests', 2346329Sgblack@eecs.umich.edu state.Result.Skipped: 'skipped' 2356329Sgblack@eecs.umich.edu } 2366329Sgblack@eecs.umich.edu 2376329Sgblack@eecs.umich.edu def __init__(self, suite_result): 2386329Sgblack@eecs.umich.edu results = suite_result.aggregate_test_results() 2396329Sgblack@eecs.umich.edu 2406329Sgblack@eecs.umich.edu self.attributes = [ 2416329Sgblack@eecs.umich.edu XMLAttribute('name', suite_result.name) 2426329Sgblack@eecs.umich.edu ] 2437720Sgblack@eecs.umich.edu for result, tests in results.items(): 2446329Sgblack@eecs.umich.edu self.attributes.append(self.result_attribute(result, 2457678Sgblack@eecs.umich.edu str(len(tests)))) 2467678Sgblack@eecs.umich.edu 2477693SAli.Saidi@ARM.com self.elements = [] 2487693SAli.Saidi@ARM.com for test in suite_result: 2497720Sgblack@eecs.umich.edu self.elements.append(JUnitTestCase(test)) 2507720Sgblack@eecs.umich.edu 2517720Sgblack@eecs.umich.edu def result_attribute(self, result, count): 2527693SAli.Saidi@ARM.com return XMLAttribute(self.result_map[result], count) 2537693SAli.Saidi@ARM.com 2547693SAli.Saidi@ARM.comclass JUnitTestCase(XMLElement): 2557693SAli.Saidi@ARM.com name = 'testcase' 2567678Sgblack@eecs.umich.edu def __init__(self, test_result): 2577678Sgblack@eecs.umich.edu self.attributes = [ 2587678Sgblack@eecs.umich.edu XMLAttribute('name', test_result.name), 2597678Sgblack@eecs.umich.edu # TODO JUnit expects class of test.. add as test metadata. 2607678Sgblack@eecs.umich.edu XMLAttribute('classname', str(test_result.uid)), 2617678Sgblack@eecs.umich.edu XMLAttribute('status', str(test_result.result)), 2627678Sgblack@eecs.umich.edu ] 2637741Sgblack@eecs.umich.edu 264 # TODO JUnit expects a message for the reason a test was 265 # skipped or errored, save this with the test metadata. 266 # http://llg.cubic.org/docs/junit/ 267 self.elements = [ 268 LargeFileElement('system-err', test_result.stderr), 269 LargeFileElement('system-out', test_result.stdout), 270 ] 271 272class LargeFileElement(XMLElement): 273 def __init__(self, name, filename): 274 self.name = name 275 self.filename = filename 276 self.attributes = [] 277 278 def body(self, file_): 279 try: 280 with open(self.filename, 'r') as f: 281 for line in f: 282 file_.write(xml.sax.saxutils.escape(line)) 283 except IOError: 284 # TODO Better error logic, this is sometimes O.K. 285 # if there was no stdout/stderr captured for the test 286 # 287 # TODO If that was the case, the file should still be made and it 288 # should just be empty instead of not existing. 289 pass 290 291 292 293class JUnitSavedResults: 294 @staticmethod 295 def save(results, path): 296 ''' 297 Compile the internal results into JUnit format writting it to the 298 given file. 299 ''' 300 results = JUnitTestSuites(results) 301 with open(path, 'w') as f: 302 results.write(f) 303 304