regression.py revision 13540:da30e62884ee
113516Sgabeblack@google.com#!/usr/bin/env python2.7 213516Sgabeblack@google.com 313516Sgabeblack@google.com# Copyright (c) 2010-2013 Advanced Micro Devices, Inc. 413516Sgabeblack@google.com# All rights reserved. 513516Sgabeblack@google.com# 613516Sgabeblack@google.com# Redistribution and use in source and binary forms, with or without 713516Sgabeblack@google.com# modification, are permitted provided that the following conditions are 813516Sgabeblack@google.com# met: redistributions of source code must retain the above copyright 913516Sgabeblack@google.com# notice, this list of conditions and the following disclaimer; 1013516Sgabeblack@google.com# redistributions in binary form must reproduce the above copyright 1113516Sgabeblack@google.com# notice, this list of conditions and the following disclaimer in the 1213516Sgabeblack@google.com# documentation and/or other materials provided with the distribution; 1313516Sgabeblack@google.com# neither the name of the copyright holders nor the names of its 1413516Sgabeblack@google.com# contributors may be used to endorse or promote products derived from 1513516Sgabeblack@google.com# this software without specific prior written permission. 1613516Sgabeblack@google.com# 1713516Sgabeblack@google.com# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS 1813516Sgabeblack@google.com# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT 1913516Sgabeblack@google.com# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR 2013516Sgabeblack@google.com# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT 2113516Sgabeblack@google.com# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, 2213516Sgabeblack@google.com# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT 2313516Sgabeblack@google.com# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, 2413516Sgabeblack@google.com# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY 2513516Sgabeblack@google.com# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT 2613516Sgabeblack@google.com# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE 2713516Sgabeblack@google.com# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 2813516Sgabeblack@google.com 2913516Sgabeblack@google.com""" 3013516Sgabeblack@google.comSYNOPSIS 3113516Sgabeblack@google.com 3213516Sgabeblack@google.com ./regression/regression.py ./regression/ 3313516Sgabeblack@google.com 3413516Sgabeblack@google.comDESCRIPTION 3513516Sgabeblack@google.com 3613516Sgabeblack@google.com Runs regression tester for McPAT. 3713516Sgabeblack@google.com This tester can compile and runs McPAT on the input contained in the 3813516Sgabeblack@google.com specified directory, and then compares the output to that of a prior run in 3913516Sgabeblack@google.com order to ensure that specific power and area calculations do not change. 4013516Sgabeblack@google.com 4113516Sgabeblack@google.comAUTHORS 4213516Sgabeblack@google.com 4313516Sgabeblack@google.com Joel Hestness <hestness@cs.wisc.edu> (while interning at AMD) 4413516Sgabeblack@google.com Yasuko Eckert <yasuko.eckert@amd.com> 4513516Sgabeblack@google.com""" 4613516Sgabeblack@google.com 4713516Sgabeblack@google.comimport os 4813516Sgabeblack@google.comimport sys 4913516Sgabeblack@google.comimport optparse 5013516Sgabeblack@google.comimport re 5113516Sgabeblack@google.com 5213516Sgabeblack@google.com################################ 5313516Sgabeblack@google.com# Global Variables 5413516Sgabeblack@google.com################################ 5513516Sgabeblack@google.com 5613516Sgabeblack@google.comglobal mcpat_binary 5713516Sgabeblack@google.commcpat_binary = "../../build/mcpat/mcpat" 5813516Sgabeblack@google.comglobal optionsparser 5913516Sgabeblack@google.com 6013516Sgabeblack@google.com################################ 6113516Sgabeblack@google.com# Global Functions 6213516Sgabeblack@google.com################################ 6313516Sgabeblack@google.com 6413516Sgabeblack@google.comdef run_test(testdir): 6513516Sgabeblack@google.com test_passed = True 6613516Sgabeblack@google.com testfiles = os.listdir(testdir) 6713516Sgabeblack@google.com for testfile in testfiles: 6813516Sgabeblack@google.com # For each power_region file, run McPAT on it and check the 6913516Sgabeblack@google.com # output created against the regression 7013516Sgabeblack@google.com if re.match("power_region.*\.xml$", testfile): 7113516Sgabeblack@google.com # Get the region index of the test 7213516Sgabeblack@google.com fileparts = testfile.split(".") 7313516Sgabeblack@google.com region_index = fileparts[0][12:] 7413516Sgabeblack@google.com regression_test = os.path.join(testdir, testfile) 7513516Sgabeblack@google.com regression_output = os.path.join( 7613516Sgabeblack@google.com testdir, "region%s.out" % region_index) 7713516Sgabeblack@google.com regression_correct = os.path.join( 7813516Sgabeblack@google.com testdir, "region%s.out.ref" % region_index) 7913516Sgabeblack@google.com print "Running test: %s..." % regression_test 8013516Sgabeblack@google.com # Run McPAT on the input 8113516Sgabeblack@google.com os.system( 8213516Sgabeblack@google.com "%s -infile %s -print_level 10 > %s" % 8313516Sgabeblack@google.com (mcpat_binary, regression_test, regression_output) ) 8413516Sgabeblack@google.com if os.path.exists(regression_correct): 8513516Sgabeblack@google.com diff = os.popen( 8613516Sgabeblack@google.com "diff %s %s" % (regression_output, regression_correct), 8713516Sgabeblack@google.com "r").read() 8813516Sgabeblack@google.com if diff != "": 8913516Sgabeblack@google.com print "WARN: Differences found in %s" % regression_output 9013516Sgabeblack@google.com if options.verbose: 9113516Sgabeblack@google.com print diff 9213516Sgabeblack@google.com test_passed = False 9313516Sgabeblack@google.com else: 9413516Sgabeblack@google.com print "WARN: Regression test not set up: %s..." % regression_test 9513516Sgabeblack@google.com print "WARN: Not able to verify test" 9613516Sgabeblack@google.com test_passed = False 9713516Sgabeblack@google.com 9813516Sgabeblack@google.com if options.cleanup: 9913516Sgabeblack@google.com if options.verbose: 10013516Sgabeblack@google.com print "WARN: Cleaning (deleting) regression output file: "\ 10113516Sgabeblack@google.com "%s" % regression_output 10213516Sgabeblack@google.com os.system("rm -f %s" % regression_output) 10313516Sgabeblack@google.com 10413516Sgabeblack@google.com if test_passed: 10513516Sgabeblack@google.com print "PASSED: %s\n\n" % testdir 10613516Sgabeblack@google.com else: 10713516Sgabeblack@google.com print "FAILED: %s\n\n" % testdir 10813516Sgabeblack@google.com 10913516Sgabeblack@google.comdef has_power_region_files(testdir): 11013516Sgabeblack@google.com files = os.listdir(testdir) 11113516Sgabeblack@google.com for file in files: 11213516Sgabeblack@google.com if "power_region" in file and ".xml" in file: 11313516Sgabeblack@google.com return True 11413516Sgabeblack@google.com 11513516Sgabeblack@google.comdef is_valid_test_directory(testdir): 11613516Sgabeblack@google.com valid_regression = True 11713516Sgabeblack@google.com power_region_file_found = False 11813516Sgabeblack@google.com 11913516Sgabeblack@google.com files = os.listdir(testdir) 12013516Sgabeblack@google.com for file in files: 12113516Sgabeblack@google.com if "power_region" in file and ".xml" in file: 12213516Sgabeblack@google.com power_region_file_found = True 12313516Sgabeblack@google.com fileparts = file.split(".") 12413516Sgabeblack@google.com region_index = fileparts[0][12:] 12513516Sgabeblack@google.com regression_output = os.path.join( 12613516Sgabeblack@google.com testdir, "region%s.out.ref" % region_index) 12713516Sgabeblack@google.com if os.path.exists(regression_output): 12813516Sgabeblack@google.com if options.verbose: 12913516Sgabeblack@google.com print "Valid regression test: %s/%s" % (testdir, file) 13013516Sgabeblack@google.com else: 13113516Sgabeblack@google.com valid_regression = False 13213516Sgabeblack@google.com 13313516Sgabeblack@google.com return valid_regression and power_region_file_found 13413516Sgabeblack@google.com 13513516Sgabeblack@google.com################################ 13613516Sgabeblack@google.com# Execute here 13713516Sgabeblack@google.com################################ 13813516Sgabeblack@google.com 13913516Sgabeblack@google.comoptionsparser = optparse.OptionParser( 14013516Sgabeblack@google.com formatter = optparse.TitledHelpFormatter(), 14113516Sgabeblack@google.com usage = globals()['__doc__']) 14213516Sgabeblack@google.comoptionsparser.add_option( 14313516Sgabeblack@google.com "-b", "--build", action = "store_true", default = False, 14413516Sgabeblack@google.com help = "Build McPAT before running tests") 14513516Sgabeblack@google.comoptionsparser.add_option( 14613516Sgabeblack@google.com "-c", "--cleanup", action = "store_true", default = False, 14713516Sgabeblack@google.com help = "Clean up the specified regression directory") 14813516Sgabeblack@google.comoptionsparser.add_option( 14913516Sgabeblack@google.com "-f", "--force", action = "store_true", default = False, 15013516Sgabeblack@google.com help = "Force run regression even if directory isn't set up") 15113516Sgabeblack@google.comoptionsparser.add_option( 15213516Sgabeblack@google.com "-m", "--maketest", action = "store_true", default = False, 15313516Sgabeblack@google.com help = "Set up the specified test directory") 15413516Sgabeblack@google.comoptionsparser.add_option( 15513516Sgabeblack@google.com "-v", "--verbose", action = "store_true", default = False, 15613516Sgabeblack@google.com help = "Print verbose output") 15713516Sgabeblack@google.com(options, args) = optionsparser.parse_args() 15813516Sgabeblack@google.com 15913516Sgabeblack@google.comif not os.path.exists(mcpat_binary) and not options.build: 16013516Sgabeblack@google.com print "ERROR: McPAT binary does not exist: %s" % mcpat_binary 16113516Sgabeblack@google.com exit(0) 16213516Sgabeblack@google.com 16313516Sgabeblack@google.comif options.build: 16413516Sgabeblack@google.com print "Building McPAT..." 16513516Sgabeblack@google.com bin_dir = os.path.dirname(mcpat_binary) 16613516Sgabeblack@google.com directory = os.path.join(bin_dir, "../../ext/mcpat") 16713516Sgabeblack@google.com build_output = os.popen( 16813516Sgabeblack@google.com "cd %s; make clean; make -j 8 dbg 2>&1" % directory).read() 16913516Sgabeblack@google.com if "error" in build_output.lower(): 17013516Sgabeblack@google.com print "Error during build: %s" % build_output 17113516Sgabeblack@google.com exit(0) 17213516Sgabeblack@google.com 17313516Sgabeblack@google.comif len(args) < 1: 17413516Sgabeblack@google.com print "ERROR: Must specify regressions directory" 17513516Sgabeblack@google.com exit(0) 17613516Sgabeblack@google.com 17713516Sgabeblack@google.com# check params 17813516Sgabeblack@google.comrootdir = args[0]; 17913516Sgabeblack@google.comif not os.path.exists(rootdir): 18013516Sgabeblack@google.com print "ERROR: Regressions directory does not exist: %s" % rootdir 18113516Sgabeblack@google.com exit(0) 18213516Sgabeblack@google.com 18313516Sgabeblack@google.comif options.maketest: 18413516Sgabeblack@google.com # The specified rootdir must exist since we got here 18513516Sgabeblack@google.com # Check if directory has tests 18613516Sgabeblack@google.com list = os.listdir(rootdir) 18713516Sgabeblack@google.com found_test = False 18813516Sgabeblack@google.com for file in list: 18913516Sgabeblack@google.com if "power_region" in file and "out" not in file and "ref" not in file: 19013516Sgabeblack@google.com found_test = True 19113516Sgabeblack@google.com # Prepare to run the test in order to set it up 19213516Sgabeblack@google.com fileparts = file.split(".") 19313516Sgabeblack@google.com region_index = fileparts[0][12:] 19413516Sgabeblack@google.com regression_test = os.path.join(rootdir, file) 19513516Sgabeblack@google.com regression_output = os.path.join( 19613516Sgabeblack@google.com rootdir, "region%s.out.ref" % region_index) 19713516Sgabeblack@google.com if os.path.exists(regression_output): 19813516Sgabeblack@google.com print "WARN: Overwriting old regression output: " \ 19913516Sgabeblack@google.com "%s" % regression_output 20013516Sgabeblack@google.com # Run the test to set it up 20113516Sgabeblack@google.com print "Writing new regression output..." 20213516Sgabeblack@google.com os.system( 20313516Sgabeblack@google.com "%s -infile %s -print_level 10 > %s" % 20413516Sgabeblack@google.com (mcpat_binary, regression_test, regression_output)) 20513516Sgabeblack@google.com 20613516Sgabeblack@google.com if not found_test: 20713516Sgabeblack@google.com print "ERROR: Invalid test directory: %s" % rootdir 20813516Sgabeblack@google.com print "ERROR: Must contain XML file power_region*.xml" 20913516Sgabeblack@google.com 21013516Sgabeblack@google.com exit(0) 21113516Sgabeblack@google.com 21213516Sgabeblack@google.comfound_test = False 21313516Sgabeblack@google.comif has_power_region_files(rootdir): 21413516Sgabeblack@google.com found_test = True 21513516Sgabeblack@google.com if is_valid_test_directory(rootdir) or options.force: 21613516Sgabeblack@google.com run_test(rootdir) 21713516Sgabeblack@google.com else: 21813516Sgabeblack@google.com print "WARN: Regression directory is not set up: %s" % rootdir 21913516Sgabeblack@google.comelse: 22013516Sgabeblack@google.com folders = os.listdir(rootdir) 22113516Sgabeblack@google.com folders.sort() 22213516Sgabeblack@google.com for folder in folders: 22313516Sgabeblack@google.com testdir = os.path.join(rootdir, folder) 22413516Sgabeblack@google.com if os.path.isdir(testdir): 22513516Sgabeblack@google.com if has_power_region_files(testdir): 22613516Sgabeblack@google.com found_test = True 22713516Sgabeblack@google.com if is_valid_test_directory(testdir): 22813516Sgabeblack@google.com run_test(testdir) 22913516Sgabeblack@google.com else: 23013516Sgabeblack@google.com if options.force: 23113516Sgabeblack@google.com print "WARN: Regression directory is not set up: " \ 23213516Sgabeblack@google.com "%s" % testdir 23313516Sgabeblack@google.com print "WARN: Running test anyway: %s..." % testdir 23413516Sgabeblack@google.com run_test(testdir) 23513516Sgabeblack@google.com else: 23613516Sgabeblack@google.com print "Regression directory is not set up: %s" % testdir 23713516Sgabeblack@google.com else: 23813516Sgabeblack@google.com print "Not a valid test directory: %s" % testdir 23913516Sgabeblack@google.com 24013516Sgabeblack@google.comif not found_test: 24113516Sgabeblack@google.com print "No valid regressions found in %s" % rootdir 24213516Sgabeblack@google.com