regression.py revision 13540 
1#!/usr/bin/env python2.7
2
3# Copyright (c) 2010-2013 Advanced Micro Devices, Inc.
4# All rights reserved.
5#
6# Redistribution and use in source and binary forms, with or without
7# modification, are permitted provided that the following conditions are
8# met: redistributions of source code must retain the above copyright
9# notice, this list of conditions and the following disclaimer;
10# redistributions in binary form must reproduce the above copyright
11# notice, this list of conditions and the following disclaimer in the
12# documentation and/or other materials provided with the distribution;
13# neither the name of the copyright holders nor the names of its
14# contributors may be used to endorse or promote products derived from
15# this software without specific prior written permission.
16#
17# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
18# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
19# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
20# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
21# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
22# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
23# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
24# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
25# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
26# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
27# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
28
29"""
30SYNOPSIS
31
32    ./regression/regression.py ./regression/
33
34DESCRIPTION
35
36    Runs regression tester for McPAT.
37    This tester can compile and runs McPAT on the input contained in the
38    specified directory, and then compares the output to that of a prior run in
39    order to ensure that specific power and area calculations do not change.
40
41AUTHORS
42
43    Joel Hestness <hestness@cs.wisc.edu> (while interning at AMD)
44    Yasuko Eckert <yasuko.eckert@amd.com>
45"""
46
47import os
48import sys
49import optparse
50import re
51
52################################
53# Global Variables
54################################
55
56global mcpat_binary
57mcpat_binary = "../../build/mcpat/mcpat"
58global optionsparser
59
60################################
61# Global Functions
62################################
63
64def run_test(testdir):
65    test_passed = True
66    testfiles = os.listdir(testdir)
67    for testfile in testfiles:
68        # For each power_region file, run McPAT on it and check the
69        # output created against the regression
70        if re.match("power_region.*\.xml$", testfile):
71            # Get the region index of the test
72            fileparts = testfile.split(".")
73            region_index = fileparts[0][12:]
74            regression_test = os.path.join(testdir, testfile)
75            regression_output = os.path.join(
76                    testdir, "region%s.out" % region_index)
77            regression_correct = os.path.join(
78                    testdir, "region%s.out.ref" % region_index)
79            print "Running test: %s..." % regression_test
80            # Run McPAT on the input
81            os.system(
82                    "%s -infile %s -print_level 10 > %s" %
83                    (mcpat_binary, regression_test, regression_output) )
84            if os.path.exists(regression_correct):
85                diff = os.popen(
86                        "diff %s %s" % (regression_output, regression_correct),
87                        "r").read()
88                if diff != "":
89                    print "WARN: Differences found in %s" % regression_output
90                    if options.verbose:
91                        print diff
92                    test_passed = False
93            else:
94                print "WARN: Regression test not set up: %s..." % regression_test
95                print "WARN: Not able to verify test"
96                test_passed = False
97
98            if options.cleanup:
99                if options.verbose:
100                    print "WARN: Cleaning (deleting) regression output file: "\
101                            "%s" % regression_output
102                os.system("rm -f %s" % regression_output)
103
104    if test_passed:
105        print "PASSED: %s\n\n" % testdir
106    else:
107        print "FAILED: %s\n\n" % testdir
108
109def has_power_region_files(testdir):
110    files = os.listdir(testdir)
111    for file in files:
112        if "power_region" in file and ".xml" in file:
113            return True
114
115def is_valid_test_directory(testdir):
116    valid_regression = True
117    power_region_file_found = False
118
119    files = os.listdir(testdir)
120    for file in files:
121        if "power_region" in file and ".xml" in file:
122            power_region_file_found = True
123            fileparts = file.split(".")
124            region_index = fileparts[0][12:]
125            regression_output = os.path.join(
126                    testdir, "region%s.out.ref" % region_index)
127            if os.path.exists(regression_output):
128                if options.verbose:
129                    print "Valid regression test: %s/%s" % (testdir, file)
130            else:
131                valid_regression = False
132
133    return valid_regression and power_region_file_found
134
135################################
136# Execute here
137################################
138
139optionsparser = optparse.OptionParser(
140        formatter = optparse.TitledHelpFormatter(),
141        usage = globals()['__doc__'])
142optionsparser.add_option(
143        "-b", "--build", action = "store_true", default = False,
144        help = "Build McPAT before running tests")
145optionsparser.add_option(
146        "-c", "--cleanup", action = "store_true", default = False,
147        help = "Clean up the specified regression directory")
148optionsparser.add_option(
149        "-f", "--force", action = "store_true", default = False,
150        help = "Force run regression even if directory isn't set up")
151optionsparser.add_option(
152        "-m", "--maketest", action = "store_true", default = False,
153        help = "Set up the specified test directory")
154optionsparser.add_option(
155        "-v", "--verbose", action = "store_true", default = False,
156        help = "Print verbose output")
157(options, args) = optionsparser.parse_args()
158
159if not os.path.exists(mcpat_binary) and not options.build:
160    print "ERROR: McPAT binary does not exist: %s" % mcpat_binary
161    exit(0)
162
163if options.build:
164    print "Building McPAT..."
165    bin_dir = os.path.dirname(mcpat_binary)
166    directory = os.path.join(bin_dir, "../../ext/mcpat")
167    build_output = os.popen(
168            "cd %s; make clean; make -j 8 dbg 2>&1" % directory).read()
169    if "error" in build_output.lower():
170        print "Error during build: %s" % build_output
171        exit(0)
172
173if len(args) < 1:
174    print "ERROR: Must specify regressions directory"
175    exit(0)
176
177# check params
178rootdir = args[0];
179if not os.path.exists(rootdir):
180    print "ERROR: Regressions directory does not exist: %s" % rootdir
181    exit(0)
182
183if options.maketest:
184    # The specified rootdir must exist since we got here
185    # Check if directory has tests
186    list = os.listdir(rootdir)
187    found_test = False
188    for file in list:
189        if "power_region" in file and "out" not in file and "ref" not in file:
190            found_test = True
191            # Prepare to run the test in order to set it up
192            fileparts = file.split(".")
193            region_index = fileparts[0][12:]
194            regression_test = os.path.join(rootdir, file)
195            regression_output = os.path.join(
196                    rootdir, "region%s.out.ref" % region_index)
197            if os.path.exists(regression_output):
198                print "WARN: Overwriting old regression output: " \
199                        "%s" % regression_output
200            # Run the test to set it up
201            print "Writing new regression output..."
202            os.system(
203                    "%s -infile %s -print_level 10 > %s" %
204                    (mcpat_binary, regression_test, regression_output))
205
206    if not found_test:
207        print "ERROR: Invalid test directory: %s" % rootdir
208        print "ERROR: Must contain XML file power_region*.xml"
209
210    exit(0)
211
212found_test = False
213if has_power_region_files(rootdir):
214    found_test = True
215    if is_valid_test_directory(rootdir) or options.force:
216        run_test(rootdir)
217    else:
218        print "WARN: Regression directory is not set up: %s" % rootdir
219else:
220    folders = os.listdir(rootdir)
221    folders.sort()
222    for folder in folders:
223        testdir = os.path.join(rootdir, folder)
224        if os.path.isdir(testdir):
225            if has_power_region_files(testdir):
226                found_test = True
227                if is_valid_test_directory(testdir):
228                    run_test(testdir)
229                else:
230                    if options.force:
231                        print "WARN: Regression directory is not set up: " \
232                                "%s" % testdir
233                        print "WARN: Running test anyway: %s..." % testdir
234                        run_test(testdir)
235                    else:
236                        print "Regression directory is not set up: %s" % testdir
237            else:
238                print "Not a valid test directory: %s" % testdir
239
240if not found_test:
241    print "No valid regressions found in %s" % rootdir
242