diff-out revision 8838
12929Sktlim@umich.edu#!/usr/bin/perl
22929Sktlim@umich.edu# Copyright (c) 2001-2005 The Regents of The University of Michigan
32929Sktlim@umich.edu# All rights reserved.
42929Sktlim@umich.edu#
52929Sktlim@umich.edu# Redistribution and use in source and binary forms, with or without
62929Sktlim@umich.edu# modification, are permitted provided that the following conditions are
72929Sktlim@umich.edu# met: redistributions of source code must retain the above copyright
82929Sktlim@umich.edu# notice, this list of conditions and the following disclaimer;
92929Sktlim@umich.edu# redistributions in binary form must reproduce the above copyright
102929Sktlim@umich.edu# notice, this list of conditions and the following disclaimer in the
112929Sktlim@umich.edu# documentation and/or other materials provided with the distribution;
122929Sktlim@umich.edu# neither the name of the copyright holders nor the names of its
132929Sktlim@umich.edu# contributors may be used to endorse or promote products derived from
142929Sktlim@umich.edu# this software without specific prior written permission.
152929Sktlim@umich.edu#
162929Sktlim@umich.edu# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
172929Sktlim@umich.edu# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
182929Sktlim@umich.edu# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
192929Sktlim@umich.edu# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
202929Sktlim@umich.edu# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
212929Sktlim@umich.edu# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
222929Sktlim@umich.edu# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
232929Sktlim@umich.edu# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
242929Sktlim@umich.edu# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
252929Sktlim@umich.edu# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
262929Sktlim@umich.edu# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
272929Sktlim@umich.edu#
282929Sktlim@umich.edu# Authors: Steve Reinhardt
292929Sktlim@umich.edu
302929Sktlim@umich.edu#
312929Sktlim@umich.edu# This script diffs two SimpleScalar statistics output files.
322929Sktlim@umich.edu#
332929Sktlim@umich.edu
342929Sktlim@umich.eduuse Getopt::Std;
352929Sktlim@umich.edu
367686Ssteve.reinhardt@amd.comgetopts('adn:t:h');
372929Sktlim@umich.edu
382929Sktlim@umich.eduif ($#ARGV < 1)
392929Sktlim@umich.edu{
402929Sktlim@umich.edu    print "\nError: need two file arguments (<reference> <new>).\n";
417686Ssteve.reinhardt@amd.com    print "   Options: -d = Ignore distributions\n";
427686Ssteve.reinhardt@amd.com    print "            -a = Sort errors alphabetically (default: by percentage)\n";
437686Ssteve.reinhardt@amd.com    print "            -h = Diff header info separately from stats\n";
447686Ssteve.reinhardt@amd.com    print "            -n <num> = Print top <num> errors (default 20, 0 for all)\n";
457686Ssteve.reinhardt@amd.com    print "            -t <num> = Ignore errors below <num> percent (default 0)\n\n";
467686Ssteve.reinhardt@amd.com    exit;
472929Sktlim@umich.edu}
482929Sktlim@umich.edu
492929Sktlim@umich.eduopen(REF, "<$ARGV[0]") or die "Error: can't open $ARGV[0].\n";
502929Sktlim@umich.eduopen(NEW, "<$ARGV[1]") or die "Error: can't open $ARGV[1].\n";
512929Sktlim@umich.edu
522929Sktlim@umich.edu
532929Sktlim@umich.edu#
542929Sktlim@umich.edu# Things that really should be adjustable via the command line
552929Sktlim@umich.edu#
562929Sktlim@umich.edu
572929Sktlim@umich.edu# Ignorable error (in percent)
587686Ssteve.reinhardt@amd.com$err_thresh = defined($opt_t) ? $opt_t : 0;
592929Sktlim@umich.edu
602929Sktlim@umich.edu# Number of stats to print before omitting
617686Ssteve.reinhardt@amd.com$omit_count = defined($opt_n) ? $opt_n : 20;
622929Sktlim@umich.edu
632929Sktlim@umich.edu
642929Sktlim@umich.edu#
652929Sktlim@umich.edu# First copy everything up to the simulation statistics to a pair of
662929Sktlim@umich.edu# temporary files, stripping out date-related items, and do a plain
672929Sktlim@umich.edu# diff.  Any differences in the arguments are not necessarily an issue;
682929Sktlim@umich.edu# any differences in the program output should be caught by the EIO
692929Sktlim@umich.edu# mechanism if an EIO file is used.
702929Sktlim@umich.edu# 
712929Sktlim@umich.edu
722929Sktlim@umich.edu# copy_header takes input filehandle and output filename
732929Sktlim@umich.edu
742929Sktlim@umich.edusub copy_header
752929Sktlim@umich.edu{
762929Sktlim@umich.edu    my ($inhandle, $outname) = @_;
772929Sktlim@umich.edu
782929Sktlim@umich.edu    open(OUTPUT, ">$outname") or die "Error: can't open $outname.\n";
792929Sktlim@umich.edu
802929Sktlim@umich.edu    while (<$inhandle>)
812929Sktlim@umich.edu    {
822929Sktlim@umich.edu	# strip out lines that can vary
832929Sktlim@umich.edu	next if /^(command line:|M5 compiled on |M5 simulation started |M5 executing on )/;
842929Sktlim@umich.edu	last if /Begin Simulation Statistics/;
852929Sktlim@umich.edu	print OUTPUT;
862929Sktlim@umich.edu    }
872929Sktlim@umich.edu    close OUTPUT;
882929Sktlim@umich.edu}
892929Sktlim@umich.edu
902929Sktlim@umich.eduif ($opt_h) {
912929Sktlim@umich.edu
922929Sktlim@umich.edu    # Diff header separately from stats
932929Sktlim@umich.edu
942929Sktlim@umich.edu    $refheader = "/tmp/smt-test.refheader.$$";
952929Sktlim@umich.edu    $newheader = "/tmp/smt-test.newheader.$$";
962929Sktlim@umich.edu
972929Sktlim@umich.edu    copy_header(\*REF, $refheader);
982929Sktlim@umich.edu    copy_header(\*NEW, $newheader);
992929Sktlim@umich.edu
1002929Sktlim@umich.edu    print "\n===== Header and program output differences =====\n\n";
1012929Sktlim@umich.edu
1022929Sktlim@umich.edu    print `diff $refheader $newheader`;
1032929Sktlim@umich.edu
1042929Sktlim@umich.edu    print "\n===== Statistics differences =====\n\n";
1052929Sktlim@umich.edu}
1062929Sktlim@umich.edu
1072929Sktlim@umich.edu#
1082929Sktlim@umich.edu# Now parse statistics
1092929Sktlim@umich.edu#
1102929Sktlim@umich.edu
1112929Sktlim@umich.edu#
1122929Sktlim@umich.edu# This function takes an open filehandle and returns a reference to
1132929Sktlim@umich.edu# a hash containing all the statistics variables and their values.
1142929Sktlim@umich.edu#
1152929Sktlim@umich.edusub parse_file
1162929Sktlim@umich.edu{
1172929Sktlim@umich.edu    $stathandle = shift;
1182929Sktlim@umich.edu
1192929Sktlim@umich.edu    $in_dist = undef;
1202929Sktlim@umich.edu    $hashref = { };	# initialize hash for values
1212929Sktlim@umich.edu
1222929Sktlim@umich.edu    while (<$stathandle>)
1232929Sktlim@umich.edu    {
1242929Sktlim@umich.edu	next if /^\s*$/;	# skip blank lines
1252929Sktlim@umich.edu	last if /End Simulation Statistics/;
1262929Sktlim@umich.edu
1272929Sktlim@umich.edu	s/ *#.*//;		# strip comments
1282929Sktlim@umich.edu
1292929Sktlim@umich.edu	if (/^Memory usage: (\d+) KBytes/) {
1302929Sktlim@umich.edu	    $stat = 'memory usage';
1312929Sktlim@umich.edu	    $value = $1;
1322929Sktlim@umich.edu	}
1332929Sktlim@umich.edu	elsif ($in_dist) {
1347448Sstever@gmail.com	    if (/(.*)\.end_dist/) {
1357448Sstever@gmail.com		# end line of distribution: clear $in_dist flag
1367448Sstever@gmail.com		$in_dist = undef;
1377448Sstever@gmail.com		next;
1382929Sktlim@umich.edu	    }
1397448Sstever@gmail.com	    if ($opt_d) {
1407448Sstever@gmail.com		next;		#  bail out if we are ignoring dists...
1417448Sstever@gmail.com	    } elsif (/(.*)\.(min|max)_value/) {
1427448Sstever@gmail.com		# treat these like normal stats
1437448Sstever@gmail.com		($stat, $value) = /^(\S+)\s+(.*)/;
1447448Sstever@gmail.com	    } else {
1457448Sstever@gmail.com		($stat, $value) =
1467448Sstever@gmail.com		  /^(\S+(?:.*\S)?)\s+(\d+)\s+\d+\.\d+%/;
1477448Sstever@gmail.com		$stat = $in_dist . '::' . $stat;
1482929Sktlim@umich.edu	    }
1492929Sktlim@umich.edu	}
1502929Sktlim@umich.edu	else {
1512929Sktlim@umich.edu	    if (/(.*)\.start_dist/) {
1522929Sktlim@umich.edu		# start line of distribution: set $in_dist flag
1532929Sktlim@umich.edu		# and save distribution name for future reference
1542929Sktlim@umich.edu		$in_dist = $1;
1552929Sktlim@umich.edu		$stat = $1;
1562929Sktlim@umich.edu		$value = 0;
1572929Sktlim@umich.edu	    }
1582929Sktlim@umich.edu	    else {
1592929Sktlim@umich.edu		($stat, $value) = /^(\S+)\s+(.*)/;
1602929Sktlim@umich.edu	    }
1612929Sktlim@umich.edu	}
1622929Sktlim@umich.edu
1632929Sktlim@umich.edu	$$hashref{$stat} = $value;
1642929Sktlim@umich.edu    }
1652929Sktlim@umich.edu
1662929Sktlim@umich.edu    close($stathandle);
1672929Sktlim@umich.edu    return $hashref;
1682929Sktlim@umich.edu}
1692929Sktlim@umich.edu
1702929Sktlim@umich.edu
1712929Sktlim@umich.edu#
1722929Sktlim@umich.edu# pct_diff($old, $new) returns percent difference from $old to $new.
1732929Sktlim@umich.edu#
1742929Sktlim@umich.edusub pct_diff
1752929Sktlim@umich.edu{
1762929Sktlim@umich.edu    my ($old, $new) = @_;
1772929Sktlim@umich.edu    return ($old == 0) ? (($new == 0) ? 0 : 9999) : 100 * ($new - $old) / $old;
1782929Sktlim@umich.edu}
1792929Sktlim@umich.edu
1802929Sktlim@umich.edu
1812929Sktlim@umich.edu#
1822929Sktlim@umich.edu# Statistics to ignore: these relate to simulator performance, not
1832929Sktlim@umich.edu# correctness, so don't fail on changes here.
1842929Sktlim@umich.edu#
1852929Sktlim@umich.edu%ignore = (
1862929Sktlim@umich.edu  'host_seconds' => 1,
1872929Sktlim@umich.edu  'host_tick_rate' => 1,
1882929Sktlim@umich.edu  'host_inst_rate' => 1,
1898838Ssaidi@eecs.umich.edu  'host_op_rate' => 1,
1902929Sktlim@umich.edu  'host_mem_usage' => 1
1912929Sktlim@umich.edu);
1922929Sktlim@umich.edu
1932929Sktlim@umich.edu#
1942929Sktlim@umich.edu# List of key statistics (always displayed)
1952929Sktlim@umich.edu#  ==> list stats here WITHOUT trailing thread ID
1962929Sktlim@umich.edu#
1972929Sktlim@umich.edu@key_stat_list = (
1988838Ssaidi@eecs.umich.edu  'ipc',
1998838Ssaidi@eecs.umich.edu  'committedInsts',
2008838Ssaidi@eecs.umich.edu  'committedOps',
2018838Ssaidi@eecs.umich.edu  'sim_insts',
2028838Ssaidi@eecs.umich.edu  'sim_ops',
2038838Ssaidi@eecs.umich.edu  'sim_ticks',
2042929Sktlim@umich.edu  'host_inst_rate',
2052929Sktlim@umich.edu  'host_mem_usage'
2062929Sktlim@umich.edu);
2072929Sktlim@umich.edu
2082929Sktlim@umich.edu$key_stat_pattern = join('|', @key_stat_list);
2092929Sktlim@umich.edu
2102929Sktlim@umich.edu# initialize first statistics from each file
2112929Sktlim@umich.edu
2122929Sktlim@umich.edu$max_err_mag = 0;
2132929Sktlim@umich.edu
2142929Sktlim@umich.edu$refhash = parse_file(\*REF);
2152929Sktlim@umich.edu$newhash = parse_file(\*NEW);
2162929Sktlim@umich.edu
2172929Sktlim@umich.edu# The string sim-smt prints on a divide by zero
2182929Sktlim@umich.edu$divbyzero = '<err: divide by zero>';
2192929Sktlim@umich.edu
2202929Sktlim@umich.eduforeach $stat (sort keys %$refhash)
2212929Sktlim@umich.edu{
2222929Sktlim@umich.edu    $refvalue = $$refhash{$stat};
2232929Sktlim@umich.edu    $newvalue = $$newhash{$stat};
2242929Sktlim@umich.edu
2252929Sktlim@umich.edu    if (!defined($newvalue)) {
2262929Sktlim@umich.edu	# stat missing from new file
2272929Sktlim@umich.edu	push @missing_stats, $stat;
2282929Sktlim@umich.edu	next;
2292929Sktlim@umich.edu    }
2302929Sktlim@umich.edu
2312929Sktlim@umich.edu    if ($stat =~ /($key_stat_pattern)/o) {
2322929Sktlim@umich.edu	# key statistics: always record & display changes in these
2332929Sktlim@umich.edu	push @key_stats, [$stat, $refvalue, $newvalue];
2342929Sktlim@umich.edu    }
2352929Sktlim@umich.edu
2362929Sktlim@umich.edu    if ($ignore{$stat} or $refvalue eq $newvalue) {
2372929Sktlim@umich.edu	# stat is in "ignore" list, or hasn't changed
2382929Sktlim@umich.edu    }
2392929Sktlim@umich.edu    else {
2402929Sktlim@umich.edu	if ($refvalue eq $divbyzero || $newvalue eq $divbyzero) {
2412929Sktlim@umich.edu	    # one or the other was a divide by zero:
2422929Sktlim@umich.edu	    # no point in trying to quantify error
2432929Sktlim@umich.edu	    print "$stat: $refvalue --> $newvalue\n";
2442929Sktlim@umich.edu	}
2452929Sktlim@umich.edu	else {
2462929Sktlim@umich.edu	    $reldiff = pct_diff($refvalue, $newvalue);
2472929Sktlim@umich.edu	    $diffmag = abs($reldiff);
2482929Sktlim@umich.edu
2492929Sktlim@umich.edu	    if ($diffmag > $err_thresh) {
2502929Sktlim@umich.edu		push @errs,
2512929Sktlim@umich.edu		[$stat, $refvalue, $newvalue, $reldiff];
2522929Sktlim@umich.edu	    }
2532929Sktlim@umich.edu
2542929Sktlim@umich.edu	    if ($diffmag > $max_err_mag) {
2552929Sktlim@umich.edu		$max_err_mag = $diffmag;
2562929Sktlim@umich.edu	    }
2572929Sktlim@umich.edu	}
2582929Sktlim@umich.edu    }
2592929Sktlim@umich.edu
2602929Sktlim@umich.edu    # remove from new hash so we can detect added stats
2612929Sktlim@umich.edu    delete $$newhash{$stat};
2622929Sktlim@umich.edu}
2632929Sktlim@umich.edu
2642929Sktlim@umich.edu
2652929Sktlim@umich.edu#
2662929Sktlim@umich.edu# All done.  Print comparison summary.
2672929Sktlim@umich.edu#
2682929Sktlim@umich.edu
2692929Sktlim@umich.eduprintf("Maximum error magnitude: %+f%%\n\n", $max_err_mag);
2702929Sktlim@umich.edu
2712929Sktlim@umich.eduprintf("  %-30s %10s %10s %10s   %7s\n", ' ', 'Reference', 'New Value', 'Abs Diff', 'Pct Chg');
2722929Sktlim@umich.edu
2732929Sktlim@umich.eduprintf("Key statistics:\n\n");
2742929Sktlim@umich.edu
2752929Sktlim@umich.eduforeach $key_stat (@key_stats)
2762929Sktlim@umich.edu{
2772929Sktlim@umich.edu    ($statname, $refvalue, $newvalue, $reldiff) = @$key_stat;
2782929Sktlim@umich.edu
2792929Sktlim@umich.edu    # deduce format from reference value
2802929Sktlim@umich.edu    $pointpos = rindex($refvalue, '.');
2812929Sktlim@umich.edu    $digits = ($pointpos < 0) ? 0 :(length($refvalue) - $pointpos - 1);
2822929Sktlim@umich.edu    $fmt = "%10.${digits}f";
2832929Sktlim@umich.edu
2842929Sktlim@umich.edu    # print differing values with absolute and relative error
2852929Sktlim@umich.edu    printf("  %-30s $fmt $fmt $fmt  %+7.2f%%\n",
2862929Sktlim@umich.edu	   $statname, $refvalue, $newvalue,
2872929Sktlim@umich.edu	   $newvalue - $refvalue, pct_diff($refvalue, $newvalue));
2882929Sktlim@umich.edu}
2892929Sktlim@umich.edu
2907686Ssteve.reinhardt@amd.comprintf("\nDifferences > %d%%:\n\n", $err_thresh);
2912929Sktlim@umich.edu
2927686Ssteve.reinhardt@amd.comif ($opt_a) {
2937686Ssteve.reinhardt@amd.com    # leave stats sorted alphabetically, doesn't make sense to cut them off
2947686Ssteve.reinhardt@amd.com    $omit_count = 0;
2957686Ssteve.reinhardt@amd.com} else {
2962929Sktlim@umich.edu    # sort differences by percent change
2972929Sktlim@umich.edu    @errs = sort { abs($$b[3]) <=> abs($$a[3]) } @errs;
2982929Sktlim@umich.edu}
2992929Sktlim@umich.edu
3007686Ssteve.reinhardt@amd.com$num_errs = 0;
3017686Ssteve.reinhardt@amd.com
3022929Sktlim@umich.eduforeach $err (@errs)
3032929Sktlim@umich.edu{
3042929Sktlim@umich.edu    ($statname, $refvalue, $newvalue, $reldiff) = @$err;
3052929Sktlim@umich.edu
3062929Sktlim@umich.edu    # deduce format from reference value
3072929Sktlim@umich.edu    $pointpos1 = rindex($refvalue, '.');
3082929Sktlim@umich.edu    $digits1 = ($pointpos1 < 0) ? 0 :(length($refvalue) - $pointpos1 - 1);
3092929Sktlim@umich.edu    $pointpos2 = rindex($newvalue, '.');
3102929Sktlim@umich.edu    $digits2 = ($pointpos2 < 0) ? 0 :(length($newvalue) - $pointpos2 - 1);
3112929Sktlim@umich.edu    $digits = ($digits1 > $digits2) ? $digits1 : $digits2;
3122929Sktlim@umich.edu    $fmt = "%10.${digits}f";
3132929Sktlim@umich.edu
3142929Sktlim@umich.edu    # print differing values with absolute and relative error
3152929Sktlim@umich.edu    printf("  %-30s $fmt $fmt $fmt  %+7.2f%%\n",
3162929Sktlim@umich.edu	   $statname, $refvalue, $newvalue, $newvalue - $refvalue, $reldiff);
3172929Sktlim@umich.edu
3182929Sktlim@umich.edu    # only print top N errors
3197686Ssteve.reinhardt@amd.com    if ($omit_count > 0 && ++$num_errs >= $omit_count)
3202929Sktlim@umich.edu    {
3217686Ssteve.reinhardt@amd.com	print "[... showing top $omit_count errors only, additional errors omitted ...]\n";
3222929Sktlim@umich.edu	last;
3232929Sktlim@umich.edu    }
3242929Sktlim@umich.edu}
3252929Sktlim@umich.edu
3262929Sktlim@umich.edu#
3277448Sstever@gmail.com# Report missing stats
3287448Sstever@gmail.com#
3292929Sktlim@umich.edu# get count
3302929Sktlim@umich.edu$missing_stats = scalar(@missing_stats);
3312929Sktlim@umich.edu
3322929Sktlim@umich.eduif ($missing_stats)
3332929Sktlim@umich.edu{
3342929Sktlim@umich.edu    print "\nMissing $missing_stats reference statistics:\n\n";
3352929Sktlim@umich.edu    foreach $stat (@missing_stats)
3362929Sktlim@umich.edu    {
3372929Sktlim@umich.edu#	print "\t$stat\n";
3382929Sktlim@umich.edu	printf "  %-50s    ", $stat;
3392929Sktlim@umich.edu	print  "$$refhash{$stat}\n";
3402929Sktlim@umich.edu    }
3412929Sktlim@umich.edu}
3422929Sktlim@umich.edu
3432929Sktlim@umich.edu#
3442929Sktlim@umich.edu# Any stats left in newhash are added since the reference file
3452929Sktlim@umich.edu#
3462929Sktlim@umich.edu
3472929Sktlim@umich.edu@added_stats = keys %$newhash;
3482929Sktlim@umich.edu
3492929Sktlim@umich.edu# get count
3502929Sktlim@umich.edu$added_stats = scalar(@added_stats);
3512929Sktlim@umich.edu
3522929Sktlim@umich.eduif ($added_stats)
3532929Sktlim@umich.edu{
3542929Sktlim@umich.edu    print "\nFound $added_stats new statistics:\n\n";
3552929Sktlim@umich.edu    foreach $stat (sort @added_stats)
3562929Sktlim@umich.edu    {
3572929Sktlim@umich.edu#	print "\t$stat\n";
3582929Sktlim@umich.edu	printf "  %-50s    ", $stat;
3592929Sktlim@umich.edu	print  "$$newhash{$stat}\n";
3602929Sktlim@umich.edu    }
3612929Sktlim@umich.edu}
3622929Sktlim@umich.edu
3632929Sktlim@umich.educleanup();
3647448Sstever@gmail.com# Exit code is 0 if all stats are found (with no extras) & no stats error, 1 otherwise
3657448Sstever@gmail.com$status = ($missing_stats == 0 && $added_stats == 0 && $max_err_mag == 0.0) ? 0 : 1;
3662929Sktlim@umich.eduexit $status;
3672929Sktlim@umich.edu
3682929Sktlim@umich.edusub cleanup
3692929Sktlim@umich.edu{
3702929Sktlim@umich.edu    unlink($refheader) if ($refheader);
3712929Sktlim@umich.edu    unlink($newheader) if ($newheader);
3722929Sktlim@umich.edu}
373