diff-out revision 2929
113511Sgabeblack@google.com#!/usr/bin/perl
213511Sgabeblack@google.com# Copyright (c) 2001-2005 The Regents of The University of Michigan
313511Sgabeblack@google.com# All rights reserved.
413511Sgabeblack@google.com#
513511Sgabeblack@google.com# Redistribution and use in source and binary forms, with or without
613511Sgabeblack@google.com# modification, are permitted provided that the following conditions are
713511Sgabeblack@google.com# met: redistributions of source code must retain the above copyright
813511Sgabeblack@google.com# notice, this list of conditions and the following disclaimer;
913511Sgabeblack@google.com# redistributions in binary form must reproduce the above copyright
1013511Sgabeblack@google.com# notice, this list of conditions and the following disclaimer in the
1113511Sgabeblack@google.com# documentation and/or other materials provided with the distribution;
1213511Sgabeblack@google.com# neither the name of the copyright holders nor the names of its
1313511Sgabeblack@google.com# contributors may be used to endorse or promote products derived from
1413511Sgabeblack@google.com# this software without specific prior written permission.
1513511Sgabeblack@google.com#
1613511Sgabeblack@google.com# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
1713511Sgabeblack@google.com# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
1813511Sgabeblack@google.com# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
1913511Sgabeblack@google.com# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
2013511Sgabeblack@google.com# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
2113511Sgabeblack@google.com# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
2213511Sgabeblack@google.com# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
2313511Sgabeblack@google.com# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
2413511Sgabeblack@google.com# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
2513511Sgabeblack@google.com# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
2613511Sgabeblack@google.com# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
2713511Sgabeblack@google.com#
2813511Sgabeblack@google.com# Authors: Steve Reinhardt
2913511Sgabeblack@google.com
3013511Sgabeblack@google.com#
3113511Sgabeblack@google.com# This script diffs two SimpleScalar statistics output files.
3213511Sgabeblack@google.com#
3313511Sgabeblack@google.com
3413511Sgabeblack@google.comuse Getopt::Std;
3513511Sgabeblack@google.com
3613511Sgabeblack@google.com#
3713511Sgabeblack@google.com# -t thresh sets threshold for ignoring differences (in %)
3813511Sgabeblack@google.com# -p sorts differences by % chg (default is alphabetic)
3913511Sgabeblack@google.com# -f ignores fetch-loss statistics
4013511Sgabeblack@google.com# -d ignores all distributions
4113511Sgabeblack@google.com#
4213511Sgabeblack@google.com
4313511Sgabeblack@google.comgetopts('dfn:pt:h');
4413511Sgabeblack@google.com
4513511Sgabeblack@google.comif ($#ARGV < 1)
4613511Sgabeblack@google.com{
4713511Sgabeblack@google.com    print "\nError: need two file arguments (<reference> <new>).\n";
4813511Sgabeblack@google.com    print "   Options: -d  =  Ignore distributions\n";
4913511Sgabeblack@google.com    print "            -f  =  Ignore fetch-loss stats\n";
5013511Sgabeblack@google.com    print "            -p  =  Sort errors by percentage\n";
5113511Sgabeblack@google.com    print "            -h  =  Diff header info separately from stats\n";
5213511Sgabeblack@google.com    print "            -n <num>  =  Print top <num> errors (default 20)\n";
5313511Sgabeblack@google.com    print "            -t <num>  =  Error threshold in percent (default 1)\n\n";
5413511Sgabeblack@google.com    die -1;
5513511Sgabeblack@google.com}
5613511Sgabeblack@google.com
5713511Sgabeblack@google.comopen(REF, "<$ARGV[0]") or die "Error: can't open $ARGV[0].\n";
5813511Sgabeblack@google.comopen(NEW, "<$ARGV[1]") or die "Error: can't open $ARGV[1].\n";
5913511Sgabeblack@google.com
6013511Sgabeblack@google.com
6113511Sgabeblack@google.com#
6213511Sgabeblack@google.com# Things that really should be adjustable via the command line
6313511Sgabeblack@google.com#
6413511Sgabeblack@google.com
6513511Sgabeblack@google.com# Ignorable error (in percent)
6613511Sgabeblack@google.com$err_thresh = ($opt_t) ? $opt_t : 0;
6713511Sgabeblack@google.com
6813511Sgabeblack@google.com# Number of stats to print before omitting
6913511Sgabeblack@google.com$omit_count = ($opt_n) ? $opt_n : 20;
7013511Sgabeblack@google.com
7113511Sgabeblack@google.com
7213511Sgabeblack@google.com#
7313511Sgabeblack@google.com# First copy everything up to the simulation statistics to a pair of
7413511Sgabeblack@google.com# temporary files, stripping out date-related items, and do a plain
7513511Sgabeblack@google.com# diff.  Any differences in the arguments are not necessarily an issue;
7613511Sgabeblack@google.com# any differences in the program output should be caught by the EIO
7713511Sgabeblack@google.com# mechanism if an EIO file is used.
7813511Sgabeblack@google.com# 
7913511Sgabeblack@google.com
8013511Sgabeblack@google.com# copy_header takes input filehandle and output filename
8113511Sgabeblack@google.com
8213511Sgabeblack@google.comsub copy_header
8313511Sgabeblack@google.com{
8413511Sgabeblack@google.com    my ($inhandle, $outname) = @_;
8513511Sgabeblack@google.com
8613511Sgabeblack@google.com    open(OUTPUT, ">$outname") or die "Error: can't open $outname.\n";
8713511Sgabeblack@google.com
8813511Sgabeblack@google.com    while (<$inhandle>)
8913511Sgabeblack@google.com    {
9013511Sgabeblack@google.com	# strip out lines that can vary
9113511Sgabeblack@google.com	next if /^(command line:|M5 compiled on |M5 simulation started |M5 executing on )/;
9213511Sgabeblack@google.com	last if /Begin Simulation Statistics/;
9313511Sgabeblack@google.com	print OUTPUT;
9413511Sgabeblack@google.com    }
9513511Sgabeblack@google.com    close OUTPUT;
9613511Sgabeblack@google.com}
9713511Sgabeblack@google.com
9813511Sgabeblack@google.comif ($opt_h) {
9913511Sgabeblack@google.com
10013511Sgabeblack@google.com    # Diff header separately from stats
10113511Sgabeblack@google.com
10213511Sgabeblack@google.com    $refheader = "/tmp/smt-test.refheader.$$";
10313511Sgabeblack@google.com    $newheader = "/tmp/smt-test.newheader.$$";
10413511Sgabeblack@google.com
10513511Sgabeblack@google.com    copy_header(\*REF, $refheader);
10613511Sgabeblack@google.com    copy_header(\*NEW, $newheader);
10713511Sgabeblack@google.com
10813511Sgabeblack@google.com    print "\n===== Header and program output differences =====\n\n";
10913511Sgabeblack@google.com
11013511Sgabeblack@google.com    print `diff $refheader $newheader`;
11113511Sgabeblack@google.com
11213511Sgabeblack@google.com    print "\n===== Statistics differences =====\n\n";
11313511Sgabeblack@google.com}
11413511Sgabeblack@google.com
11513511Sgabeblack@google.com#
11613511Sgabeblack@google.com# Now parse statistics
11713511Sgabeblack@google.com#
11813511Sgabeblack@google.com
11913511Sgabeblack@google.com#
12013511Sgabeblack@google.com# This function takes an open filehandle and returns a reference to
12113511Sgabeblack@google.com# a hash containing all the statistics variables and their values.
12213511Sgabeblack@google.com#
12313511Sgabeblack@google.comsub parse_file
12413511Sgabeblack@google.com{
12513511Sgabeblack@google.com    $stathandle = shift;
12613511Sgabeblack@google.com
12713511Sgabeblack@google.com    $in_dist = undef;
12813511Sgabeblack@google.com    $hashref = { };	# initialize hash for values
12913511Sgabeblack@google.com
13013511Sgabeblack@google.com    while (<$stathandle>)
13113511Sgabeblack@google.com    {
13213511Sgabeblack@google.com	next if /^\s*$/;	# skip blank lines
13313511Sgabeblack@google.com                next if /^\*\*Ignore/;   # temporary, to make totaling scripts easy for ISCA 03
13413511Sgabeblack@google.com	last if /End Simulation Statistics/;
13513511Sgabeblack@google.com
13613511Sgabeblack@google.com	s/ *#.*//;		# strip comments
13713511Sgabeblack@google.com
13813511Sgabeblack@google.com	if (/^Memory usage: (\d+) KBytes/) {
13913511Sgabeblack@google.com	    $stat = 'memory usage';
14013511Sgabeblack@google.com	    $value = $1;
14113511Sgabeblack@google.com	}
14213511Sgabeblack@google.com	elsif ($in_dist) {
14313511Sgabeblack@google.com	    if ($in_dist =~ /^fetch_loss_counters/) {
14413511Sgabeblack@google.com		if (/^fetch_loss_counters_\d+\.end/) {
14513511Sgabeblack@google.com		    # end line of distribution: clear $in_dist flag
14613511Sgabeblack@google.com		    $in_dist = undef;
14713511Sgabeblack@google.com		    next;
14813511Sgabeblack@google.com		}
14913511Sgabeblack@google.com		else {
15013511Sgabeblack@google.com		    next if $opt_f;
15113511Sgabeblack@google.com
15213511Sgabeblack@google.com		    ($stat, $value) = /^(\S+)\s+(.*)/;
15313511Sgabeblack@google.com		}
15413511Sgabeblack@google.com	    }
15513511Sgabeblack@google.com	    else {
15613511Sgabeblack@google.com		if (/(.*)\.end_dist/) {
15713511Sgabeblack@google.com		    # end line of distribution: clear $in_dist flag
15813511Sgabeblack@google.com		    $in_dist = undef;
15913511Sgabeblack@google.com		    next;
16013511Sgabeblack@google.com		}
16113511Sgabeblack@google.com		if ($opt_d) {
16213511Sgabeblack@google.com		    next;  #  bail out if we are ignoring dists...
16313511Sgabeblack@google.com		}
16413511Sgabeblack@google.com		elsif (/(.*)\.(min|max)_value/) {
16513511Sgabeblack@google.com		    # treat these like normal stats
16613511Sgabeblack@google.com		    ($stat, $value) = /^(\S+)\s+(.*)/;
16713511Sgabeblack@google.com		}
16813511Sgabeblack@google.com		else {
16913511Sgabeblack@google.com		    # this is ugly because labels in the distribution
17013511Sgabeblack@google.com		    # buckets don't start in column 0 and may include
17113511Sgabeblack@google.com		    # embedded spaces
17213511Sgabeblack@google.com		    ($stat, $value) =
17313511Sgabeblack@google.com		      /^\s*(\S+(?:.*\S)?)\s+(\d+)\s+\d+\.\d+%/;
17413511Sgabeblack@google.com		    $stat = $in_dist . '::' . $stat;
175		}
176	    }
177	}
178	else {
179	    if (/(.*)\.start_dist/) {
180		# start line of distribution: set $in_dist flag
181		# and save distribution name for future reference
182		$in_dist = $1;
183		$stat = $1;
184		$value = 0;
185	    }
186	    elsif (/^(fetch_loss_counters_\d+)\.start/) {
187		# treat fetch loss counters like distribution, sort of
188		$in_dist = $1;
189		$stat = $1;
190		$value = 0;
191	    }
192	    else {
193		($stat, $value) = /^(\S+)\s+(.*)/;
194	    }
195	}
196
197	$$hashref{$stat} = $value;
198    }
199
200    close($stathandle);
201    return $hashref;
202}
203
204
205#
206# pct_diff($old, $new) returns percent difference from $old to $new.
207#
208sub pct_diff
209{
210    my ($old, $new) = @_;
211    return ($old == 0) ? (($new == 0) ? 0 : 9999) : 100 * ($new - $old) / $old;
212}
213
214
215#
216# Statistics to ignore: these relate to simulator performance, not
217# correctness, so don't fail on changes here.
218#
219%ignore = (
220  'host_seconds' => 1,
221  'host_tick_rate' => 1,
222  'host_inst_rate' => 1,
223  'host_mem_usage' => 1
224);
225
226#
227# List of key statistics (always displayed)
228#  ==> list stats here WITHOUT trailing thread ID
229#
230@key_stat_list = (
231  'COM:IPC',
232  'ISSUE:MSIPC',
233  'COM:count',
234  'host_inst_rate',
235  'sim_insts',
236  'sim_ticks',
237  'host_mem_usage'
238);
239
240$key_stat_pattern = join('|', @key_stat_list);
241
242# initialize first statistics from each file
243
244$max_err_mag = 0;
245
246$refhash = parse_file(\*REF);
247$newhash = parse_file(\*NEW);
248
249# The string sim-smt prints on a divide by zero
250$divbyzero = '<err: divide by zero>';
251
252foreach $stat (sort keys %$refhash)
253{
254    $refvalue = $$refhash{$stat};
255    $newvalue = $$newhash{$stat};
256
257    if (!defined($newvalue)) {
258	# stat missing from new file
259	push @missing_stats, $stat;
260	next;
261    }
262
263    if ($stat =~ /($key_stat_pattern)/o) {
264	# key statistics: always record & display changes in these
265	push @key_stats, [$stat, $refvalue, $newvalue];
266    }
267
268    if ($ignore{$stat} or $refvalue eq $newvalue) {
269	# stat is in "ignore" list, or hasn't changed
270    }
271    else {
272	if ($refvalue eq $divbyzero || $newvalue eq $divbyzero) {
273	    # one or the other was a divide by zero:
274	    # no point in trying to quantify error
275	    print "$stat: $refvalue --> $newvalue\n";
276	}
277	else {
278	    $reldiff = pct_diff($refvalue, $newvalue);
279	    $diffmag = abs($reldiff);
280
281	    if ($diffmag > $err_thresh) {
282		push @errs,
283		[$stat, $refvalue, $newvalue, $reldiff];
284	    }
285
286	    if ($diffmag > $max_err_mag) {
287		$max_err_mag = $diffmag;
288	    }
289	}
290    }
291
292    # remove from new hash so we can detect added stats
293    delete $$newhash{$stat};
294}
295
296
297#
298# All done.  Print comparison summary.
299#
300
301printf("Maximum error magnitude: %+f%%\n\n", $max_err_mag);
302
303printf("  %-30s %10s %10s %10s   %7s\n", ' ', 'Reference', 'New Value', 'Abs Diff', 'Pct Chg');
304
305printf("Key statistics:\n\n");
306
307foreach $key_stat (@key_stats)
308{
309    ($statname, $refvalue, $newvalue, $reldiff) = @$key_stat;
310
311    # deduce format from reference value
312    $pointpos = rindex($refvalue, '.');
313    $digits = ($pointpos < 0) ? 0 :(length($refvalue) - $pointpos - 1);
314    $fmt = "%10.${digits}f";
315
316    # print differing values with absolute and relative error
317    printf("  %-30s $fmt $fmt $fmt  %+7.2f%%\n",
318	   $statname, $refvalue, $newvalue,
319	   $newvalue - $refvalue, pct_diff($refvalue, $newvalue));
320}
321
322printf("\nLargest $omit_count relative errors (> %d%%):\n\n", $err_thresh);
323
324$num_errs = 0;
325
326if ($opt_p)
327{
328    # sort differences by percent change
329    @errs = sort { abs($$b[3]) <=> abs($$a[3]) } @errs;
330}
331
332foreach $err (@errs)
333{
334    ($statname, $refvalue, $newvalue, $reldiff) = @$err;
335
336    # deduce format from reference value
337    $pointpos1 = rindex($refvalue, '.');
338    $digits1 = ($pointpos1 < 0) ? 0 :(length($refvalue) - $pointpos1 - 1);
339    $pointpos2 = rindex($newvalue, '.');
340    $digits2 = ($pointpos2 < 0) ? 0 :(length($newvalue) - $pointpos2 - 1);
341    $digits = ($digits1 > $digits2) ? $digits1 : $digits2;
342    $fmt = "%10.${digits}f";
343
344    # print differing values with absolute and relative error
345    printf("  %-30s $fmt $fmt $fmt  %+7.2f%%\n",
346	   $statname, $refvalue, $newvalue, $newvalue - $refvalue, $reldiff);
347
348    # only print top N errors
349    if (++$num_errs >= $omit_count)
350    {
351	print "[... additional errors omitted ...]\n";
352	last;
353    }
354}
355
356#
357# Report missing stats, but first filter out distribution buckets:
358# these are mostly noise
359
360@missing_stats = grep { !/::(\d+|overflows)?$/ } @missing_stats;
361
362# get count
363$missing_stats = scalar(@missing_stats);
364
365if ($missing_stats)
366{
367    print "\nMissing $missing_stats reference statistics:\n\n";
368    foreach $stat (@missing_stats)
369    {
370#	print "\t$stat\n";
371	printf "  %-50s    ", $stat;
372	print  "$$refhash{$stat}\n";
373    }
374}
375
376#
377# Any stats left in newhash are added since the reference file
378#
379
380@added_stats = keys %$newhash;
381
382# first filter out distribution buckets: mostly noise
383
384@added_stats = grep { !/::(\d+|overflows)?$/ } @added_stats;
385
386# get count
387$added_stats = scalar(@added_stats);
388
389if ($added_stats)
390{
391    print "\nFound $added_stats new statistics:\n\n";
392    foreach $stat (sort @added_stats)
393    {
394#	print "\t$stat\n";
395	printf "  %-50s    ", $stat;
396	print  "$$newhash{$stat}\n";
397    }
398}
399
400cleanup();
401# Exit code is 0 if no stats error, 1 otherwise
402$status = ($max_err_mag == 0.0) ? 0 : 1;
403exit $status;
404
405sub cleanup
406{
407    unlink($refheader) if ($refheader);
408    unlink($newheader) if ($newheader);
409}
410