1#!/usr/bin/perl
2# Copyright (c) 2001-2005 The Regents of The University of Michigan
3# All rights reserved.
4#
5# Redistribution and use in source and binary forms, with or without
6# modification, are permitted provided that the following conditions are
7# met: redistributions of source code must retain the above copyright
8# notice, this list of conditions and the following disclaimer;
9# redistributions in binary form must reproduce the above copyright
10# notice, this list of conditions and the following disclaimer in the
11# documentation and/or other materials provided with the distribution;
12# neither the name of the copyright holders nor the names of its
13# contributors may be used to endorse or promote products derived from
14# this software without specific prior written permission.
15#
16# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
17# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
18# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
19# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
20# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
21# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
22# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
23# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
24# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
25# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
26# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
27#
28# Authors: Steve Reinhardt
29
30#
31# This script diffs two SimpleScalar statistics output files.
32#
33
34use Getopt::Std;
35
36getopts('adn:t:h');
37
38if ($#ARGV < 1)
39{
40    print "\nError: need two file arguments (<reference> <new>).\n";
41    print "   Options: -d = Ignore distributions\n";
42    print "            -a = Sort errors alphabetically (default: by percentage)\n";
43    print "            -h = Diff header info separately from stats\n";
44    print "            -n <num> = Print top <num> errors (default 20, 0 for all)\n";
45    print "            -t <num> = Ignore errors below <num> percent (default 0)\n\n";
46    exit;
47}
48
49open(REF, "<$ARGV[0]") or die "Error: can't open $ARGV[0].\n";
50open(NEW, "<$ARGV[1]") or die "Error: can't open $ARGV[1].\n";
51
52
53#
54# Things that really should be adjustable via the command line
55#
56
57# Ignorable error (in percent)
58$err_thresh = defined($opt_t) ? $opt_t : 0;
59
60# Number of stats to print before omitting
61$omit_count = defined($opt_n) ? $opt_n : 20;
62
63
64#
65# First copy everything up to the simulation statistics to a pair of
66# temporary files, stripping out date-related items, and do a plain
67# diff.  Any differences in the arguments are not necessarily an issue;
68# any differences in the program output should be caught by the EIO
69# mechanism if an EIO file is used.
70# 
71
72# copy_header takes input filehandle and output filename
73
74sub copy_header
75{
76    my ($inhandle, $outname) = @_;
77
78    open(OUTPUT, ">$outname") or die "Error: can't open $outname.\n";
79
80    while (<$inhandle>)
81    {
82	# strip out lines that can vary
83	next if /^(command line:|M5 compiled on |M5 simulation started |M5 executing on )/;
84	last if /Begin Simulation Statistics/;
85	print OUTPUT;
86    }
87    close OUTPUT;
88}
89
90if ($opt_h) {
91
92    # Diff header separately from stats
93
94    $refheader = "/tmp/smt-test.refheader.$$";
95    $newheader = "/tmp/smt-test.newheader.$$";
96
97    copy_header(\*REF, $refheader);
98    copy_header(\*NEW, $newheader);
99
100    print "\n===== Header and program output differences =====\n\n";
101
102    print `diff $refheader $newheader`;
103
104    print "\n===== Statistics differences =====\n\n";
105}
106
107#
108# Now parse statistics
109#
110
111#
112# This function takes an open filehandle and returns a reference to
113# a hash containing all the statistics variables and their values.
114#
115sub parse_file
116{
117    $stathandle = shift;
118
119    $in_dist = undef;
120    $hashref = { };	# initialize hash for values
121
122    while (<$stathandle>)
123    {
124	next if /^\s*$/;	# skip blank lines
125	last if /End Simulation Statistics/;
126
127	s/ *#.*//;		# strip comments
128
129	if (/^Memory usage: (\d+) KBytes/) {
130	    $stat = 'memory usage';
131	    $value = $1;
132	}
133	elsif ($in_dist) {
134	    if (/(.*)\.end_dist/) {
135		# end line of distribution: clear $in_dist flag
136		$in_dist = undef;
137		next;
138	    }
139	    if ($opt_d) {
140		next;		#  bail out if we are ignoring dists...
141	    } elsif (/(.*)\.(min|max)_value/) {
142		# treat these like normal stats
143		($stat, $value) = /^(\S+)\s+(.*)/;
144	    } else {
145		($stat, $value) =
146		  /^(\S+(?:.*\S)?)\s+(\d+)\s+\d+\.\d+%/;
147		$stat = $in_dist . '::' . $stat;
148	    }
149	}
150	else {
151	    if (/(.*)\.start_dist/) {
152		# start line of distribution: set $in_dist flag
153		# and save distribution name for future reference
154		$in_dist = $1;
155		$stat = $1;
156		$value = 0;
157	    }
158	    else {
159		($stat, $value) = /^(\S+)\s+(.*)/;
160	    }
161	}
162
163	$$hashref{$stat} = $value;
164    }
165
166    close($stathandle);
167    return $hashref;
168}
169
170
171#
172# pct_diff($old, $new) returns percent difference from $old to $new.
173#
174sub pct_diff
175{
176    my ($old, $new) = @_;
177    return ($old == 0) ? (($new == 0) ? 0 : 9999) : 100 * ($new - $old) / $old;
178}
179
180
181#
182# Statistics to ignore: these relate to simulator performance, not
183# correctness, so don't fail on changes here.
184#
185%ignore = (
186  'host_seconds' => 1,
187  'host_tick_rate' => 1,
188  'host_inst_rate' => 1,
189  'host_op_rate' => 1,
190  'host_mem_usage' => 1
191);
192
193#
194# List of key statistics (always displayed)
195#  ==> list stats here WITHOUT trailing thread ID
196#
197@key_stat_list = (
198  'ipc',
199  'committedInsts',
200  'committedOps',
201  'sim_insts',
202  'sim_ops',
203  'sim_ticks',
204  'host_inst_rate',
205  'host_mem_usage'
206);
207
208$key_stat_pattern = join('|', @key_stat_list);
209
210# initialize first statistics from each file
211
212$max_err_mag = 0;
213
214$refhash = parse_file(\*REF);
215$newhash = parse_file(\*NEW);
216
217# The string sim-smt prints on a divide by zero
218$divbyzero = '<err: divide by zero>';
219
220foreach $stat (sort keys %$refhash)
221{
222    $refvalue = $$refhash{$stat};
223    $newvalue = $$newhash{$stat};
224
225    if (!defined($newvalue)) {
226	# stat missing from new file
227	push @missing_stats, $stat;
228	next;
229    }
230
231    if ($stat =~ /($key_stat_pattern)/o) {
232	# key statistics: always record & display changes in these
233	push @key_stats, [$stat, $refvalue, $newvalue];
234    }
235
236    if ($ignore{$stat} or $refvalue eq $newvalue) {
237	# stat is in "ignore" list, or hasn't changed
238    }
239    else {
240	if ($refvalue eq $divbyzero || $newvalue eq $divbyzero) {
241	    # one or the other was a divide by zero:
242	    # no point in trying to quantify error
243	    print "$stat: $refvalue --> $newvalue\n";
244	}
245	else {
246	    $reldiff = pct_diff($refvalue, $newvalue);
247	    $diffmag = abs($reldiff);
248
249	    if ($diffmag > $err_thresh) {
250		push @errs,
251		[$stat, $refvalue, $newvalue, $reldiff];
252	    }
253
254	    if ($diffmag > $max_err_mag) {
255		$max_err_mag = $diffmag;
256	    }
257	}
258    }
259
260    # remove from new hash so we can detect added stats
261    delete $$newhash{$stat};
262}
263
264
265#
266# All done.  Print comparison summary.
267#
268
269printf("Maximum error magnitude: %+f%%\n\n", $max_err_mag);
270
271printf("  %-30s %10s %10s %10s   %7s\n", ' ', 'Reference', 'New Value', 'Abs Diff', 'Pct Chg');
272
273printf("Key statistics:\n\n");
274
275foreach $key_stat (@key_stats)
276{
277    ($statname, $refvalue, $newvalue, $reldiff) = @$key_stat;
278
279    # deduce format from reference value
280    $pointpos = rindex($refvalue, '.');
281    $digits = ($pointpos < 0) ? 0 :(length($refvalue) - $pointpos - 1);
282    $fmt = "%10.${digits}f";
283
284    # print differing values with absolute and relative error
285    printf("  %-30s $fmt $fmt $fmt  %+7.2f%%\n",
286	   $statname, $refvalue, $newvalue,
287	   $newvalue - $refvalue, pct_diff($refvalue, $newvalue));
288}
289
290printf("\nDifferences > %d%%:\n\n", $err_thresh);
291
292if ($opt_a) {
293    # leave stats sorted alphabetically, doesn't make sense to cut them off
294    $omit_count = 0;
295} else {
296    # sort differences by percent change
297    @errs = sort { abs($$b[3]) <=> abs($$a[3]) } @errs;
298}
299
300$num_errs = 0;
301
302foreach $err (@errs)
303{
304    ($statname, $refvalue, $newvalue, $reldiff) = @$err;
305
306    # deduce format from reference value
307    $pointpos1 = rindex($refvalue, '.');
308    $digits1 = ($pointpos1 < 0) ? 0 :(length($refvalue) - $pointpos1 - 1);
309    $pointpos2 = rindex($newvalue, '.');
310    $digits2 = ($pointpos2 < 0) ? 0 :(length($newvalue) - $pointpos2 - 1);
311    $digits = ($digits1 > $digits2) ? $digits1 : $digits2;
312    $fmt = "%10.${digits}f";
313
314    # print differing values with absolute and relative error
315    printf("  %-30s $fmt $fmt $fmt  %+7.2f%%\n",
316	   $statname, $refvalue, $newvalue, $newvalue - $refvalue, $reldiff);
317
318    # only print top N errors
319    if ($omit_count > 0 && ++$num_errs >= $omit_count)
320    {
321	print "[... showing top $omit_count errors only, additional errors omitted ...]\n";
322	last;
323    }
324}
325
326#
327# Report missing stats
328#
329# get count
330$missing_stats = scalar(@missing_stats);
331
332if ($missing_stats)
333{
334    print "\nMissing $missing_stats reference statistics:\n\n";
335    foreach $stat (@missing_stats)
336    {
337#	print "\t$stat\n";
338	printf "  %-50s    ", $stat;
339	print  "$$refhash{$stat}\n";
340    }
341}
342
343#
344# Any stats left in newhash are added since the reference file
345#
346
347@added_stats = keys %$newhash;
348
349# get count
350$added_stats = scalar(@added_stats);
351
352if ($added_stats)
353{
354    print "\nFound $added_stats new statistics:\n\n";
355    foreach $stat (sort @added_stats)
356    {
357#	print "\t$stat\n";
358	printf "  %-50s    ", $stat;
359	print  "$$newhash{$stat}\n";
360    }
361}
362
363cleanup();
364# Exit codes:
365# 0 if all stats are found (with no extras) & no stats error
366# 1 if there are additional stats, but no stat errors
367# 2 otherwise
368$no_hard_errors = $missing_stats == 0 && $max_err_mag == 0.0;
369$status = $no_hard_errors ? ($added_stats == 0 ? 0 : 1) : 2;
370exit $status;
371
372sub cleanup
373{
374    unlink($refheader) if ($refheader);
375    unlink($newheader) if ($newheader);
376}
377