diff-out revision 7448
1#!/usr/bin/perl
2# Copyright (c) 2001-2005 The Regents of The University of Michigan
3# All rights reserved.
4#
5# Redistribution and use in source and binary forms, with or without
6# modification, are permitted provided that the following conditions are
7# met: redistributions of source code must retain the above copyright
8# notice, this list of conditions and the following disclaimer;
9# redistributions in binary form must reproduce the above copyright
10# notice, this list of conditions and the following disclaimer in the
11# documentation and/or other materials provided with the distribution;
12# neither the name of the copyright holders nor the names of its
13# contributors may be used to endorse or promote products derived from
14# this software without specific prior written permission.
15#
16# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
17# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
18# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
19# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
20# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
21# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
22# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
23# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
24# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
25# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
26# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
27#
28# Authors: Steve Reinhardt
29
30#
31# This script diffs two SimpleScalar statistics output files.
32#
33
34use Getopt::Std;
35
36#
37# -t thresh sets threshold for ignoring differences (in %)
38# -p sorts differences by % chg (default is alphabetic)
39# -d ignores all distributions
40#
41
42getopts('dfn:pt:h');
43
44if ($#ARGV < 1)
45{
46    print "\nError: need two file arguments (<reference> <new>).\n";
47    print "   Options: -d  =  Ignore distributions\n";
48    print "            -p  =  Sort errors by percentage\n";
49    print "            -h  =  Diff header info separately from stats\n";
50    print "            -n <num>  =  Print top <num> errors (default 20)\n";
51    print "            -t <num>  =  Error threshold in percent (default 1)\n\n";
52    die -1;
53}
54
55open(REF, "<$ARGV[0]") or die "Error: can't open $ARGV[0].\n";
56open(NEW, "<$ARGV[1]") or die "Error: can't open $ARGV[1].\n";
57
58
59#
60# Things that really should be adjustable via the command line
61#
62
63# Ignorable error (in percent)
64$err_thresh = ($opt_t) ? $opt_t : 0;
65
66# Number of stats to print before omitting
67$omit_count = ($opt_n) ? $opt_n : 20;
68
69
70#
71# First copy everything up to the simulation statistics to a pair of
72# temporary files, stripping out date-related items, and do a plain
73# diff.  Any differences in the arguments are not necessarily an issue;
74# any differences in the program output should be caught by the EIO
75# mechanism if an EIO file is used.
76# 
77
78# copy_header takes input filehandle and output filename
79
80sub copy_header
81{
82    my ($inhandle, $outname) = @_;
83
84    open(OUTPUT, ">$outname") or die "Error: can't open $outname.\n";
85
86    while (<$inhandle>)
87    {
88	# strip out lines that can vary
89	next if /^(command line:|M5 compiled on |M5 simulation started |M5 executing on )/;
90	last if /Begin Simulation Statistics/;
91	print OUTPUT;
92    }
93    close OUTPUT;
94}
95
96if ($opt_h) {
97
98    # Diff header separately from stats
99
100    $refheader = "/tmp/smt-test.refheader.$$";
101    $newheader = "/tmp/smt-test.newheader.$$";
102
103    copy_header(\*REF, $refheader);
104    copy_header(\*NEW, $newheader);
105
106    print "\n===== Header and program output differences =====\n\n";
107
108    print `diff $refheader $newheader`;
109
110    print "\n===== Statistics differences =====\n\n";
111}
112
113#
114# Now parse statistics
115#
116
117#
118# This function takes an open filehandle and returns a reference to
119# a hash containing all the statistics variables and their values.
120#
121sub parse_file
122{
123    $stathandle = shift;
124
125    $in_dist = undef;
126    $hashref = { };	# initialize hash for values
127
128    while (<$stathandle>)
129    {
130	next if /^\s*$/;	# skip blank lines
131	last if /End Simulation Statistics/;
132
133	s/ *#.*//;		# strip comments
134
135	if (/^Memory usage: (\d+) KBytes/) {
136	    $stat = 'memory usage';
137	    $value = $1;
138	}
139	elsif ($in_dist) {
140	    if (/(.*)\.end_dist/) {
141		# end line of distribution: clear $in_dist flag
142		$in_dist = undef;
143		next;
144	    }
145	    if ($opt_d) {
146		next;		#  bail out if we are ignoring dists...
147	    } elsif (/(.*)\.(min|max)_value/) {
148		# treat these like normal stats
149		($stat, $value) = /^(\S+)\s+(.*)/;
150	    } else {
151		($stat, $value) =
152		  /^(\S+(?:.*\S)?)\s+(\d+)\s+\d+\.\d+%/;
153		$stat = $in_dist . '::' . $stat;
154	    }
155	}
156	else {
157	    if (/(.*)\.start_dist/) {
158		# start line of distribution: set $in_dist flag
159		# and save distribution name for future reference
160		$in_dist = $1;
161		$stat = $1;
162		$value = 0;
163	    }
164	    else {
165		($stat, $value) = /^(\S+)\s+(.*)/;
166	    }
167	}
168
169	$$hashref{$stat} = $value;
170    }
171
172    close($stathandle);
173    return $hashref;
174}
175
176
177#
178# pct_diff($old, $new) returns percent difference from $old to $new.
179#
180sub pct_diff
181{
182    my ($old, $new) = @_;
183    return ($old == 0) ? (($new == 0) ? 0 : 9999) : 100 * ($new - $old) / $old;
184}
185
186
187#
188# Statistics to ignore: these relate to simulator performance, not
189# correctness, so don't fail on changes here.
190#
191%ignore = (
192  'host_seconds' => 1,
193  'host_tick_rate' => 1,
194  'host_inst_rate' => 1,
195  'host_mem_usage' => 1
196);
197
198#
199# List of key statistics (always displayed)
200#  ==> list stats here WITHOUT trailing thread ID
201#
202@key_stat_list = (
203  'COM:IPC',
204  'ISSUE:MSIPC',
205  'COM:count',
206  'host_inst_rate',
207  'sim_insts',
208  'sim_ticks',
209  'host_mem_usage'
210);
211
212$key_stat_pattern = join('|', @key_stat_list);
213
214# initialize first statistics from each file
215
216$max_err_mag = 0;
217
218$refhash = parse_file(\*REF);
219$newhash = parse_file(\*NEW);
220
221# The string sim-smt prints on a divide by zero
222$divbyzero = '<err: divide by zero>';
223
224foreach $stat (sort keys %$refhash)
225{
226    $refvalue = $$refhash{$stat};
227    $newvalue = $$newhash{$stat};
228
229    if (!defined($newvalue)) {
230	# stat missing from new file
231	push @missing_stats, $stat;
232	next;
233    }
234
235    if ($stat =~ /($key_stat_pattern)/o) {
236	# key statistics: always record & display changes in these
237	push @key_stats, [$stat, $refvalue, $newvalue];
238    }
239
240    if ($ignore{$stat} or $refvalue eq $newvalue) {
241	# stat is in "ignore" list, or hasn't changed
242    }
243    else {
244	if ($refvalue eq $divbyzero || $newvalue eq $divbyzero) {
245	    # one or the other was a divide by zero:
246	    # no point in trying to quantify error
247	    print "$stat: $refvalue --> $newvalue\n";
248	}
249	else {
250	    $reldiff = pct_diff($refvalue, $newvalue);
251	    $diffmag = abs($reldiff);
252
253	    if ($diffmag > $err_thresh) {
254		push @errs,
255		[$stat, $refvalue, $newvalue, $reldiff];
256	    }
257
258	    if ($diffmag > $max_err_mag) {
259		$max_err_mag = $diffmag;
260	    }
261	}
262    }
263
264    # remove from new hash so we can detect added stats
265    delete $$newhash{$stat};
266}
267
268
269#
270# All done.  Print comparison summary.
271#
272
273printf("Maximum error magnitude: %+f%%\n\n", $max_err_mag);
274
275printf("  %-30s %10s %10s %10s   %7s\n", ' ', 'Reference', 'New Value', 'Abs Diff', 'Pct Chg');
276
277printf("Key statistics:\n\n");
278
279foreach $key_stat (@key_stats)
280{
281    ($statname, $refvalue, $newvalue, $reldiff) = @$key_stat;
282
283    # deduce format from reference value
284    $pointpos = rindex($refvalue, '.');
285    $digits = ($pointpos < 0) ? 0 :(length($refvalue) - $pointpos - 1);
286    $fmt = "%10.${digits}f";
287
288    # print differing values with absolute and relative error
289    printf("  %-30s $fmt $fmt $fmt  %+7.2f%%\n",
290	   $statname, $refvalue, $newvalue,
291	   $newvalue - $refvalue, pct_diff($refvalue, $newvalue));
292}
293
294printf("\nLargest $omit_count relative errors (> %d%%):\n\n", $err_thresh);
295
296$num_errs = 0;
297
298if ($opt_p)
299{
300    # sort differences by percent change
301    @errs = sort { abs($$b[3]) <=> abs($$a[3]) } @errs;
302}
303
304foreach $err (@errs)
305{
306    ($statname, $refvalue, $newvalue, $reldiff) = @$err;
307
308    # deduce format from reference value
309    $pointpos1 = rindex($refvalue, '.');
310    $digits1 = ($pointpos1 < 0) ? 0 :(length($refvalue) - $pointpos1 - 1);
311    $pointpos2 = rindex($newvalue, '.');
312    $digits2 = ($pointpos2 < 0) ? 0 :(length($newvalue) - $pointpos2 - 1);
313    $digits = ($digits1 > $digits2) ? $digits1 : $digits2;
314    $fmt = "%10.${digits}f";
315
316    # print differing values with absolute and relative error
317    printf("  %-30s $fmt $fmt $fmt  %+7.2f%%\n",
318	   $statname, $refvalue, $newvalue, $newvalue - $refvalue, $reldiff);
319
320    # only print top N errors
321    if (++$num_errs >= $omit_count)
322    {
323	print "[... additional errors omitted ...]\n";
324	last;
325    }
326}
327
328#
329# Report missing stats
330#
331# get count
332$missing_stats = scalar(@missing_stats);
333
334if ($missing_stats)
335{
336    print "\nMissing $missing_stats reference statistics:\n\n";
337    foreach $stat (@missing_stats)
338    {
339#	print "\t$stat\n";
340	printf "  %-50s    ", $stat;
341	print  "$$refhash{$stat}\n";
342    }
343}
344
345#
346# Any stats left in newhash are added since the reference file
347#
348
349@added_stats = keys %$newhash;
350
351# get count
352$added_stats = scalar(@added_stats);
353
354if ($added_stats)
355{
356    print "\nFound $added_stats new statistics:\n\n";
357    foreach $stat (sort @added_stats)
358    {
359#	print "\t$stat\n";
360	printf "  %-50s    ", $stat;
361	print  "$$newhash{$stat}\n";
362    }
363}
364
365cleanup();
366# Exit code is 0 if all stats are found (with no extras) & no stats error, 1 otherwise
367$status = ($missing_stats == 0 && $added_stats == 0 && $max_err_mag == 0.0) ? 0 : 1;
368exit $status;
369
370sub cleanup
371{
372    unlink($refheader) if ($refheader);
373    unlink($newheader) if ($newheader);
374}
375