1#!/usr/bin/perl
2# Copyright (c) 2001-2005 The Regents of The University of Michigan
3# All rights reserved.
4#
5# Redistribution and use in source and binary forms, with or without
6# modification, are permitted provided that the following conditions are
7# met: redistributions of source code must retain the above copyright
8# notice, this list of conditions and the following disclaimer;
9# redistributions in binary form must reproduce the above copyright
10# notice, this list of conditions and the following disclaimer in the
11# documentation and/or other materials provided with the distribution;
12# neither the name of the copyright holders nor the names of its
13# contributors may be used to endorse or promote products derived from
14# this software without specific prior written permission.
15#
16# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
17# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
18# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
19# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
20# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
21# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
22# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
23# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
24# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
25# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
26# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
27#
28# Authors: Steve Reinhardt
29
30#
31# This script diffs two SimpleScalar statistics output files.
32#
33
34use Getopt::Std;
35
| 1#!/usr/bin/perl
2# Copyright (c) 2001-2005 The Regents of The University of Michigan
3# All rights reserved.
4#
5# Redistribution and use in source and binary forms, with or without
6# modification, are permitted provided that the following conditions are
7# met: redistributions of source code must retain the above copyright
8# notice, this list of conditions and the following disclaimer;
9# redistributions in binary form must reproduce the above copyright
10# notice, this list of conditions and the following disclaimer in the
11# documentation and/or other materials provided with the distribution;
12# neither the name of the copyright holders nor the names of its
13# contributors may be used to endorse or promote products derived from
14# this software without specific prior written permission.
15#
16# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
17# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
18# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
19# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
20# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
21# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
22# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
23# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
24# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
25# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
26# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
27#
28# Authors: Steve Reinhardt
29
30#
31# This script diffs two SimpleScalar statistics output files.
32#
33
34use Getopt::Std;
35
|
36#
37# -t thresh sets threshold for ignoring differences (in %)
38# -p sorts differences by % chg (default is alphabetic)
39# -d ignores all distributions
40#
| 36getopts('adn:t:h');
|
41
| 37
|
42getopts('dfn:pt:h');
43
| |
44if ($#ARGV < 1)
45{
46 print "\nError: need two file arguments (<reference> <new>).\n";
| 38if ($#ARGV < 1)
39{
40 print "\nError: need two file arguments (<reference> <new>).\n";
|
47 print " Options: -d = Ignore distributions\n";
48 print " -p = Sort errors by percentage\n";
49 print " -h = Diff header info separately from stats\n";
50 print " -n <num> = Print top <num> errors (default 20)\n";
51 print " -t <num> = Error threshold in percent (default 1)\n\n";
52 die -1;
| 41 print " Options: -d = Ignore distributions\n";
42 print " -a = Sort errors alphabetically (default: by percentage)\n";
43 print " -h = Diff header info separately from stats\n";
44 print " -n <num> = Print top <num> errors (default 20, 0 for all)\n";
45 print " -t <num> = Ignore errors below <num> percent (default 0)\n\n";
46 exit;
|
53}
54
55open(REF, "<$ARGV[0]") or die "Error: can't open $ARGV[0].\n";
56open(NEW, "<$ARGV[1]") or die "Error: can't open $ARGV[1].\n";
57
58
59#
60# Things that really should be adjustable via the command line
61#
62
63# Ignorable error (in percent)
| 47}
48
49open(REF, "<$ARGV[0]") or die "Error: can't open $ARGV[0].\n";
50open(NEW, "<$ARGV[1]") or die "Error: can't open $ARGV[1].\n";
51
52
53#
54# Things that really should be adjustable via the command line
55#
56
57# Ignorable error (in percent)
|
64$err_thresh = ($opt_t) ? $opt_t : 0;
| 58$err_thresh = defined($opt_t) ? $opt_t : 0;
|
65
66# Number of stats to print before omitting
| 59
60# Number of stats to print before omitting
|
67$omit_count = ($opt_n) ? $opt_n : 20;
| 61$omit_count = defined($opt_n) ? $opt_n : 20;
|
68
69
70#
71# First copy everything up to the simulation statistics to a pair of
72# temporary files, stripping out date-related items, and do a plain
73# diff. Any differences in the arguments are not necessarily an issue;
74# any differences in the program output should be caught by the EIO
75# mechanism if an EIO file is used.
76#
77
78# copy_header takes input filehandle and output filename
79
80sub copy_header
81{
82 my ($inhandle, $outname) = @_;
83
84 open(OUTPUT, ">$outname") or die "Error: can't open $outname.\n";
85
86 while (<$inhandle>)
87 {
88 # strip out lines that can vary
89 next if /^(command line:|M5 compiled on |M5 simulation started |M5 executing on )/;
90 last if /Begin Simulation Statistics/;
91 print OUTPUT;
92 }
93 close OUTPUT;
94}
95
96if ($opt_h) {
97
98 # Diff header separately from stats
99
100 $refheader = "/tmp/smt-test.refheader.$$";
101 $newheader = "/tmp/smt-test.newheader.$$";
102
103 copy_header(\*REF, $refheader);
104 copy_header(\*NEW, $newheader);
105
106 print "\n===== Header and program output differences =====\n\n";
107
108 print `diff $refheader $newheader`;
109
110 print "\n===== Statistics differences =====\n\n";
111}
112
113#
114# Now parse statistics
115#
116
117#
118# This function takes an open filehandle and returns a reference to
119# a hash containing all the statistics variables and their values.
120#
121sub parse_file
122{
123 $stathandle = shift;
124
125 $in_dist = undef;
126 $hashref = { }; # initialize hash for values
127
128 while (<$stathandle>)
129 {
130 next if /^\s*$/; # skip blank lines
131 last if /End Simulation Statistics/;
132
133 s/ *#.*//; # strip comments
134
135 if (/^Memory usage: (\d+) KBytes/) {
136 $stat = 'memory usage';
137 $value = $1;
138 }
139 elsif ($in_dist) {
140 if (/(.*)\.end_dist/) {
141 # end line of distribution: clear $in_dist flag
142 $in_dist = undef;
143 next;
144 }
145 if ($opt_d) {
146 next; # bail out if we are ignoring dists...
147 } elsif (/(.*)\.(min|max)_value/) {
148 # treat these like normal stats
149 ($stat, $value) = /^(\S+)\s+(.*)/;
150 } else {
151 ($stat, $value) =
152 /^(\S+(?:.*\S)?)\s+(\d+)\s+\d+\.\d+%/;
153 $stat = $in_dist . '::' . $stat;
154 }
155 }
156 else {
157 if (/(.*)\.start_dist/) {
158 # start line of distribution: set $in_dist flag
159 # and save distribution name for future reference
160 $in_dist = $1;
161 $stat = $1;
162 $value = 0;
163 }
164 else {
165 ($stat, $value) = /^(\S+)\s+(.*)/;
166 }
167 }
168
169 $$hashref{$stat} = $value;
170 }
171
172 close($stathandle);
173 return $hashref;
174}
175
176
177#
178# pct_diff($old, $new) returns percent difference from $old to $new.
179#
180sub pct_diff
181{
182 my ($old, $new) = @_;
183 return ($old == 0) ? (($new == 0) ? 0 : 9999) : 100 * ($new - $old) / $old;
184}
185
186
187#
188# Statistics to ignore: these relate to simulator performance, not
189# correctness, so don't fail on changes here.
190#
191%ignore = (
192 'host_seconds' => 1,
193 'host_tick_rate' => 1,
194 'host_inst_rate' => 1,
195 'host_mem_usage' => 1
196);
197
198#
199# List of key statistics (always displayed)
200# ==> list stats here WITHOUT trailing thread ID
201#
202@key_stat_list = (
203 'COM:IPC',
204 'ISSUE:MSIPC',
205 'COM:count',
206 'host_inst_rate',
207 'sim_insts',
208 'sim_ticks',
209 'host_mem_usage'
210);
211
212$key_stat_pattern = join('|', @key_stat_list);
213
214# initialize first statistics from each file
215
216$max_err_mag = 0;
217
218$refhash = parse_file(\*REF);
219$newhash = parse_file(\*NEW);
220
221# The string sim-smt prints on a divide by zero
222$divbyzero = '<err: divide by zero>';
223
224foreach $stat (sort keys %$refhash)
225{
226 $refvalue = $$refhash{$stat};
227 $newvalue = $$newhash{$stat};
228
229 if (!defined($newvalue)) {
230 # stat missing from new file
231 push @missing_stats, $stat;
232 next;
233 }
234
235 if ($stat =~ /($key_stat_pattern)/o) {
236 # key statistics: always record & display changes in these
237 push @key_stats, [$stat, $refvalue, $newvalue];
238 }
239
240 if ($ignore{$stat} or $refvalue eq $newvalue) {
241 # stat is in "ignore" list, or hasn't changed
242 }
243 else {
244 if ($refvalue eq $divbyzero || $newvalue eq $divbyzero) {
245 # one or the other was a divide by zero:
246 # no point in trying to quantify error
247 print "$stat: $refvalue --> $newvalue\n";
248 }
249 else {
250 $reldiff = pct_diff($refvalue, $newvalue);
251 $diffmag = abs($reldiff);
252
253 if ($diffmag > $err_thresh) {
254 push @errs,
255 [$stat, $refvalue, $newvalue, $reldiff];
256 }
257
258 if ($diffmag > $max_err_mag) {
259 $max_err_mag = $diffmag;
260 }
261 }
262 }
263
264 # remove from new hash so we can detect added stats
265 delete $$newhash{$stat};
266}
267
268
269#
270# All done. Print comparison summary.
271#
272
273printf("Maximum error magnitude: %+f%%\n\n", $max_err_mag);
274
275printf(" %-30s %10s %10s %10s %7s\n", ' ', 'Reference', 'New Value', 'Abs Diff', 'Pct Chg');
276
277printf("Key statistics:\n\n");
278
279foreach $key_stat (@key_stats)
280{
281 ($statname, $refvalue, $newvalue, $reldiff) = @$key_stat;
282
283 # deduce format from reference value
284 $pointpos = rindex($refvalue, '.');
285 $digits = ($pointpos < 0) ? 0 :(length($refvalue) - $pointpos - 1);
286 $fmt = "%10.${digits}f";
287
288 # print differing values with absolute and relative error
289 printf(" %-30s $fmt $fmt $fmt %+7.2f%%\n",
290 $statname, $refvalue, $newvalue,
291 $newvalue - $refvalue, pct_diff($refvalue, $newvalue));
292}
293
| 62
63
64#
65# First copy everything up to the simulation statistics to a pair of
66# temporary files, stripping out date-related items, and do a plain
67# diff. Any differences in the arguments are not necessarily an issue;
68# any differences in the program output should be caught by the EIO
69# mechanism if an EIO file is used.
70#
71
72# copy_header takes input filehandle and output filename
73
74sub copy_header
75{
76 my ($inhandle, $outname) = @_;
77
78 open(OUTPUT, ">$outname") or die "Error: can't open $outname.\n";
79
80 while (<$inhandle>)
81 {
82 # strip out lines that can vary
83 next if /^(command line:|M5 compiled on |M5 simulation started |M5 executing on )/;
84 last if /Begin Simulation Statistics/;
85 print OUTPUT;
86 }
87 close OUTPUT;
88}
89
90if ($opt_h) {
91
92 # Diff header separately from stats
93
94 $refheader = "/tmp/smt-test.refheader.$$";
95 $newheader = "/tmp/smt-test.newheader.$$";
96
97 copy_header(\*REF, $refheader);
98 copy_header(\*NEW, $newheader);
99
100 print "\n===== Header and program output differences =====\n\n";
101
102 print `diff $refheader $newheader`;
103
104 print "\n===== Statistics differences =====\n\n";
105}
106
107#
108# Now parse statistics
109#
110
111#
112# This function takes an open filehandle and returns a reference to
113# a hash containing all the statistics variables and their values.
114#
115sub parse_file
116{
117 $stathandle = shift;
118
119 $in_dist = undef;
120 $hashref = { }; # initialize hash for values
121
122 while (<$stathandle>)
123 {
124 next if /^\s*$/; # skip blank lines
125 last if /End Simulation Statistics/;
126
127 s/ *#.*//; # strip comments
128
129 if (/^Memory usage: (\d+) KBytes/) {
130 $stat = 'memory usage';
131 $value = $1;
132 }
133 elsif ($in_dist) {
134 if (/(.*)\.end_dist/) {
135 # end line of distribution: clear $in_dist flag
136 $in_dist = undef;
137 next;
138 }
139 if ($opt_d) {
140 next; # bail out if we are ignoring dists...
141 } elsif (/(.*)\.(min|max)_value/) {
142 # treat these like normal stats
143 ($stat, $value) = /^(\S+)\s+(.*)/;
144 } else {
145 ($stat, $value) =
146 /^(\S+(?:.*\S)?)\s+(\d+)\s+\d+\.\d+%/;
147 $stat = $in_dist . '::' . $stat;
148 }
149 }
150 else {
151 if (/(.*)\.start_dist/) {
152 # start line of distribution: set $in_dist flag
153 # and save distribution name for future reference
154 $in_dist = $1;
155 $stat = $1;
156 $value = 0;
157 }
158 else {
159 ($stat, $value) = /^(\S+)\s+(.*)/;
160 }
161 }
162
163 $$hashref{$stat} = $value;
164 }
165
166 close($stathandle);
167 return $hashref;
168}
169
170
171#
172# pct_diff($old, $new) returns percent difference from $old to $new.
173#
174sub pct_diff
175{
176 my ($old, $new) = @_;
177 return ($old == 0) ? (($new == 0) ? 0 : 9999) : 100 * ($new - $old) / $old;
178}
179
180
181#
182# Statistics to ignore: these relate to simulator performance, not
183# correctness, so don't fail on changes here.
184#
185%ignore = (
186 'host_seconds' => 1,
187 'host_tick_rate' => 1,
188 'host_inst_rate' => 1,
189 'host_mem_usage' => 1
190);
191
192#
193# List of key statistics (always displayed)
194# ==> list stats here WITHOUT trailing thread ID
195#
196@key_stat_list = (
197 'COM:IPC',
198 'ISSUE:MSIPC',
199 'COM:count',
200 'host_inst_rate',
201 'sim_insts',
202 'sim_ticks',
203 'host_mem_usage'
204);
205
206$key_stat_pattern = join('|', @key_stat_list);
207
208# initialize first statistics from each file
209
210$max_err_mag = 0;
211
212$refhash = parse_file(\*REF);
213$newhash = parse_file(\*NEW);
214
215# The string sim-smt prints on a divide by zero
216$divbyzero = '<err: divide by zero>';
217
218foreach $stat (sort keys %$refhash)
219{
220 $refvalue = $$refhash{$stat};
221 $newvalue = $$newhash{$stat};
222
223 if (!defined($newvalue)) {
224 # stat missing from new file
225 push @missing_stats, $stat;
226 next;
227 }
228
229 if ($stat =~ /($key_stat_pattern)/o) {
230 # key statistics: always record & display changes in these
231 push @key_stats, [$stat, $refvalue, $newvalue];
232 }
233
234 if ($ignore{$stat} or $refvalue eq $newvalue) {
235 # stat is in "ignore" list, or hasn't changed
236 }
237 else {
238 if ($refvalue eq $divbyzero || $newvalue eq $divbyzero) {
239 # one or the other was a divide by zero:
240 # no point in trying to quantify error
241 print "$stat: $refvalue --> $newvalue\n";
242 }
243 else {
244 $reldiff = pct_diff($refvalue, $newvalue);
245 $diffmag = abs($reldiff);
246
247 if ($diffmag > $err_thresh) {
248 push @errs,
249 [$stat, $refvalue, $newvalue, $reldiff];
250 }
251
252 if ($diffmag > $max_err_mag) {
253 $max_err_mag = $diffmag;
254 }
255 }
256 }
257
258 # remove from new hash so we can detect added stats
259 delete $$newhash{$stat};
260}
261
262
263#
264# All done. Print comparison summary.
265#
266
267printf("Maximum error magnitude: %+f%%\n\n", $max_err_mag);
268
269printf(" %-30s %10s %10s %10s %7s\n", ' ', 'Reference', 'New Value', 'Abs Diff', 'Pct Chg');
270
271printf("Key statistics:\n\n");
272
273foreach $key_stat (@key_stats)
274{
275 ($statname, $refvalue, $newvalue, $reldiff) = @$key_stat;
276
277 # deduce format from reference value
278 $pointpos = rindex($refvalue, '.');
279 $digits = ($pointpos < 0) ? 0 :(length($refvalue) - $pointpos - 1);
280 $fmt = "%10.${digits}f";
281
282 # print differing values with absolute and relative error
283 printf(" %-30s $fmt $fmt $fmt %+7.2f%%\n",
284 $statname, $refvalue, $newvalue,
285 $newvalue - $refvalue, pct_diff($refvalue, $newvalue));
286}
287
|
294printf("\nLargest $omit_count relative errors (> %d%%):\n\n", $err_thresh);
| 288printf("\nDifferences > %d%%:\n\n", $err_thresh);
|
295
| 289
|
296$num_errs = 0;
297
298if ($opt_p)
299{
| 290if ($opt_a) {
291 # leave stats sorted alphabetically, doesn't make sense to cut them off
292 $omit_count = 0;
293} else {
|
300 # sort differences by percent change
301 @errs = sort { abs($$b[3]) <=> abs($$a[3]) } @errs;
302}
303
| 294 # sort differences by percent change
295 @errs = sort { abs($$b[3]) <=> abs($$a[3]) } @errs;
296}
297
|
| 298$num_errs = 0;
299
|
304foreach $err (@errs)
305{
306 ($statname, $refvalue, $newvalue, $reldiff) = @$err;
307
308 # deduce format from reference value
309 $pointpos1 = rindex($refvalue, '.');
310 $digits1 = ($pointpos1 < 0) ? 0 :(length($refvalue) - $pointpos1 - 1);
311 $pointpos2 = rindex($newvalue, '.');
312 $digits2 = ($pointpos2 < 0) ? 0 :(length($newvalue) - $pointpos2 - 1);
313 $digits = ($digits1 > $digits2) ? $digits1 : $digits2;
314 $fmt = "%10.${digits}f";
315
316 # print differing values with absolute and relative error
317 printf(" %-30s $fmt $fmt $fmt %+7.2f%%\n",
318 $statname, $refvalue, $newvalue, $newvalue - $refvalue, $reldiff);
319
320 # only print top N errors
| 300foreach $err (@errs)
301{
302 ($statname, $refvalue, $newvalue, $reldiff) = @$err;
303
304 # deduce format from reference value
305 $pointpos1 = rindex($refvalue, '.');
306 $digits1 = ($pointpos1 < 0) ? 0 :(length($refvalue) - $pointpos1 - 1);
307 $pointpos2 = rindex($newvalue, '.');
308 $digits2 = ($pointpos2 < 0) ? 0 :(length($newvalue) - $pointpos2 - 1);
309 $digits = ($digits1 > $digits2) ? $digits1 : $digits2;
310 $fmt = "%10.${digits}f";
311
312 # print differing values with absolute and relative error
313 printf(" %-30s $fmt $fmt $fmt %+7.2f%%\n",
314 $statname, $refvalue, $newvalue, $newvalue - $refvalue, $reldiff);
315
316 # only print top N errors
|
321 if (++$num_errs >= $omit_count)
| 317 if ($omit_count > 0 && ++$num_errs >= $omit_count)
|
322 {
| 318 {
|
323 print "[... additional errors omitted ...]\n";
| 319 print "[... showing top $omit_count errors only, additional errors omitted ...]\n";
|
324 last;
325 }
326}
327
328#
329# Report missing stats
330#
331# get count
332$missing_stats = scalar(@missing_stats);
333
334if ($missing_stats)
335{
336 print "\nMissing $missing_stats reference statistics:\n\n";
337 foreach $stat (@missing_stats)
338 {
339# print "\t$stat\n";
340 printf " %-50s ", $stat;
341 print "$$refhash{$stat}\n";
342 }
343}
344
345#
346# Any stats left in newhash are added since the reference file
347#
348
349@added_stats = keys %$newhash;
350
351# get count
352$added_stats = scalar(@added_stats);
353
354if ($added_stats)
355{
356 print "\nFound $added_stats new statistics:\n\n";
357 foreach $stat (sort @added_stats)
358 {
359# print "\t$stat\n";
360 printf " %-50s ", $stat;
361 print "$$newhash{$stat}\n";
362 }
363}
364
365cleanup();
366# Exit code is 0 if all stats are found (with no extras) & no stats error, 1 otherwise
367$status = ($missing_stats == 0 && $added_stats == 0 && $max_err_mag == 0.0) ? 0 : 1;
368exit $status;
369
370sub cleanup
371{
372 unlink($refheader) if ($refheader);
373 unlink($newheader) if ($newheader);
374}
| 320 last;
321 }
322}
323
324#
325# Report missing stats
326#
327# get count
328$missing_stats = scalar(@missing_stats);
329
330if ($missing_stats)
331{
332 print "\nMissing $missing_stats reference statistics:\n\n";
333 foreach $stat (@missing_stats)
334 {
335# print "\t$stat\n";
336 printf " %-50s ", $stat;
337 print "$$refhash{$stat}\n";
338 }
339}
340
341#
342# Any stats left in newhash are added since the reference file
343#
344
345@added_stats = keys %$newhash;
346
347# get count
348$added_stats = scalar(@added_stats);
349
350if ($added_stats)
351{
352 print "\nFound $added_stats new statistics:\n\n";
353 foreach $stat (sort @added_stats)
354 {
355# print "\t$stat\n";
356 printf " %-50s ", $stat;
357 print "$$newhash{$stat}\n";
358 }
359}
360
361cleanup();
362# Exit code is 0 if all stats are found (with no extras) & no stats error, 1 otherwise
363$status = ($missing_stats == 0 && $added_stats == 0 && $max_err_mag == 0.0) ? 0 : 1;
364exit $status;
365
366sub cleanup
367{
368 unlink($refheader) if ($refheader);
369 unlink($newheader) if ($newheader);
370}
|