diff-out revision 7448:ba1a0193c050
1955SN/A#!/usr/bin/perl
2955SN/A# Copyright (c) 2001-2005 The Regents of The University of Michigan
31762SN/A# All rights reserved.
4955SN/A#
5955SN/A# Redistribution and use in source and binary forms, with or without
6955SN/A# modification, are permitted provided that the following conditions are
7955SN/A# met: redistributions of source code must retain the above copyright
8955SN/A# notice, this list of conditions and the following disclaimer;
9955SN/A# redistributions in binary form must reproduce the above copyright
10955SN/A# notice, this list of conditions and the following disclaimer in the
11955SN/A# documentation and/or other materials provided with the distribution;
12955SN/A# neither the name of the copyright holders nor the names of its
13955SN/A# contributors may be used to endorse or promote products derived from
14955SN/A# this software without specific prior written permission.
15955SN/A#
16955SN/A# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
17955SN/A# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
18955SN/A# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
19955SN/A# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
20955SN/A# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
21955SN/A# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
22955SN/A# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
23955SN/A# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
24955SN/A# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
25955SN/A# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
26955SN/A# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
27955SN/A#
282665Ssaidi@eecs.umich.edu# Authors: Steve Reinhardt
294762Snate@binkert.org
30955SN/A#
3112563Sgabeblack@google.com# This script diffs two SimpleScalar statistics output files.
3212563Sgabeblack@google.com#
335522Snate@binkert.org
346143Snate@binkert.orguse Getopt::Std;
3512371Sgabeblack@google.com
364762Snate@binkert.org#
375522Snate@binkert.org# -t thresh sets threshold for ignoring differences (in %)
38955SN/A# -p sorts differences by % chg (default is alphabetic)
395522Snate@binkert.org# -d ignores all distributions
4011974Sgabeblack@google.com#
41955SN/A
425522Snate@binkert.orggetopts('dfn:pt:h');
434202Sbinkertn@umich.edu
445742Snate@binkert.orgif ($#ARGV < 1)
45955SN/A{
464381Sbinkertn@umich.edu    print "\nError: need two file arguments (<reference> <new>).\n";
474381Sbinkertn@umich.edu    print "   Options: -d  =  Ignore distributions\n";
4812246Sgabeblack@google.com    print "            -p  =  Sort errors by percentage\n";
4912246Sgabeblack@google.com    print "            -h  =  Diff header info separately from stats\n";
508334Snate@binkert.org    print "            -n <num>  =  Print top <num> errors (default 20)\n";
51955SN/A    print "            -t <num>  =  Error threshold in percent (default 1)\n\n";
52955SN/A    die -1;
534202Sbinkertn@umich.edu}
54955SN/A
554382Sbinkertn@umich.eduopen(REF, "<$ARGV[0]") or die "Error: can't open $ARGV[0].\n";
564382Sbinkertn@umich.eduopen(NEW, "<$ARGV[1]") or die "Error: can't open $ARGV[1].\n";
574382Sbinkertn@umich.edu
586654Snate@binkert.org
595517Snate@binkert.org#
608614Sgblack@eecs.umich.edu# Things that really should be adjustable via the command line
617674Snate@binkert.org#
626143Snate@binkert.org
636143Snate@binkert.org# Ignorable error (in percent)
646143Snate@binkert.org$err_thresh = ($opt_t) ? $opt_t : 0;
6512302Sgabeblack@google.com
6612302Sgabeblack@google.com# Number of stats to print before omitting
6712302Sgabeblack@google.com$omit_count = ($opt_n) ? $opt_n : 20;
6812371Sgabeblack@google.com
6912371Sgabeblack@google.com
7012371Sgabeblack@google.com#
7112371Sgabeblack@google.com# First copy everything up to the simulation statistics to a pair of
7212371Sgabeblack@google.com# temporary files, stripping out date-related items, and do a plain
7312371Sgabeblack@google.com# diff.  Any differences in the arguments are not necessarily an issue;
7412371Sgabeblack@google.com# any differences in the program output should be caught by the EIO
7512371Sgabeblack@google.com# mechanism if an EIO file is used.
7612371Sgabeblack@google.com# 
7712371Sgabeblack@google.com
7812371Sgabeblack@google.com# copy_header takes input filehandle and output filename
7912371Sgabeblack@google.com
8012371Sgabeblack@google.comsub copy_header
8112371Sgabeblack@google.com{
8212371Sgabeblack@google.com    my ($inhandle, $outname) = @_;
8312371Sgabeblack@google.com
8412371Sgabeblack@google.com    open(OUTPUT, ">$outname") or die "Error: can't open $outname.\n";
8512371Sgabeblack@google.com
8612371Sgabeblack@google.com    while (<$inhandle>)
8712371Sgabeblack@google.com    {
8812371Sgabeblack@google.com	# strip out lines that can vary
8912371Sgabeblack@google.com	next if /^(command line:|M5 compiled on |M5 simulation started |M5 executing on )/;
9012371Sgabeblack@google.com	last if /Begin Simulation Statistics/;
9112371Sgabeblack@google.com	print OUTPUT;
9212371Sgabeblack@google.com    }
9312371Sgabeblack@google.com    close OUTPUT;
9412371Sgabeblack@google.com}
9512371Sgabeblack@google.com
9612371Sgabeblack@google.comif ($opt_h) {
9712371Sgabeblack@google.com
9812371Sgabeblack@google.com    # Diff header separately from stats
9912371Sgabeblack@google.com
10012371Sgabeblack@google.com    $refheader = "/tmp/smt-test.refheader.$$";
10112371Sgabeblack@google.com    $newheader = "/tmp/smt-test.newheader.$$";
10212371Sgabeblack@google.com
10312371Sgabeblack@google.com    copy_header(\*REF, $refheader);
10412371Sgabeblack@google.com    copy_header(\*NEW, $newheader);
10512371Sgabeblack@google.com
10612371Sgabeblack@google.com    print "\n===== Header and program output differences =====\n\n";
10712371Sgabeblack@google.com
10812371Sgabeblack@google.com    print `diff $refheader $newheader`;
10912371Sgabeblack@google.com
11012371Sgabeblack@google.com    print "\n===== Statistics differences =====\n\n";
11112371Sgabeblack@google.com}
11212371Sgabeblack@google.com
11312371Sgabeblack@google.com#
11412371Sgabeblack@google.com# Now parse statistics
11512302Sgabeblack@google.com#
11612371Sgabeblack@google.com
11712302Sgabeblack@google.com#
11812371Sgabeblack@google.com# This function takes an open filehandle and returns a reference to
11912302Sgabeblack@google.com# a hash containing all the statistics variables and their values.
12012302Sgabeblack@google.com#
12112371Sgabeblack@google.comsub parse_file
12212371Sgabeblack@google.com{
12312371Sgabeblack@google.com    $stathandle = shift;
12412371Sgabeblack@google.com
12512302Sgabeblack@google.com    $in_dist = undef;
12612371Sgabeblack@google.com    $hashref = { };	# initialize hash for values
12712371Sgabeblack@google.com
12812371Sgabeblack@google.com    while (<$stathandle>)
12912371Sgabeblack@google.com    {
13011983Sgabeblack@google.com	next if /^\s*$/;	# skip blank lines
1316143Snate@binkert.org	last if /End Simulation Statistics/;
1328233Snate@binkert.org
13312302Sgabeblack@google.com	s/ *#.*//;		# strip comments
1346143Snate@binkert.org
1356143Snate@binkert.org	if (/^Memory usage: (\d+) KBytes/) {
13612302Sgabeblack@google.com	    $stat = 'memory usage';
1374762Snate@binkert.org	    $value = $1;
1386143Snate@binkert.org	}
1398233Snate@binkert.org	elsif ($in_dist) {
1408233Snate@binkert.org	    if (/(.*)\.end_dist/) {
14112302Sgabeblack@google.com		# end line of distribution: clear $in_dist flag
14212302Sgabeblack@google.com		$in_dist = undef;
1436143Snate@binkert.org		next;
14412362Sgabeblack@google.com	    }
14512362Sgabeblack@google.com	    if ($opt_d) {
14612362Sgabeblack@google.com		next;		#  bail out if we are ignoring dists...
14712362Sgabeblack@google.com	    } elsif (/(.*)\.(min|max)_value/) {
14812302Sgabeblack@google.com		# treat these like normal stats
14912302Sgabeblack@google.com		($stat, $value) = /^(\S+)\s+(.*)/;
15012302Sgabeblack@google.com	    } else {
15112302Sgabeblack@google.com		($stat, $value) =
15212302Sgabeblack@google.com		  /^(\S+(?:.*\S)?)\s+(\d+)\s+\d+\.\d+%/;
15312363Sgabeblack@google.com		$stat = $in_dist . '::' . $stat;
15412363Sgabeblack@google.com	    }
15512363Sgabeblack@google.com	}
15612363Sgabeblack@google.com	else {
15712302Sgabeblack@google.com	    if (/(.*)\.start_dist/) {
15812363Sgabeblack@google.com		# start line of distribution: set $in_dist flag
15912363Sgabeblack@google.com		# and save distribution name for future reference
16012363Sgabeblack@google.com		$in_dist = $1;
16112363Sgabeblack@google.com		$stat = $1;
16212363Sgabeblack@google.com		$value = 0;
1638233Snate@binkert.org	    }
1646143Snate@binkert.org	    else {
1656143Snate@binkert.org		($stat, $value) = /^(\S+)\s+(.*)/;
1666143Snate@binkert.org	    }
1676143Snate@binkert.org	}
1686143Snate@binkert.org
1696143Snate@binkert.org	$$hashref{$stat} = $value;
1706143Snate@binkert.org    }
1716143Snate@binkert.org
1726143Snate@binkert.org    close($stathandle);
1737065Snate@binkert.org    return $hashref;
1746143Snate@binkert.org}
17512362Sgabeblack@google.com
17612362Sgabeblack@google.com
17712362Sgabeblack@google.com#
17812362Sgabeblack@google.com# pct_diff($old, $new) returns percent difference from $old to $new.
17912362Sgabeblack@google.com#
18012362Sgabeblack@google.comsub pct_diff
18112362Sgabeblack@google.com{
18212362Sgabeblack@google.com    my ($old, $new) = @_;
18312362Sgabeblack@google.com    return ($old == 0) ? (($new == 0) ? 0 : 9999) : 100 * ($new - $old) / $old;
18412362Sgabeblack@google.com}
18512362Sgabeblack@google.com
18612362Sgabeblack@google.com
1878233Snate@binkert.org#
1888233Snate@binkert.org# Statistics to ignore: these relate to simulator performance, not
1898233Snate@binkert.org# correctness, so don't fail on changes here.
1908233Snate@binkert.org#
1918233Snate@binkert.org%ignore = (
1928233Snate@binkert.org  'host_seconds' => 1,
1938233Snate@binkert.org  'host_tick_rate' => 1,
1948233Snate@binkert.org  'host_inst_rate' => 1,
1958233Snate@binkert.org  'host_mem_usage' => 1
1968233Snate@binkert.org);
1978233Snate@binkert.org
1988233Snate@binkert.org#
1998233Snate@binkert.org# List of key statistics (always displayed)
2008233Snate@binkert.org#  ==> list stats here WITHOUT trailing thread ID
2018233Snate@binkert.org#
2028233Snate@binkert.org@key_stat_list = (
2038233Snate@binkert.org  'COM:IPC',
2048233Snate@binkert.org  'ISSUE:MSIPC',
2058233Snate@binkert.org  'COM:count',
2068233Snate@binkert.org  'host_inst_rate',
2078233Snate@binkert.org  'sim_insts',
2086143Snate@binkert.org  'sim_ticks',
2096143Snate@binkert.org  'host_mem_usage'
2106143Snate@binkert.org);
2116143Snate@binkert.org
2126143Snate@binkert.org$key_stat_pattern = join('|', @key_stat_list);
2136143Snate@binkert.org
2149982Satgutier@umich.edu# initialize first statistics from each file
2156143Snate@binkert.org
21612302Sgabeblack@google.com$max_err_mag = 0;
21712302Sgabeblack@google.com
21812302Sgabeblack@google.com$refhash = parse_file(\*REF);
21912302Sgabeblack@google.com$newhash = parse_file(\*NEW);
22012302Sgabeblack@google.com
22112302Sgabeblack@google.com# The string sim-smt prints on a divide by zero
22212302Sgabeblack@google.com$divbyzero = '<err: divide by zero>';
22312302Sgabeblack@google.com
22411983Sgabeblack@google.comforeach $stat (sort keys %$refhash)
22511983Sgabeblack@google.com{
22611983Sgabeblack@google.com    $refvalue = $$refhash{$stat};
22712302Sgabeblack@google.com    $newvalue = $$newhash{$stat};
22812302Sgabeblack@google.com
22912302Sgabeblack@google.com    if (!defined($newvalue)) {
23012302Sgabeblack@google.com	# stat missing from new file
23112302Sgabeblack@google.com	push @missing_stats, $stat;
23212302Sgabeblack@google.com	next;
23311983Sgabeblack@google.com    }
2346143Snate@binkert.org
23512305Sgabeblack@google.com    if ($stat =~ /($key_stat_pattern)/o) {
23612302Sgabeblack@google.com	# key statistics: always record & display changes in these
23712302Sgabeblack@google.com	push @key_stats, [$stat, $refvalue, $newvalue];
23812302Sgabeblack@google.com    }
2396143Snate@binkert.org
2406143Snate@binkert.org    if ($ignore{$stat} or $refvalue eq $newvalue) {
2416143Snate@binkert.org	# stat is in "ignore" list, or hasn't changed
2425522Snate@binkert.org    }
2436143Snate@binkert.org    else {
2446143Snate@binkert.org	if ($refvalue eq $divbyzero || $newvalue eq $divbyzero) {
2456143Snate@binkert.org	    # one or the other was a divide by zero:
2469982Satgutier@umich.edu	    # no point in trying to quantify error
24712302Sgabeblack@google.com	    print "$stat: $refvalue --> $newvalue\n";
24812302Sgabeblack@google.com	}
24912302Sgabeblack@google.com	else {
2506143Snate@binkert.org	    $reldiff = pct_diff($refvalue, $newvalue);
2516143Snate@binkert.org	    $diffmag = abs($reldiff);
2526143Snate@binkert.org
2536143Snate@binkert.org	    if ($diffmag > $err_thresh) {
2545522Snate@binkert.org		push @errs,
2555522Snate@binkert.org		[$stat, $refvalue, $newvalue, $reldiff];
2565522Snate@binkert.org	    }
2575522Snate@binkert.org
2585604Snate@binkert.org	    if ($diffmag > $max_err_mag) {
2595604Snate@binkert.org		$max_err_mag = $diffmag;
2606143Snate@binkert.org	    }
2616143Snate@binkert.org	}
2624762Snate@binkert.org    }
2634762Snate@binkert.org
2646143Snate@binkert.org    # remove from new hash so we can detect added stats
2656727Ssteve.reinhardt@amd.com    delete $$newhash{$stat};
2666727Ssteve.reinhardt@amd.com}
2676727Ssteve.reinhardt@amd.com
2684762Snate@binkert.org
2696143Snate@binkert.org#
2706143Snate@binkert.org# All done.  Print comparison summary.
2716143Snate@binkert.org#
2726143Snate@binkert.org
2736727Ssteve.reinhardt@amd.comprintf("Maximum error magnitude: %+f%%\n\n", $max_err_mag);
2746143Snate@binkert.org
2757674Snate@binkert.orgprintf("  %-30s %10s %10s %10s   %7s\n", ' ', 'Reference', 'New Value', 'Abs Diff', 'Pct Chg');
2767674Snate@binkert.org
2775604Snate@binkert.orgprintf("Key statistics:\n\n");
2786143Snate@binkert.org
2796143Snate@binkert.orgforeach $key_stat (@key_stats)
2806143Snate@binkert.org{
2814762Snate@binkert.org    ($statname, $refvalue, $newvalue, $reldiff) = @$key_stat;
2826143Snate@binkert.org
2834762Snate@binkert.org    # deduce format from reference value
2844762Snate@binkert.org    $pointpos = rindex($refvalue, '.');
2854762Snate@binkert.org    $digits = ($pointpos < 0) ? 0 :(length($refvalue) - $pointpos - 1);
2866143Snate@binkert.org    $fmt = "%10.${digits}f";
2876143Snate@binkert.org
2884762Snate@binkert.org    # print differing values with absolute and relative error
28912302Sgabeblack@google.com    printf("  %-30s $fmt $fmt $fmt  %+7.2f%%\n",
29012302Sgabeblack@google.com	   $statname, $refvalue, $newvalue,
2918233Snate@binkert.org	   $newvalue - $refvalue, pct_diff($refvalue, $newvalue));
29212302Sgabeblack@google.com}
2936143Snate@binkert.org
2946143Snate@binkert.orgprintf("\nLargest $omit_count relative errors (> %d%%):\n\n", $err_thresh);
2954762Snate@binkert.org
2966143Snate@binkert.org$num_errs = 0;
2974762Snate@binkert.org
2989396Sandreas.hansson@arm.comif ($opt_p)
2999396Sandreas.hansson@arm.com{
3009396Sandreas.hansson@arm.com    # sort differences by percent change
30112302Sgabeblack@google.com    @errs = sort { abs($$b[3]) <=> abs($$a[3]) } @errs;
30212302Sgabeblack@google.com}
30312302Sgabeblack@google.com
3049396Sandreas.hansson@arm.comforeach $err (@errs)
3059396Sandreas.hansson@arm.com{
3069396Sandreas.hansson@arm.com    ($statname, $refvalue, $newvalue, $reldiff) = @$err;
3079396Sandreas.hansson@arm.com
3089396Sandreas.hansson@arm.com    # deduce format from reference value
3099396Sandreas.hansson@arm.com    $pointpos1 = rindex($refvalue, '.');
3109396Sandreas.hansson@arm.com    $digits1 = ($pointpos1 < 0) ? 0 :(length($refvalue) - $pointpos1 - 1);
3119930Sandreas.hansson@arm.com    $pointpos2 = rindex($newvalue, '.');
3129930Sandreas.hansson@arm.com    $digits2 = ($pointpos2 < 0) ? 0 :(length($newvalue) - $pointpos2 - 1);
3139396Sandreas.hansson@arm.com    $digits = ($digits1 > $digits2) ? $digits1 : $digits2;
3146143Snate@binkert.org    $fmt = "%10.${digits}f";
31512797Sgabeblack@google.com
31612797Sgabeblack@google.com    # print differing values with absolute and relative error
31712797Sgabeblack@google.com    printf("  %-30s $fmt $fmt $fmt  %+7.2f%%\n",
3188235Snate@binkert.org	   $statname, $refvalue, $newvalue, $newvalue - $refvalue, $reldiff);
31912797Sgabeblack@google.com
32012797Sgabeblack@google.com    # only print top N errors
32112797Sgabeblack@google.com    if (++$num_errs >= $omit_count)
32212797Sgabeblack@google.com    {
32312797Sgabeblack@google.com	print "[... additional errors omitted ...]\n";
32412797Sgabeblack@google.com	last;
32512797Sgabeblack@google.com    }
32612797Sgabeblack@google.com}
32712797Sgabeblack@google.com
32812797Sgabeblack@google.com#
32912797Sgabeblack@google.com# Report missing stats
33012797Sgabeblack@google.com#
33112797Sgabeblack@google.com# get count
33212797Sgabeblack@google.com$missing_stats = scalar(@missing_stats);
33312797Sgabeblack@google.com
33412757Sgabeblack@google.comif ($missing_stats)
33512757Sgabeblack@google.com{
33612797Sgabeblack@google.com    print "\nMissing $missing_stats reference statistics:\n\n";
33712797Sgabeblack@google.com    foreach $stat (@missing_stats)
33812797Sgabeblack@google.com    {
33912757Sgabeblack@google.com#	print "\t$stat\n";
34012757Sgabeblack@google.com	printf "  %-50s    ", $stat;
34112757Sgabeblack@google.com	print  "$$refhash{$stat}\n";
34212757Sgabeblack@google.com    }
3438235Snate@binkert.org}
34412302Sgabeblack@google.com
3458235Snate@binkert.org#
3468235Snate@binkert.org# Any stats left in newhash are added since the reference file
34712757Sgabeblack@google.com#
3488235Snate@binkert.org
3498235Snate@binkert.org@added_stats = keys %$newhash;
3508235Snate@binkert.org
35112757Sgabeblack@google.com# get count
35212313Sgabeblack@google.com$added_stats = scalar(@added_stats);
35312797Sgabeblack@google.com
35412797Sgabeblack@google.comif ($added_stats)
35512797Sgabeblack@google.com{
35612797Sgabeblack@google.com    print "\nFound $added_stats new statistics:\n\n";
35712797Sgabeblack@google.com    foreach $stat (sort @added_stats)
35812797Sgabeblack@google.com    {
35912797Sgabeblack@google.com#	print "\t$stat\n";
36012797Sgabeblack@google.com	printf "  %-50s    ", $stat;
36112797Sgabeblack@google.com	print  "$$newhash{$stat}\n";
36212797Sgabeblack@google.com    }
36312797Sgabeblack@google.com}
36412797Sgabeblack@google.com
36512797Sgabeblack@google.comcleanup();
36612797Sgabeblack@google.com# Exit code is 0 if all stats are found (with no extras) & no stats error, 1 otherwise
36712797Sgabeblack@google.com$status = ($missing_stats == 0 && $added_stats == 0 && $max_err_mag == 0.0) ? 0 : 1;
36812797Sgabeblack@google.comexit $status;
36912797Sgabeblack@google.com
37012797Sgabeblack@google.comsub cleanup
37112797Sgabeblack@google.com{
37212797Sgabeblack@google.com    unlink($refheader) if ($refheader);
37312797Sgabeblack@google.com    unlink($newheader) if ($newheader);
37412797Sgabeblack@google.com}
37512797Sgabeblack@google.com