units.py revision 11482:2ca1efb451e4 
1#!/usr/bin/env python
2#
3# Copyright (c) 2016 ARM Limited
4# All rights reserved
5#
6# The license below extends only to copyright in the software and shall
7# not be construed as granting a license to any other intellectual
8# property including but not limited to intellectual property relating
9# to a hardware implementation of the functionality of the software
10# licensed hereunder.  You may use the software subject to the license
11# terms below provided that you ensure that this notice is replicated
12# unmodified and in its entirety in all distributions of the software,
13# modified or unmodified, in source code or in binary form.
14#
15# Redistribution and use in source and binary forms, with or without
16# modification, are permitted provided that the following conditions are
17# met: redistributions of source code must retain the above copyright
18# notice, this list of conditions and the following disclaimer;
19# redistributions in binary form must reproduce the above copyright
20# notice, this list of conditions and the following disclaimer in the
21# documentation and/or other materials provided with the distribution;
22# neither the name of the copyright holders nor the names of its
23# contributors may be used to endorse or promote products derived from
24# this software without specific prior written permission.
25#
26# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
27# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
28# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
29# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
30# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
31# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
32# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
33# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
34# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
35# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
36# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
37#
38# Authors: Andreas Sandberg
39
40from abc import ABCMeta, abstractmethod
41from datetime import datetime
42import difflib
43import functools
44import os
45import re
46import subprocess
47import sys
48import traceback
49
50from results import UnitResult
51from helpers import *
52
53_test_base = os.path.join(os.path.dirname(__file__), "..")
54
55class TestUnit(object):
56    """Base class for all test units.
57
58    A test unit is a part of a larger test case. Test cases usually
59    contain two types of units, run units (run gem5) and verify units
60    (diff output files). All unit implementations inherit from this
61    class.
62
63    A unit implementation overrides the _run() method. The test runner
64    calls the run() method, which wraps _run() to protect against
65    exceptions.
66
67    """
68
69    __metaclass__ = ABCMeta
70
71    def __init__(self, name, ref_dir, test_dir, skip=False):
72        self.name = name
73        self.ref_dir = ref_dir
74        self.test_dir = test_dir
75        self.force_skip = skip
76        self.start_time = None
77        self.stop_time = None
78
79    def result(self, state, **kwargs):
80        if self.start_time is not None and "runtime" not in kwargs:
81            self.stop_time = datetime.utcnow()
82            delta = self.stop_time - self.start_time
83            kwargs["runtime"] = delta.total_seconds()
84
85        return UnitResult(self.name, state, **kwargs)
86
87    def ok(self, **kwargs):
88        return self.result(UnitResult.STATE_OK, **kwargs)
89
90    def skip(self, **kwargs):
91        return self.result(UnitResult.STATE_SKIPPED, **kwargs)
92
93    def error(self, message, **kwargs):
94        return self.result(UnitResult.STATE_ERROR, message=message, **kwargs)
95
96    def failure(self, message, **kwargs):
97        return self.result(UnitResult.STATE_FAILURE, message=message, **kwargs)
98
99    def ref_file(self, fname):
100        return os.path.join(self.ref_dir, fname)
101
102    def out_file(self, fname):
103        return os.path.join(self.test_dir, fname)
104
105    def _read_output(self, fname, default=""):
106        try:
107            with open(self.out_file(fname), "r") as f:
108                return f.read()
109        except IOError:
110            return default
111
112    def run(self):
113        self.start_time = datetime.utcnow()
114        try:
115            if self.force_skip:
116                return self.skip()
117            else:
118                return self._run()
119        except:
120            return self.error("Python exception:\n%s" % traceback.format_exc())
121
122    @abstractmethod
123    def _run(self):
124        pass
125
126class RunGem5(TestUnit):
127    """Test unit representing a gem5 run.
128
129    Possible failure modes:
130       - gem5 failed to run -> STATE_ERROR
131       - timeout -> STATE_ERROR
132       - non-zero exit code -> STATE_ERROR
133
134    Possible non-failure results:
135       - exit code == 0 -> STATE_OK
136       - exit code == 2 -> STATE_SKIPPED
137    """
138
139    def __init__(self, gem5, gem5_args, timeout=0, **kwargs):
140        super(RunGem5, self).__init__("gem5", **kwargs)
141        self.gem5 = gem5
142        self.args = gem5_args
143        self.timeout = timeout
144
145    def _run(self):
146        gem5_cmd = [
147            self.gem5,
148            "-d", self.test_dir,
149            "-re",
150        ] + self.args
151
152        try:
153            with ProcessHelper(gem5_cmd, stdout=subprocess.PIPE,
154                               stderr=subprocess.PIPE) as p:
155                status, gem5_stdout, gem5_stderr = p.call(timeout=self.timeout)
156        except CallTimeoutException as te:
157            return self.error("Timeout", stdout=te.stdout, stderr=te.stderr)
158        except OSError as ose:
159            return self.error("Failed to launch gem5: %s" % ose)
160
161        stderr = "\n".join([
162            "*** gem5 stderr ***",
163            gem5_stderr,
164            "",
165            "*** m5out/simerr ***",
166            self._read_output("simerr"),
167        ])
168
169        stdout = "\n".join([
170            "*** gem5 stdout ***",
171            gem5_stdout,
172            "",
173            "*** m5out/simout ***",
174            self._read_output("simout"),
175        ])
176
177        # Signal
178        if status < 0:
179            return self.error("gem5 terminated by signal %i" % (-status, ),
180                              stdout=stdout, stderr=stderr)
181        elif status == 2:
182            return self.skip(stdout=stdout, stderr=stderr)
183        elif status > 0:
184            return self.error("gem5 exited with non-zero status: %i" % status,
185                              stdout=stdout, stderr=stderr)
186        else:
187            return self.ok(stdout=stdout, stderr=stderr)
188
189class DiffOutFile(TestUnit):
190    """Test unit comparing and output file and a reference file."""
191
192    # regular expressions of lines to ignore when diffing outputs
193    diff_ignore_regexes = {
194        "simout" : [
195            re.compile('^Redirecting (stdout|stderr) to'),
196            re.compile('^gem5 compiled '),
197            re.compile('^gem5 started '),
198            re.compile('^gem5 executing on '),
199            re.compile('^command line:'),
200            re.compile("^Couldn't import dot_parser,"),
201            re.compile("^info: kernel located at:"),
202            re.compile("^Couldn't unlink "),
203            re.compile("^Using GPU kernel code file\(s\) "),
204        ],
205        "simerr" : [
206            #re.compile('^Simulation complete at'),
207        ],
208        "config.ini" : [
209            re.compile("^(executable|readfile|kernel|image_file)="),
210            re.compile("^(cwd|input|codefile)="),
211        ],
212        "config.json" : [
213            re.compile(r'''^\s*"(executable|readfile|kernel|image_file)":'''),
214            re.compile(r'''^\s*"(cwd|input|codefile)":'''),
215        ],
216    }
217
218    def __init__(self, fname, **kwargs):
219        super(DiffOutFile, self).__init__("diff[%s]" % fname,
220                                          **kwargs)
221
222        self.fname = fname
223        self.line_filters = DiffOutFile.diff_ignore_regexes.get(fname, tuple())
224
225    def _filter_file(self, fname):
226        def match_line(l):
227            for r in self.line_filters:
228                if r.match(l):
229                    return True
230            return False
231
232        with open(fname, "r") as f:
233            for l in f:
234                if not match_line(l):
235                    yield l
236
237
238    def _run(self):
239        fname = self.fname
240        ref = self.ref_file(fname)
241        out = self.out_file(fname)
242
243        if not os.path.exists(ref):
244            return self.error("%s doesn't exist in reference directory" \
245                              % fname)
246
247        if not os.path.exists(out):
248            return self.error("%s doesn't exist in output directory" % fname)
249
250        diff = difflib.unified_diff(
251            tuple(self._filter_file(ref)),
252            tuple(self._filter_file(out)),
253            fromfile="ref/%s" % fname, tofile="out/%s" % fname)
254
255        diff = list(diff)
256        if diff:
257            return self.error("ref/%s and out/%s differ" % (fname, fname),
258                              stderr="".join(diff))
259        else:
260            return self.ok(stdout="-- ref/%s and out/%s are identical --" \
261                           % (fname, fname))
262
263class DiffStatFile(TestUnit):
264    """Test unit comparing two gem5 stat files."""
265
266    def __init__(self, **kwargs):
267        super(DiffStatFile, self).__init__("stat_diff", **kwargs)
268
269        self.stat_diff = os.path.join(_test_base, "diff-out")
270
271    def _run(self):
272        stats = "stats.txt"
273
274        cmd = [
275            self.stat_diff,
276            self.ref_file(stats), self.out_file(stats),
277        ]
278        with ProcessHelper(cmd,
279                           stdout=subprocess.PIPE,
280                           stderr=subprocess.PIPE) as p:
281            status, stdout, stderr = p.call()
282
283        if status == 0:
284            return self.ok(stdout=stdout, stderr=stderr)
285        if status == 1:
286            return self.failure("Statistics mismatch",
287                                stdout=stdout, stderr=stderr)
288        else:
289            return self.error("diff-out returned an error: %i" % status,
290                              stdout=stdout, stderr=stderr)
291