units.py revision 12142:f6ccdb328a23 
1#!/usr/bin/env python2
2#
3# Copyright (c) 2016 ARM Limited
4# All rights reserved
5#
6# The license below extends only to copyright in the software and shall
7# not be construed as granting a license to any other intellectual
8# property including but not limited to intellectual property relating
9# to a hardware implementation of the functionality of the software
10# licensed hereunder.  You may use the software subject to the license
11# terms below provided that you ensure that this notice is replicated
12# unmodified and in its entirety in all distributions of the software,
13# modified or unmodified, in source code or in binary form.
14#
15# Redistribution and use in source and binary forms, with or without
16# modification, are permitted provided that the following conditions are
17# met: redistributions of source code must retain the above copyright
18# notice, this list of conditions and the following disclaimer;
19# redistributions in binary form must reproduce the above copyright
20# notice, this list of conditions and the following disclaimer in the
21# documentation and/or other materials provided with the distribution;
22# neither the name of the copyright holders nor the names of its
23# contributors may be used to endorse or promote products derived from
24# this software without specific prior written permission.
25#
26# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
27# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
28# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
29# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
30# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
31# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
32# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
33# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
34# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
35# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
36# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
37#
38# Authors: Andreas Sandberg
39
40from abc import ABCMeta, abstractmethod
41from datetime import datetime
42import difflib
43import functools
44import os
45import re
46import subprocess
47import sys
48import traceback
49
50from results import UnitResult
51from helpers import *
52
53_test_base = os.path.join(os.path.dirname(__file__), "..")
54
55class TestUnit(object):
56    """Base class for all test units.
57
58    A test unit is a part of a larger test case. Test cases usually
59    contain two types of units, run units (run gem5) and verify units
60    (diff output files). All unit implementations inherit from this
61    class.
62
63    A unit implementation overrides the _run() method. The test runner
64    calls the run() method, which wraps _run() to protect against
65    exceptions.
66
67    """
68
69    __metaclass__ = ABCMeta
70
71    def __init__(self, name, ref_dir, test_dir, skip=False):
72        self.name = name
73        self.ref_dir = ref_dir
74        self.test_dir = test_dir
75        self.force_skip = skip
76        self.start_time = None
77        self.stop_time = None
78
79    def result(self, state, **kwargs):
80        if self.start_time is not None and "runtime" not in kwargs:
81            self.stop_time = datetime.utcnow()
82            delta = self.stop_time - self.start_time
83            kwargs["runtime"] = delta.total_seconds()
84
85        return UnitResult(self.name, state, **kwargs)
86
87    def ok(self, **kwargs):
88        return self.result(UnitResult.STATE_OK, **kwargs)
89
90    def skip(self, **kwargs):
91        return self.result(UnitResult.STATE_SKIPPED, **kwargs)
92
93    def error(self, message, **kwargs):
94        return self.result(UnitResult.STATE_ERROR, message=message, **kwargs)
95
96    def failure(self, message, **kwargs):
97        return self.result(UnitResult.STATE_FAILURE, message=message, **kwargs)
98
99    def ref_file(self, fname):
100        return os.path.join(self.ref_dir, fname)
101
102    def out_file(self, fname):
103        return os.path.join(self.test_dir, fname)
104
105    def _read_output(self, fname, default=""):
106        try:
107            with open(self.out_file(fname), "r") as f:
108                return f.read()
109        except IOError:
110            return default
111
112    def run(self):
113        self.start_time = datetime.utcnow()
114        try:
115            if self.force_skip:
116                return self.skip()
117            else:
118                return self._run()
119        except:
120            return self.error("Python exception:\n%s" % traceback.format_exc())
121
122    @abstractmethod
123    def _run(self):
124        pass
125
126class RunGem5(TestUnit):
127    """Test unit representing a gem5 run.
128
129    Possible failure modes:
130       - gem5 failed to run -> STATE_ERROR
131       - timeout -> STATE_ERROR
132       - non-zero exit code -> STATE_ERROR
133
134    Possible non-failure results:
135       - exit code == 0 -> STATE_OK
136       - exit code == 2 -> STATE_SKIPPED
137    """
138
139    def __init__(self, gem5, gem5_args, timeout=0, **kwargs):
140        super(RunGem5, self).__init__("gem5", **kwargs)
141        self.gem5 = gem5
142        self.args = gem5_args
143        self.timeout = timeout
144
145    def _run(self):
146        gem5_cmd = [
147            self.gem5,
148            "-d", self.test_dir,
149            "--stats-file", "text://stats.txt?desc=False",
150            "-re",
151        ] + self.args
152
153        try:
154            with ProcessHelper(gem5_cmd, stdout=subprocess.PIPE,
155                               stderr=subprocess.PIPE) as p:
156                status, gem5_stdout, gem5_stderr = p.call(timeout=self.timeout)
157        except CallTimeoutException as te:
158            return self.error("Timeout", stdout=te.stdout, stderr=te.stderr)
159        except OSError as ose:
160            return self.error("Failed to launch gem5: %s" % ose)
161
162        stderr = "\n".join([
163            "*** gem5 stderr ***",
164            gem5_stderr,
165            "",
166            "*** m5out/simerr ***",
167            self._read_output("simerr"),
168        ])
169
170        stdout = "\n".join([
171            "*** gem5 stdout ***",
172            gem5_stdout,
173            "",
174            "*** m5out/simout ***",
175            self._read_output("simout"),
176        ])
177
178        # Signal
179        if status < 0:
180            return self.error("gem5 terminated by signal %i" % (-status, ),
181                              stdout=stdout, stderr=stderr)
182        elif status == 2:
183            return self.skip(stdout=stdout, stderr=stderr)
184        elif status > 0:
185            return self.error("gem5 exited with non-zero status: %i" % status,
186                              stdout=stdout, stderr=stderr)
187        else:
188            return self.ok(stdout=stdout, stderr=stderr)
189
190class DiffOutFile(TestUnit):
191    """Test unit comparing and output file and a reference file."""
192
193    # regular expressions of lines to ignore when diffing outputs
194    diff_ignore_regexes = {
195        "simout" : [
196            re.compile('^Redirecting (stdout|stderr) to'),
197            re.compile('^gem5 compiled '),
198            re.compile('^gem5 started '),
199            re.compile('^gem5 executing on '),
200            re.compile('^command line:'),
201            re.compile("^Couldn't import dot_parser,"),
202            re.compile("^info: kernel located at:"),
203            re.compile("^Couldn't unlink "),
204            re.compile("^Using GPU kernel code file\(s\) "),
205        ],
206        "simerr" : [
207            #re.compile('^Simulation complete at'),
208        ],
209        "config.ini" : [
210            re.compile("^(executable|readfile|kernel|image_file)="),
211            re.compile("^(cwd|input|codefile)="),
212        ],
213        "config.json" : [
214            re.compile(r'''^\s*"(executable|readfile|kernel|image_file)":'''),
215            re.compile(r'''^\s*"(cwd|input|codefile)":'''),
216        ],
217    }
218
219    def __init__(self, fname, **kwargs):
220        super(DiffOutFile, self).__init__("diff[%s]" % fname,
221                                          **kwargs)
222
223        self.fname = fname
224        self.line_filters = DiffOutFile.diff_ignore_regexes.get(fname, tuple())
225
226    def _filter_file(self, fname):
227        def match_line(l):
228            for r in self.line_filters:
229                if r.match(l):
230                    return True
231            return False
232
233        with open(fname, "r") as f:
234            for l in f:
235                if not match_line(l):
236                    yield l
237
238
239    def _run(self):
240        fname = self.fname
241        ref = self.ref_file(fname)
242        out = self.out_file(fname)
243
244        if not os.path.exists(ref):
245            return self.error("%s doesn't exist in reference directory" \
246                              % fname)
247
248        if not os.path.exists(out):
249            return self.error("%s doesn't exist in output directory" % fname)
250
251        diff = difflib.unified_diff(
252            tuple(self._filter_file(ref)),
253            tuple(self._filter_file(out)),
254            fromfile="ref/%s" % fname, tofile="out/%s" % fname)
255
256        diff = list(diff)
257        if diff:
258            return self.error("ref/%s and out/%s differ" % (fname, fname),
259                              stderr="".join(diff))
260        else:
261            return self.ok(stdout="-- ref/%s and out/%s are identical --" \
262                           % (fname, fname))
263
264class DiffStatFile(TestUnit):
265    """Test unit comparing two gem5 stat files."""
266
267    def __init__(self, **kwargs):
268        super(DiffStatFile, self).__init__("stat_diff", **kwargs)
269
270        self.stat_diff = os.path.join(_test_base, "diff-out")
271
272    def _run(self):
273        STATUS_OK = 0
274        STATUS_NEW_STATS = 1
275        STATUS_FAILED = 2
276
277        stats = "stats.txt"
278
279        cmd = [
280            self.stat_diff,
281            self.ref_file(stats), self.out_file(stats),
282        ]
283        with ProcessHelper(cmd,
284                           stdout=subprocess.PIPE,
285                           stderr=subprocess.PIPE) as p:
286            status, stdout, stderr = p.call()
287
288        if status in (STATUS_OK, STATUS_NEW_STATS):
289            return self.ok(stdout=stdout, stderr=stderr)
290        elif status == STATUS_FAILED:
291            return self.failure("Statistics mismatch",
292                                stdout=stdout, stderr=stderr)
293        else:
294            return self.error("diff-out returned an error: %i" % status,
295                              stdout=stdout, stderr=stderr)
296