verifiers.py revision 11541:3d518944f0cc 
1#!/usr/bin/env python
2#
3# Copyright (c) 2014, 2016 ARM Limited
4# All rights reserved
5#
6# The license below extends only to copyright in the software and shall
7# not be construed as granting a license to any other intellectual
8# property including but not limited to intellectual property relating
9# to a hardware implementation of the functionality of the software
10# licensed hereunder.  You may use the software subject to the license
11# terms below provided that you ensure that this notice is replicated
12# unmodified and in its entirety in all distributions of the software,
13# modified or unmodified, in source code or in binary form.
14#
15# Copyright (c) 2006 The Regents of The University of Michigan
16# Copyright (c) 2007,2011 The Hewlett-Packard Development Company
17# Copyright (c) 2016 Advanced Micro Devices, Inc.
18# All rights reserved.
19#
20# Redistribution and use in source and binary forms, with or without
21# modification, are permitted provided that the following conditions are
22# met: redistributions of source code must retain the above copyright
23# notice, this list of conditions and the following disclaimer;
24# redistributions in binary form must reproduce the above copyright
25# notice, this list of conditions and the following disclaimer in the
26# documentation and/or other materials provided with the distribution;
27# neither the name of the copyright holders nor the names of its
28# contributors may be used to endorse or promote products derived from
29# this software without specific prior written permission.
30#
31# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
32# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
33# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
34# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
35# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
36# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
37# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
38# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
39# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
40# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
41# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
42#
43# Authors: Nathan Binkert
44#          Steve Reinhardt
45#          Andreas Sandberg
46
47from abc import ABCMeta, abstractmethod
48from difflib import SequenceMatcher
49import inspect
50import os
51import re
52import sys
53
54import style
55import sort_includes
56from region import *
57from file_types import lang_type
58
59def _modified_regions(old, new):
60    try:
61        m = SequenceMatcher(a=old, b=new, autojunk=False)
62    except TypeError:
63        # autojunk was introduced in Python 2.7. We need a fallback
64        # mechanism to support old Python versions.
65        m = SequenceMatcher(a=old, b=new)
66    regions = Regions()
67    for tag, i1, i2, j1, j2 in m.get_opcodes():
68        if tag != "equal":
69            regions.extend(Region(i1, i2))
70    return regions
71
72
73class Verifier(object):
74    """Base class for style verifiers
75
76    Verifiers check for style violations and optionally fix such
77    violations. Implementations should either inherit from this class
78    (Verifier) if they need to work on entire files or LineVerifier if
79    they operate on a line-by-line basis.
80
81    Subclasses must define these class attributes:
82      languages = set of strings identifying applicable languages
83      test_name = long descriptive name of test, will be used in
84                  messages such as "error in <foo>" or "invalid <foo>"
85      opt_name = short name used to generate command-line options to
86                 control the test (--fix-<foo>, --ignore-<foo>, etc.)
87
88    """
89
90    __metaclass__ = ABCMeta
91
92    def __init__(self, ui, opts, base=None):
93        self.ui = ui
94        self.base = base
95
96        # opt_name must be defined as a class attribute of derived classes.
97        # Check test-specific opts first as these have precedence.
98        self.opt_fix = opts.get('fix_' + self.opt_name, False)
99        self.opt_ignore = opts.get('ignore_' + self.opt_name, False)
100        self.opt_skip = opts.get('skip_' + self.opt_name, False)
101        # If no test-specific opts were set, then set based on "-all" opts.
102        if not (self.opt_fix or self.opt_ignore or self.opt_skip):
103            self.opt_fix = opts.get('fix_all', False)
104            self.opt_ignore = opts.get('ignore_all', False)
105            self.opt_skip = opts.get('skip_all', False)
106
107    def normalize_filename(self, name):
108        abs_name = os.path.abspath(name)
109        if self.base is None:
110            return abs_name
111
112        abs_base = os.path.abspath(self.base)
113        return os.path.relpath(abs_name, start=abs_base)
114
115    def open(self, filename, mode):
116        try:
117            f = file(filename, mode)
118        except OSError, msg:
119            print 'could not open file %s: %s' % (filename, msg)
120            return None
121
122        return f
123
124    def skip(self, filename):
125        # We never want to handle symlinks, so always skip them: If the location
126        # pointed to is a directory, skip it. If the location is a file inside
127        # the gem5 directory, it will be checked as a file, so symlink can be
128        # skipped. If the location is a file outside gem5, we don't want to
129        # check it anyway.
130        if os.path.islink(filename):
131            return True
132        return lang_type(filename) not in self.languages
133
134    def apply(self, filename, regions=all_regions):
135        """Possibly apply to specified regions of file 'filename'.
136
137        Verifier is skipped if --skip-<test> option was provided or if
138        file is not of an applicable type.  Otherwise file is checked
139        and error messages printed.  Errors are fixed or ignored if
140        the corresponding --fix-<test> or --ignore-<test> options were
141        provided.  If neither, the user is prompted for an action.
142
143        Returns True to abort, False otherwise.
144        """
145        if not (self.opt_skip or self.skip(filename)):
146            errors = self.check(filename, regions)
147            if errors and not self.opt_ignore:
148                if self.opt_fix:
149                    self.fix(filename, regions)
150                else:
151                    result = self.ui.prompt("(a)bort, (i)gnore, or (f)ix?",
152                                            'aif', 'a')
153                    if result == 'f':
154                        self.fix(filename, regions)
155                    elif result == 'a':
156                        return True # abort
157
158        return False
159
160    @abstractmethod
161    def check(self, filename, regions=all_regions):
162        """Check specified regions of file 'filename'.
163
164        Line-by-line checks can simply provide a check_line() method
165        that returns True if the line is OK and False if it has an
166        error.  Verifiers that need a multi-line view (like
167        SortedIncludes) must override this entire function.
168
169        Returns a count of errors (0 if none), though actual non-zero
170        count value is not currently used anywhere.
171        """
172        pass
173
174    @abstractmethod
175    def fix(self, filename, regions=all_regions):
176        """Fix specified regions of file 'filename'.
177
178        Line-by-line fixes can simply provide a fix_line() method that
179        returns the fixed line. Verifiers that need a multi-line view
180        (like SortedIncludes) must override this entire function.
181        """
182        pass
183
184class LineVerifier(Verifier):
185    def check(self, filename, regions=all_regions):
186        f = self.open(filename, 'r')
187
188        lang = lang_type(filename)
189        assert lang in self.languages
190
191        errors = 0
192        for num,line in enumerate(f):
193            if num not in regions:
194                continue
195            line = line.rstrip('\n')
196            if not self.check_line(line, language=lang):
197                self.ui.write("invalid %s in %s:%d\n" % \
198                              (self.test_name, filename, num + 1))
199                if self.ui.verbose:
200                    self.ui.write(">>%s<<\n" % line[:-1])
201                errors += 1
202        f.close()
203        return errors
204
205    def fix(self, filename, regions=all_regions):
206        f = self.open(filename, 'r+')
207
208        lang = lang_type(filename)
209        assert lang in self.languages
210
211        lines = list(f)
212
213        f.seek(0)
214        f.truncate()
215
216        for i,line in enumerate(lines):
217            line = line.rstrip('\n')
218            if i in regions:
219                line = self.fix_line(line, language=lang)
220
221            f.write(line)
222            f.write("\n")
223        f.close()
224        self.current_language = None
225
226    @abstractmethod
227    def check_line(self, line, **kwargs):
228        pass
229
230    @abstractmethod
231    def fix_line(self, line, **kwargs):
232        pass
233
234class Whitespace(LineVerifier):
235    """Check whitespace.
236
237    Specifically:
238    - No tabs used for indent
239    - No trailing whitespace
240    """
241
242    languages = set(('C', 'C++', 'swig', 'python', 'asm', 'isa', 'scons',
243                     'make', 'dts'))
244    trail_only = set(('make', 'dts'))
245
246    test_name = 'whitespace'
247    opt_name = 'white'
248
249    _lead = re.compile(r'^([ \t]+)')
250    _trail = re.compile(r'([ \t]+)$')
251
252
253    def skip_lead(self, language):
254        return language in Whitespace.trail_only
255
256    def check_line(self, line, language):
257        if not self.skip_lead(language):
258            match = Whitespace._lead.search(line)
259            if match and match.group(1).find('\t') != -1:
260                return False
261
262        match = Whitespace._trail.search(line)
263        if match:
264            return False
265
266        return True
267
268    def fix_line(self, line, language):
269        if not self.skip_lead(language) and Whitespace._lead.search(line):
270            newline = ''
271            for i,c in enumerate(line):
272                if c == ' ':
273                    newline += ' '
274                elif c == '\t':
275                    newline += ' ' * (tabsize - len(newline) % tabsize)
276                else:
277                    newline += line[i:]
278                    break
279
280            line = newline
281
282        return line.rstrip()
283
284
285class SortedIncludes(Verifier):
286    """Check for proper sorting of include statements"""
287
288    languages = sort_includes.default_languages
289    test_name = 'include file order'
290    opt_name = 'include'
291
292    def __init__(self, *args, **kwargs):
293        super(SortedIncludes, self).__init__(*args, **kwargs)
294        self.sort_includes = sort_includes.SortIncludes()
295
296    def check(self, filename, regions=all_regions):
297        f = self.open(filename, 'r')
298        norm_fname = self.normalize_filename(filename)
299
300        old = [ l.rstrip('\n') for l in f.xreadlines() ]
301        f.close()
302
303        if len(old) == 0:
304            return 0
305
306        language = lang_type(filename, old[0])
307        new = list(self.sort_includes(old, norm_fname, language))
308
309        modified = _modified_regions(old, new) & regions
310
311        if modified:
312            self.ui.write("invalid sorting of includes in %s\n" % (filename))
313            if self.ui.verbose:
314                for start, end in modified.regions:
315                    self.ui.write("bad region [%d, %d)\n" % (start, end))
316            return 1
317
318        return 0
319
320    def fix(self, filename, regions=all_regions):
321        f = self.open(filename, 'r+')
322
323        old = f.readlines()
324        lines = [ l.rstrip('\n') for l in old ]
325        language = lang_type(filename, lines[0])
326        sort_lines = list(self.sort_includes(lines, filename, language))
327        new = ''.join(line + '\n' for line in sort_lines)
328
329        f.seek(0)
330        f.truncate()
331
332        for i,line in enumerate(sort_lines):
333            f.write(line)
334            f.write('\n')
335        f.close()
336
337
338class ControlSpace(LineVerifier):
339    """Check for exactly one space after if/while/for"""
340
341    languages = set(('C', 'C++'))
342    test_name = 'spacing after if/while/for'
343    opt_name = 'control'
344
345    _any_control = re.compile(r'\b(if|while|for)([ \t]*)\(')
346
347    def check_line(self, line, **kwargs):
348        match = ControlSpace._any_control.search(line)
349        return not (match and match.group(2) != " ")
350
351    def fix_line(self, line, **kwargs):
352        new_line = _any_control.sub(r'\1 (', line)
353        return new_line
354
355
356class LineLength(LineVerifier):
357    languages = set(('C', 'C++', 'swig', 'python', 'asm', 'isa', 'scons'))
358    test_name = 'line length'
359    opt_name = 'length'
360
361    def check_line(self, line, **kwargs):
362        return style.normalized_len(line) <= 79
363
364    def fix(self, filename, regions=all_regions, **kwargs):
365        self.ui.write("Warning: cannot automatically fix overly long lines.\n")
366
367    def fix_line(self, line):
368        pass
369
370class ControlCharacters(LineVerifier):
371    languages = set(('C', 'C++', 'swig', 'python', 'asm', 'isa', 'scons'))
372    test_name = 'control character'
373    opt_name = 'ascii'
374
375    valid = ('\n', '\t')
376    invalid = "".join([chr(i) for i in range(0, 0x20) if chr(i) not in valid])
377
378    def check_line(self, line, **kwargs):
379        return self.fix_line(line) == line
380
381    def fix_line(self, line, **kwargs):
382        return line.translate(None, ControlCharacters.invalid)
383
384class BoolCompare(LineVerifier):
385    languages = set(('C', 'C++', 'python'))
386    test_name = 'boolean comparison'
387    opt_name = 'boolcomp'
388
389    regex = re.compile(r'\s*==\s*([Tt]rue|[Ff]alse)\b')
390
391    def check_line(self, line, **kwargs):
392        return self.regex.search(line) == None
393
394    def fix_line(self, line, **kwargs):
395        match = self.regex.search(line)
396        if match:
397            if match.group(1) in ('true', 'True'):
398                line = self.regex.sub('', line)
399            else:
400                self.ui.write("Warning: cannot automatically fix "
401                              "comparisons with false/False.\n")
402        return line
403
404def is_verifier(cls):
405    """Determine if a class is a Verifier that can be instantiated"""
406
407    return inspect.isclass(cls) and issubclass(cls, Verifier) and \
408        not inspect.isabstract(cls)
409
410# list of all verifier classes
411all_verifiers = [ v for n, v in \
412                  inspect.getmembers(sys.modules[__name__], is_verifier) ]
413