verifiers.py revision 11549:4e5e087419df 
1#!/usr/bin/env python
2#
3# Copyright (c) 2014, 2016 ARM Limited
4# All rights reserved
5#
6# The license below extends only to copyright in the software and shall
7# not be construed as granting a license to any other intellectual
8# property including but not limited to intellectual property relating
9# to a hardware implementation of the functionality of the software
10# licensed hereunder.  You may use the software subject to the license
11# terms below provided that you ensure that this notice is replicated
12# unmodified and in its entirety in all distributions of the software,
13# modified or unmodified, in source code or in binary form.
14#
15# Copyright (c) 2006 The Regents of The University of Michigan
16# Copyright (c) 2007,2011 The Hewlett-Packard Development Company
17# Copyright (c) 2016 Advanced Micro Devices, Inc.
18# All rights reserved.
19#
20# Redistribution and use in source and binary forms, with or without
21# modification, are permitted provided that the following conditions are
22# met: redistributions of source code must retain the above copyright
23# notice, this list of conditions and the following disclaimer;
24# redistributions in binary form must reproduce the above copyright
25# notice, this list of conditions and the following disclaimer in the
26# documentation and/or other materials provided with the distribution;
27# neither the name of the copyright holders nor the names of its
28# contributors may be used to endorse or promote products derived from
29# this software without specific prior written permission.
30#
31# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
32# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
33# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
34# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
35# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
36# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
37# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
38# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
39# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
40# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
41# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
42#
43# Authors: Nathan Binkert
44#          Steve Reinhardt
45#          Andreas Sandberg
46
47from abc import ABCMeta, abstractmethod
48from difflib import SequenceMatcher
49import inspect
50import os
51import re
52import sys
53
54import style
55import sort_includes
56from region import *
57from file_types import lang_type
58
59def _modified_regions(old, new):
60    try:
61        m = SequenceMatcher(a=old, b=new, autojunk=False)
62    except TypeError:
63        # autojunk was introduced in Python 2.7. We need a fallback
64        # mechanism to support old Python versions.
65        m = SequenceMatcher(a=old, b=new)
66    regions = Regions()
67    for tag, i1, i2, j1, j2 in m.get_opcodes():
68        if tag != "equal":
69            regions.extend(Region(i1, i2))
70    return regions
71
72
73class Verifier(object):
74    """Base class for style verifiers
75
76    Verifiers check for style violations and optionally fix such
77    violations. Implementations should either inherit from this class
78    (Verifier) if they need to work on entire files or LineVerifier if
79    they operate on a line-by-line basis.
80
81    Subclasses must define these class attributes:
82      languages = set of strings identifying applicable languages
83      test_name = long descriptive name of test, will be used in
84                  messages such as "error in <foo>" or "invalid <foo>"
85      opt_name = short name used to generate command-line options to
86                 control the test (--fix-<foo>, --ignore-<foo>, etc.)
87
88    """
89
90    __metaclass__ = ABCMeta
91
92    def __init__(self, ui, opts, base=None):
93        self.ui = ui
94        self.base = base
95
96        # opt_name must be defined as a class attribute of derived classes.
97        # Check test-specific opts first as these have precedence.
98        self.opt_fix = opts.get('fix_' + self.opt_name, False)
99        self.opt_ignore = opts.get('ignore_' + self.opt_name, False)
100        self.opt_skip = opts.get('skip_' + self.opt_name, False)
101        # If no test-specific opts were set, then set based on "-all" opts.
102        if not (self.opt_fix or self.opt_ignore or self.opt_skip):
103            self.opt_fix = opts.get('fix_all', False)
104            self.opt_ignore = opts.get('ignore_all', False)
105            self.opt_skip = opts.get('skip_all', False)
106
107    def normalize_filename(self, name):
108        abs_name = os.path.abspath(name)
109        if self.base is None:
110            return abs_name
111
112        abs_base = os.path.abspath(self.base)
113        return os.path.relpath(abs_name, start=abs_base)
114
115    def open(self, filename, mode):
116        try:
117            f = file(filename, mode)
118        except OSError, msg:
119            print 'could not open file %s: %s' % (filename, msg)
120            return None
121
122        return f
123
124    def skip(self, filename):
125        # We never want to handle symlinks, so always skip them: If the location
126        # pointed to is a directory, skip it. If the location is a file inside
127        # the gem5 directory, it will be checked as a file, so symlink can be
128        # skipped. If the location is a file outside gem5, we don't want to
129        # check it anyway.
130        if os.path.islink(filename):
131            return True
132        return lang_type(filename) not in self.languages
133
134    def apply(self, filename, regions=all_regions):
135        """Possibly apply to specified regions of file 'filename'.
136
137        Verifier is skipped if --skip-<test> option was provided or if
138        file is not of an applicable type.  Otherwise file is checked
139        and error messages printed.  Errors are fixed or ignored if
140        the corresponding --fix-<test> or --ignore-<test> options were
141        provided.  If neither, the user is prompted for an action.
142
143        Returns True to abort, False otherwise.
144        """
145        if not (self.opt_skip or self.skip(filename)):
146            errors = self.check(filename, regions)
147            if errors and not self.opt_ignore:
148                if self.opt_fix:
149                    self.fix(filename, regions)
150                else:
151                    result = self.ui.prompt("(a)bort, (i)gnore, or (f)ix?",
152                                            'aif', 'a')
153                    if result == 'f':
154                        self.fix(filename, regions)
155                    elif result == 'a':
156                        return True # abort
157
158        return False
159
160    @abstractmethod
161    def check(self, filename, regions=all_regions):
162        """Check specified regions of file 'filename'.
163
164        Line-by-line checks can simply provide a check_line() method
165        that returns True if the line is OK and False if it has an
166        error.  Verifiers that need a multi-line view (like
167        SortedIncludes) must override this entire function.
168
169        Returns a count of errors (0 if none), though actual non-zero
170        count value is not currently used anywhere.
171        """
172        pass
173
174    @abstractmethod
175    def fix(self, filename, regions=all_regions):
176        """Fix specified regions of file 'filename'.
177
178        Line-by-line fixes can simply provide a fix_line() method that
179        returns the fixed line. Verifiers that need a multi-line view
180        (like SortedIncludes) must override this entire function.
181        """
182        pass
183
184class LineVerifier(Verifier):
185    def check(self, filename, regions=all_regions):
186        f = self.open(filename, 'r')
187
188        lang = lang_type(filename)
189        assert lang in self.languages
190
191        errors = 0
192        for num,line in enumerate(f):
193            if num not in regions:
194                continue
195            line = line.rstrip('\n')
196            if not self.check_line(line, language=lang):
197                self.ui.write("invalid %s in %s:%d\n" % \
198                              (self.test_name, filename, num + 1))
199                if self.ui.verbose:
200                    self.ui.write(">>%s<<\n" % line[:-1])
201                errors += 1
202        f.close()
203        return errors
204
205    def fix(self, filename, regions=all_regions):
206        f = self.open(filename, 'r+')
207
208        lang = lang_type(filename)
209        assert lang in self.languages
210
211        lines = list(f)
212
213        f.seek(0)
214        f.truncate()
215
216        for i,line in enumerate(lines):
217            line = line.rstrip('\n')
218            if i in regions:
219                line = self.fix_line(line, language=lang)
220
221            f.write(line)
222            f.write("\n")
223        f.close()
224        self.current_language = None
225
226    @abstractmethod
227    def check_line(self, line, **kwargs):
228        pass
229
230    @abstractmethod
231    def fix_line(self, line, **kwargs):
232        pass
233
234class Whitespace(LineVerifier):
235    """Check whitespace.
236
237    Specifically:
238    - No tabs used for indent
239    - No trailing whitespace
240    """
241
242    languages = set(('C', 'C++', 'swig', 'python', 'asm', 'isa', 'scons',
243                     'make', 'dts'))
244    trail_only = set(('make', 'dts'))
245
246    test_name = 'whitespace'
247    opt_name = 'white'
248
249    _lead = re.compile(r'^([ \t]+)')
250    _trail = re.compile(r'([ \t]+)$')
251
252
253    def skip_lead(self, language):
254        return language in Whitespace.trail_only
255
256    def check_line(self, line, language):
257        if not self.skip_lead(language):
258            match = Whitespace._lead.search(line)
259            if match and match.group(1).find('\t') != -1:
260                return False
261
262        match = Whitespace._trail.search(line)
263        if match:
264            return False
265
266        return True
267
268    def fix_line(self, line, language):
269        if not self.skip_lead(language) and Whitespace._lead.search(line):
270            newline = ''
271            for i,c in enumerate(line):
272                if c == ' ':
273                    newline += ' '
274                elif c == '\t':
275                    newline += ' ' * (style.tabsize - \
276                                      len(newline) % style.tabsize)
277                else:
278                    newline += line[i:]
279                    break
280
281            line = newline
282
283        return line.rstrip()
284
285
286class SortedIncludes(Verifier):
287    """Check for proper sorting of include statements"""
288
289    languages = sort_includes.default_languages
290    test_name = 'include file order'
291    opt_name = 'include'
292
293    def __init__(self, *args, **kwargs):
294        super(SortedIncludes, self).__init__(*args, **kwargs)
295        self.sort_includes = sort_includes.SortIncludes()
296
297    def check(self, filename, regions=all_regions):
298        f = self.open(filename, 'r')
299        norm_fname = self.normalize_filename(filename)
300
301        old = [ l.rstrip('\n') for l in f.xreadlines() ]
302        f.close()
303
304        if len(old) == 0:
305            return 0
306
307        language = lang_type(filename, old[0])
308        new = list(self.sort_includes(old, norm_fname, language))
309
310        modified = _modified_regions(old, new) & regions
311
312        if modified:
313            self.ui.write("invalid sorting of includes in %s\n" % (filename))
314            if self.ui.verbose:
315                for start, end in modified.regions:
316                    self.ui.write("bad region [%d, %d)\n" % (start, end))
317            return 1
318
319        return 0
320
321    def fix(self, filename, regions=all_regions):
322        f = self.open(filename, 'r+')
323
324        old = f.readlines()
325        lines = [ l.rstrip('\n') for l in old ]
326        language = lang_type(filename, lines[0])
327        sort_lines = list(self.sort_includes(lines, filename, language))
328        new = ''.join(line + '\n' for line in sort_lines)
329
330        f.seek(0)
331        f.truncate()
332
333        for i,line in enumerate(sort_lines):
334            f.write(line)
335            f.write('\n')
336        f.close()
337
338
339class ControlSpace(LineVerifier):
340    """Check for exactly one space after if/while/for"""
341
342    languages = set(('C', 'C++'))
343    test_name = 'spacing after if/while/for'
344    opt_name = 'control'
345
346    _any_control = re.compile(r'\b(if|while|for)([ \t]*)\(')
347
348    def check_line(self, line, **kwargs):
349        match = ControlSpace._any_control.search(line)
350        return not (match and match.group(2) != " ")
351
352    def fix_line(self, line, **kwargs):
353        new_line = ControlSpace._any_control.sub(r'\1 (', line)
354        return new_line
355
356
357class LineLength(LineVerifier):
358    languages = set(('C', 'C++', 'swig', 'python', 'asm', 'isa', 'scons'))
359    test_name = 'line length'
360    opt_name = 'length'
361
362    def check_line(self, line, **kwargs):
363        return style.normalized_len(line) <= 79
364
365    def fix(self, filename, regions=all_regions, **kwargs):
366        self.ui.write("Warning: cannot automatically fix overly long lines.\n")
367
368    def fix_line(self, line):
369        pass
370
371class ControlCharacters(LineVerifier):
372    languages = set(('C', 'C++', 'swig', 'python', 'asm', 'isa', 'scons'))
373    test_name = 'control character'
374    opt_name = 'ascii'
375
376    valid = ('\n', '\t')
377    invalid = "".join([chr(i) for i in range(0, 0x20) if chr(i) not in valid])
378
379    def check_line(self, line, **kwargs):
380        return self.fix_line(line) == line
381
382    def fix_line(self, line, **kwargs):
383        return line.translate(None, ControlCharacters.invalid)
384
385class BoolCompare(LineVerifier):
386    languages = set(('C', 'C++', 'python'))
387    test_name = 'boolean comparison'
388    opt_name = 'boolcomp'
389
390    regex = re.compile(r'\s*==\s*([Tt]rue|[Ff]alse)\b')
391
392    def check_line(self, line, **kwargs):
393        return self.regex.search(line) == None
394
395    def fix_line(self, line, **kwargs):
396        match = self.regex.search(line)
397        if match:
398            if match.group(1) in ('true', 'True'):
399                line = self.regex.sub('', line)
400            else:
401                self.ui.write("Warning: cannot automatically fix "
402                              "comparisons with false/False.\n")
403        return line
404
405def is_verifier(cls):
406    """Determine if a class is a Verifier that can be instantiated"""
407
408    return inspect.isclass(cls) and issubclass(cls, Verifier) and \
409        not inspect.isabstract(cls)
410
411# list of all verifier classes
412all_verifiers = [ v for n, v in \
413                  inspect.getmembers(sys.modules[__name__], is_verifier) ]
414