verifiers.py revision 11592:92509f1b24f7
14120Sgblack@eecs.umich.edu#!/usr/bin/env python 24120Sgblack@eecs.umich.edu# 34120Sgblack@eecs.umich.edu# Copyright (c) 2014, 2016 ARM Limited 44120Sgblack@eecs.umich.edu# All rights reserved 57087Snate@binkert.org# 67087Snate@binkert.org# The license below extends only to copyright in the software and shall 77087Snate@binkert.org# not be construed as granting a license to any other intellectual 87087Snate@binkert.org# property including but not limited to intellectual property relating 97087Snate@binkert.org# to a hardware implementation of the functionality of the software 107087Snate@binkert.org# licensed hereunder. You may use the software subject to the license 117087Snate@binkert.org# terms below provided that you ensure that this notice is replicated 127087Snate@binkert.org# unmodified and in its entirety in all distributions of the software, 134120Sgblack@eecs.umich.edu# modified or unmodified, in source code or in binary form. 147087Snate@binkert.org# 157087Snate@binkert.org# Copyright (c) 2006 The Regents of The University of Michigan 167087Snate@binkert.org# Copyright (c) 2007,2011 The Hewlett-Packard Development Company 177087Snate@binkert.org# Copyright (c) 2016 Advanced Micro Devices, Inc. 187087Snate@binkert.org# All rights reserved. 197087Snate@binkert.org# 207087Snate@binkert.org# Redistribution and use in source and binary forms, with or without 217087Snate@binkert.org# modification, are permitted provided that the following conditions are 224120Sgblack@eecs.umich.edu# met: redistributions of source code must retain the above copyright 237087Snate@binkert.org# notice, this list of conditions and the following disclaimer; 244120Sgblack@eecs.umich.edu# redistributions in binary form must reproduce the above copyright 254120Sgblack@eecs.umich.edu# notice, this list of conditions and the following disclaimer in the 264120Sgblack@eecs.umich.edu# documentation and/or other materials provided with the distribution; 274120Sgblack@eecs.umich.edu# neither the name of the copyright holders nor the names of its 284120Sgblack@eecs.umich.edu# contributors may be used to endorse or promote products derived from 294120Sgblack@eecs.umich.edu# this software without specific prior written permission. 304120Sgblack@eecs.umich.edu# 314120Sgblack@eecs.umich.edu# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS 324120Sgblack@eecs.umich.edu# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT 334120Sgblack@eecs.umich.edu# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR 344120Sgblack@eecs.umich.edu# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT 354120Sgblack@eecs.umich.edu# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, 364120Sgblack@eecs.umich.edu# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT 374120Sgblack@eecs.umich.edu# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, 384120Sgblack@eecs.umich.edu# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY 394120Sgblack@eecs.umich.edu# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT 404120Sgblack@eecs.umich.edu# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE 414120Sgblack@eecs.umich.edu# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 424120Sgblack@eecs.umich.edu# 434141Sgblack@eecs.umich.edu# Authors: Nathan Binkert 444136Sgblack@eecs.umich.edu# Steve Reinhardt 4511800Sbrandon.potter@amd.com# Andreas Sandberg 466214Snate@binkert.org 474141Sgblack@eecs.umich.edufrom abc import ABCMeta, abstractmethod 484121Sgblack@eecs.umich.edufrom difflib import SequenceMatcher 494120Sgblack@eecs.umich.eduimport inspect 504120Sgblack@eecs.umich.eduimport os 514120Sgblack@eecs.umich.eduimport re 524121Sgblack@eecs.umich.eduimport sys 534121Sgblack@eecs.umich.edu 544121Sgblack@eecs.umich.eduimport style 554121Sgblack@eecs.umich.eduimport sort_includes 564121Sgblack@eecs.umich.edufrom region import * 574121Sgblack@eecs.umich.edufrom file_types import lang_type 584121Sgblack@eecs.umich.edu 594121Sgblack@eecs.umich.edu 604121Sgblack@eecs.umich.edudef safefix(fix_func): 614121Sgblack@eecs.umich.edu """ Decorator for the fix functions of the Verifier class. 624121Sgblack@eecs.umich.edu This function wraps the fix function and creates a backup file 6310318Sandreas.hansson@arm.com just in case there is an error. 6410318Sandreas.hansson@arm.com """ 654141Sgblack@eecs.umich.edu def safefix_wrapper(*args, **kwargs): 666974Stjones1@inf.ed.ac.uk # Check to be sure that this is decorating a function we expect: 676974Stjones1@inf.ed.ac.uk # a class method with filename as the first argument (after self) 687623Sgblack@eecs.umich.edu assert(os.path.exists(args[1])) 699329Sdam.sunwoo@arm.com self = args[0] 709329Sdam.sunwoo@arm.com assert(is_verifier(self.__class__)) 719329Sdam.sunwoo@arm.com filename = args[1] 7210835Sandreas.hansson@arm.com 739057SAli.Saidi@ARM.com # Now, Let's make a backup file. 749057SAli.Saidi@ARM.com from shutil import copyfile 7510924Snilay@cs.wisc.edu backup_name = filename+'.bak' 7610924Snilay@cs.wisc.edu copyfile(filename, backup_name) 779057SAli.Saidi@ARM.com 789057SAli.Saidi@ARM.com # Try to apply the fix. If it fails, then we revert the file 799057SAli.Saidi@ARM.com # Either way, we need to clean up our backup file 809057SAli.Saidi@ARM.com try: 819057SAli.Saidi@ARM.com fix_func(*args, **kwargs) 829057SAli.Saidi@ARM.com except Exception as e: 839057SAli.Saidi@ARM.com # Restore the original file to the backup file 848902Sandreas.hansson@arm.com self.ui.write("Error! Restoring the original file.\n") 854120Sgblack@eecs.umich.edu copyfile(backup_name, filename) 864120Sgblack@eecs.umich.edu raise 87 finally: 88 # Clean up the backup file 89 os.remove(backup_name) 90 91 return safefix_wrapper 92 93def _modified_regions(old, new): 94 try: 95 m = SequenceMatcher(a=old, b=new, autojunk=False) 96 except TypeError: 97 # autojunk was introduced in Python 2.7. We need a fallback 98 # mechanism to support old Python versions. 99 m = SequenceMatcher(a=old, b=new) 100 regions = Regions() 101 for tag, i1, i2, j1, j2 in m.get_opcodes(): 102 if tag != "equal": 103 regions.extend(Region(i1, i2)) 104 return regions 105 106 107class Verifier(object): 108 """Base class for style verifiers 109 110 Verifiers check for style violations and optionally fix such 111 violations. Implementations should either inherit from this class 112 (Verifier) if they need to work on entire files or LineVerifier if 113 they operate on a line-by-line basis. 114 115 Subclasses must define these class attributes: 116 languages = set of strings identifying applicable languages 117 test_name = long descriptive name of test, will be used in 118 messages such as "error in <foo>" or "invalid <foo>" 119 opt_name = short name used to generate command-line options to 120 control the test (--fix-<foo>, --ignore-<foo>, etc.) 121 122 """ 123 124 __metaclass__ = ABCMeta 125 126 def __init__(self, ui, opts, base=None): 127 self.ui = ui 128 self.base = base 129 130 # opt_name must be defined as a class attribute of derived classes. 131 # Check test-specific opts first as these have precedence. 132 self.opt_fix = opts.get('fix_' + self.opt_name, False) 133 self.opt_ignore = opts.get('ignore_' + self.opt_name, False) 134 self.opt_skip = opts.get('skip_' + self.opt_name, False) 135 # If no test-specific opts were set, then set based on "-all" opts. 136 if not (self.opt_fix or self.opt_ignore or self.opt_skip): 137 self.opt_fix = opts.get('fix_all', False) 138 self.opt_ignore = opts.get('ignore_all', False) 139 self.opt_skip = opts.get('skip_all', False) 140 141 def normalize_filename(self, name): 142 abs_name = os.path.abspath(name) 143 if self.base is None: 144 return abs_name 145 146 abs_base = os.path.abspath(self.base) 147 return os.path.relpath(abs_name, start=abs_base) 148 149 def open(self, filename, mode): 150 try: 151 f = file(filename, mode) 152 except OSError, msg: 153 print 'could not open file %s: %s' % (filename, msg) 154 return None 155 156 return f 157 158 def skip(self, filename): 159 # We never want to handle symlinks, so always skip them: If the 160 # location pointed to is a directory, skip it. If the location is a 161 # file inside the gem5 directory, it will be checked as a file, so 162 # symlink can be skipped. If the location is a file outside gem5, we 163 # don't want to check it anyway. 164 if os.path.islink(filename): 165 return True 166 return lang_type(filename) not in self.languages 167 168 def apply(self, filename, regions=all_regions): 169 """Possibly apply to specified regions of file 'filename'. 170 171 Verifier is skipped if --skip-<test> option was provided or if 172 file is not of an applicable type. Otherwise file is checked 173 and error messages printed. Errors are fixed or ignored if 174 the corresponding --fix-<test> or --ignore-<test> options were 175 provided. If neither, the user is prompted for an action. 176 177 Returns True to abort, False otherwise. 178 """ 179 if not (self.opt_skip or self.skip(filename)): 180 errors = self.check(filename, regions) 181 if errors and not self.opt_ignore: 182 if self.opt_fix: 183 self.fix(filename, regions) 184 else: 185 result = self.ui.prompt("(a)bort, (i)gnore, or (f)ix?", 186 'aif', 'a') 187 if result == 'f': 188 self.fix(filename, regions) 189 elif result == 'a': 190 return True # abort 191 192 return False 193 194 @abstractmethod 195 def check(self, filename, regions=all_regions): 196 """Check specified regions of file 'filename'. 197 198 Line-by-line checks can simply provide a check_line() method 199 that returns True if the line is OK and False if it has an 200 error. Verifiers that need a multi-line view (like 201 SortedIncludes) must override this entire function. 202 203 Returns a count of errors (0 if none), though actual non-zero 204 count value is not currently used anywhere. 205 """ 206 pass 207 208 @abstractmethod 209 def fix(self, filename, regions=all_regions): 210 """Fix specified regions of file 'filename'. 211 212 Line-by-line fixes can simply provide a fix_line() method that 213 returns the fixed line. Verifiers that need a multi-line view 214 (like SortedIncludes) must override this entire function. 215 """ 216 pass 217 218class LineVerifier(Verifier): 219 def check(self, filename, regions=all_regions): 220 f = self.open(filename, 'r') 221 222 lang = lang_type(filename) 223 assert lang in self.languages 224 225 errors = 0 226 for num,line in enumerate(f): 227 if num not in regions: 228 continue 229 line = line.rstrip('\n') 230 if not self.check_line(line, language=lang): 231 self.ui.write("invalid %s in %s:%d\n" % \ 232 (self.test_name, filename, num + 1)) 233 if self.ui.verbose: 234 self.ui.write(">>%s<<\n" % line[:-1]) 235 errors += 1 236 f.close() 237 return errors 238 239 @safefix 240 def fix(self, filename, regions=all_regions): 241 f = self.open(filename, 'r+') 242 243 lang = lang_type(filename) 244 assert lang in self.languages 245 246 lines = list(f) 247 248 f.seek(0) 249 f.truncate() 250 251 for i,line in enumerate(lines): 252 line = line.rstrip('\n') 253 if i in regions: 254 line = self.fix_line(line, language=lang) 255 256 f.write(line) 257 f.write("\n") 258 f.close() 259 self.current_language = None 260 261 @abstractmethod 262 def check_line(self, line, **kwargs): 263 pass 264 265 @abstractmethod 266 def fix_line(self, line, **kwargs): 267 pass 268 269class Whitespace(LineVerifier): 270 """Check whitespace. 271 272 Specifically: 273 - No tabs used for indent 274 - No trailing whitespace 275 """ 276 277 languages = set(('C', 'C++', 'swig', 'python', 'asm', 'isa', 'scons', 278 'make', 'dts')) 279 trail_only = set(('make', 'dts')) 280 281 test_name = 'whitespace' 282 opt_name = 'white' 283 284 _lead = re.compile(r'^([ \t]+)') 285 _trail = re.compile(r'([ \t]+)$') 286 287 288 def skip_lead(self, language): 289 return language in Whitespace.trail_only 290 291 def check_line(self, line, language): 292 if not self.skip_lead(language): 293 match = Whitespace._lead.search(line) 294 if match and match.group(1).find('\t') != -1: 295 return False 296 297 match = Whitespace._trail.search(line) 298 if match: 299 return False 300 301 return True 302 303 def fix_line(self, line, language): 304 if not self.skip_lead(language) and Whitespace._lead.search(line): 305 newline = '' 306 for i,c in enumerate(line): 307 if c == ' ': 308 newline += ' ' 309 elif c == '\t': 310 newline += ' ' * (style.tabsize - \ 311 len(newline) % style.tabsize) 312 else: 313 newline += line[i:] 314 break 315 316 line = newline 317 318 return line.rstrip() 319 320 321class SortedIncludes(Verifier): 322 """Check for proper sorting of include statements""" 323 324 languages = sort_includes.default_languages 325 test_name = 'include file order' 326 opt_name = 'include' 327 328 def __init__(self, *args, **kwargs): 329 super(SortedIncludes, self).__init__(*args, **kwargs) 330 self.sort_includes = sort_includes.SortIncludes() 331 332 def check(self, filename, regions=all_regions): 333 f = self.open(filename, 'r') 334 norm_fname = self.normalize_filename(filename) 335 336 old = [ l.rstrip('\n') for l in f.xreadlines() ] 337 f.close() 338 339 if len(old) == 0: 340 return 0 341 342 language = lang_type(filename, old[0]) 343 new = list(self.sort_includes(old, norm_fname, language)) 344 345 modified = _modified_regions(old, new) & regions 346 347 if modified: 348 self.ui.write("invalid sorting of includes in %s\n" % (filename)) 349 if self.ui.verbose: 350 for start, end in modified.regions: 351 self.ui.write("bad region [%d, %d)\n" % (start, end)) 352 return 1 353 354 return 0 355 356 @safefix 357 def fix(self, filename, regions=all_regions): 358 f = self.open(filename, 'r+') 359 360 old = f.readlines() 361 lines = [ l.rstrip('\n') for l in old ] 362 language = lang_type(filename, lines[0]) 363 sort_lines = list(self.sort_includes(lines, filename, language)) 364 new = ''.join(line + '\n' for line in sort_lines) 365 366 f.seek(0) 367 f.truncate() 368 369 for i,line in enumerate(sort_lines): 370 f.write(line) 371 f.write('\n') 372 f.close() 373 374 375class ControlSpace(LineVerifier): 376 """Check for exactly one space after if/while/for""" 377 378 languages = set(('C', 'C++')) 379 test_name = 'spacing after if/while/for' 380 opt_name = 'control' 381 382 _any_control = re.compile(r'\b(if|while|for)([ \t]*)\(') 383 384 def check_line(self, line, **kwargs): 385 match = ControlSpace._any_control.search(line) 386 return not (match and match.group(2) != " ") 387 388 def fix_line(self, line, **kwargs): 389 new_line = ControlSpace._any_control.sub(r'\1 (', line) 390 return new_line 391 392 393class LineLength(LineVerifier): 394 languages = set(('C', 'C++', 'swig', 'python', 'asm', 'isa', 'scons')) 395 test_name = 'line length' 396 opt_name = 'length' 397 398 def check_line(self, line, **kwargs): 399 return style.normalized_len(line) <= 79 400 401 def fix(self, filename, regions=all_regions, **kwargs): 402 self.ui.write("Warning: cannot automatically fix overly long lines.\n") 403 404 def fix_line(self, line): 405 pass 406 407class ControlCharacters(LineVerifier): 408 languages = set(('C', 'C++', 'swig', 'python', 'asm', 'isa', 'scons')) 409 test_name = 'control character' 410 opt_name = 'ascii' 411 412 valid = ('\n', '\t') 413 invalid = "".join([chr(i) for i in range(0, 0x20) if chr(i) not in valid]) 414 415 def check_line(self, line, **kwargs): 416 return self.fix_line(line) == line 417 418 def fix_line(self, line, **kwargs): 419 return line.translate(None, ControlCharacters.invalid) 420 421class BoolCompare(LineVerifier): 422 languages = set(('C', 'C++', 'python')) 423 test_name = 'boolean comparison' 424 opt_name = 'boolcomp' 425 426 regex = re.compile(r'\s*==\s*([Tt]rue|[Ff]alse)\b') 427 428 def check_line(self, line, **kwargs): 429 return self.regex.search(line) == None 430 431 def fix_line(self, line, **kwargs): 432 match = self.regex.search(line) 433 if match: 434 if match.group(1) in ('true', 'True'): 435 line = self.regex.sub('', line) 436 else: 437 self.ui.write("Warning: cannot automatically fix " 438 "comparisons with false/False.\n") 439 return line 440 441def is_verifier(cls): 442 """Determine if a class is a Verifier that can be instantiated""" 443 444 return inspect.isclass(cls) and issubclass(cls, Verifier) and \ 445 not inspect.isabstract(cls) 446 447# list of all verifier classes 448all_verifiers = [ v for n, v in \ 449 inspect.getmembers(sys.modules[__name__], is_verifier) ] 450