1# ----------------------------------------------------------------------------- 2# cpp.py 3# 4# Author: David Beazley (http://www.dabeaz.com) 5# Copyright (C) 2007 6# All rights reserved 7# 8# This module implements an ANSI-C style lexical preprocessor for PLY. 9# ----------------------------------------------------------------------------- 10from __future__ import generators 11 12# ----------------------------------------------------------------------------- 13# Default preprocessor lexer definitions. These tokens are enough to get 14# a basic preprocessor working. Other modules may import these if they want 15# ----------------------------------------------------------------------------- 16 17tokens = ( 18 'CPP_ID','CPP_INTEGER', 'CPP_FLOAT', 'CPP_STRING', 'CPP_CHAR', 'CPP_WS', 'CPP_COMMENT', 'CPP_POUND','CPP_DPOUND' 19) 20 21literals = "+-*/%|&~^<>=!?()[]{}.,;:\\\'\"" 22 23# Whitespace 24def t_CPP_WS(t): 25 r'\s+' 26 t.lexer.lineno += t.value.count("\n") 27 return t 28 29t_CPP_POUND = r'\#' 30t_CPP_DPOUND = r'\#\#' 31 32# Identifier 33t_CPP_ID = r'[A-Za-z_][\w_]*' 34 35# Integer literal 36def CPP_INTEGER(t): 37 r'(((((0x)|(0X))[0-9a-fA-F]+)|(\d+))([uU]|[lL]|[uU][lL]|[lL][uU])?)' 38 return t 39 40t_CPP_INTEGER = CPP_INTEGER 41 42# Floating literal 43t_CPP_FLOAT = r'((\d+)(\.\d+)(e(\+|-)?(\d+))? | (\d+)e(\+|-)?(\d+))([lL]|[fF])?' 44 45# String literal 46def t_CPP_STRING(t): 47 r'\"([^\\\n]|(\\(.|\n)))*?\"' 48 t.lexer.lineno += t.value.count("\n") 49 return t 50 51# Character constant 'c' or L'c' 52def t_CPP_CHAR(t): 53 r'(L)?\'([^\\\n]|(\\(.|\n)))*?\'' 54 t.lexer.lineno += t.value.count("\n") 55 return t 56 57# Comment 58def t_CPP_COMMENT(t): 59 r'(/\*(.|\n)*?\*/)|(//.*?\n)' 60 t.lexer.lineno += t.value.count("\n") 61 return t 62 63def t_error(t): 64 t.type = t.value[0] 65 t.value = t.value[0] 66 t.lexer.skip(1) 67 return t 68 69import re 70import copy 71import time 72import os.path 73 74# ----------------------------------------------------------------------------- 75# trigraph() 76# 77# Given an input string, this function replaces all trigraph sequences. 78# The following mapping is used: 79# 80# ??= # 81# ??/ \ 82# ??' ^ 83# ??( [ 84# ??) ] 85# ??! | 86# ??< { 87# ??> } 88# ??- ~ 89# ----------------------------------------------------------------------------- 90 91_trigraph_pat = re.compile(r'''\?\?[=/\'\(\)\!<>\-]''') 92_trigraph_rep = { 93 '=':'#', 94 '/':'\\', 95 "'":'^', 96 '(':'[', 97 ')':']', 98 '!':'|', 99 '<':'{', 100 '>':'}', 101 '-':'~' 102} 103 104def trigraph(input): 105 return _trigraph_pat.sub(lambda g: _trigraph_rep[g.group()[-1]],input) 106 107# ------------------------------------------------------------------ 108# Macro object 109# 110# This object holds information about preprocessor macros 111# 112# .name - Macro name (string) 113# .value - Macro value (a list of tokens) 114# .arglist - List of argument names 115# .variadic - Boolean indicating whether or not variadic macro 116# .vararg - Name of the variadic parameter 117# 118# When a macro is created, the macro replacement token sequence is 119# pre-scanned and used to create patch lists that are later used 120# during macro expansion 121# ------------------------------------------------------------------ 122 123class Macro(object): 124 def __init__(self,name,value,arglist=None,variadic=False): 125 self.name = name 126 self.value = value 127 self.arglist = arglist 128 self.variadic = variadic 129 if variadic: 130 self.vararg = arglist[-1] 131 self.source = None 132 133# ------------------------------------------------------------------ 134# Preprocessor object 135# 136# Object representing a preprocessor. Contains macro definitions, 137# include directories, and other information 138# ------------------------------------------------------------------ 139 140class Preprocessor(object): 141 def __init__(self,lexer=None): 142 if lexer is None: 143 lexer = lex.lexer 144 self.lexer = lexer 145 self.macros = { } 146 self.path = [] 147 self.temp_path = [] 148 149 # Probe the lexer for selected tokens 150 self.lexprobe() 151 152 tm = time.localtime() 153 self.define("__DATE__ \"%s\"" % time.strftime("%b %d %Y",tm)) 154 self.define("__TIME__ \"%s\"" % time.strftime("%H:%M:%S",tm)) 155 self.parser = None 156 157 # ----------------------------------------------------------------------------- 158 # tokenize() 159 # 160 # Utility function. Given a string of text, tokenize into a list of tokens 161 # ----------------------------------------------------------------------------- 162 163 def tokenize(self,text): 164 tokens = [] 165 self.lexer.input(text) 166 while True: 167 tok = self.lexer.token() 168 if not tok: break 169 tokens.append(tok) 170 return tokens 171 172 # --------------------------------------------------------------------- 173 # error() 174 # 175 # Report a preprocessor error/warning of some kind 176 # ---------------------------------------------------------------------- 177 178 def error(self,file,line,msg): 179 print >>sys.stderr,"%s:%d %s" % (file,line,msg) 180 181 # ---------------------------------------------------------------------- 182 # lexprobe() 183 # 184 # This method probes the preprocessor lexer object to discover 185 # the token types of symbols that are important to the preprocessor. 186 # If this works right, the preprocessor will simply "work" 187 # with any suitable lexer regardless of how tokens have been named. 188 # ---------------------------------------------------------------------- 189 190 def lexprobe(self): 191 192 # Determine the token type for identifiers 193 self.lexer.input("identifier") 194 tok = self.lexer.token() 195 if not tok or tok.value != "identifier": 196 print "Couldn't determine identifier type" 197 else: 198 self.t_ID = tok.type 199 200 # Determine the token type for integers 201 self.lexer.input("12345") 202 tok = self.lexer.token() 203 if not tok or int(tok.value) != 12345: 204 print "Couldn't determine integer type" 205 else: 206 self.t_INTEGER = tok.type 207 self.t_INTEGER_TYPE = type(tok.value) 208 209 # Determine the token type for strings enclosed in double quotes 210 self.lexer.input("\"filename\"") 211 tok = self.lexer.token() 212 if not tok or tok.value != "\"filename\"": 213 print "Couldn't determine string type" 214 else: 215 self.t_STRING = tok.type 216 217 # Determine the token type for whitespace--if any 218 self.lexer.input(" ") 219 tok = self.lexer.token() 220 if not tok or tok.value != " ": 221 self.t_SPACE = None 222 else: 223 self.t_SPACE = tok.type 224 225 # Determine the token type for newlines 226 self.lexer.input("\n") 227 tok = self.lexer.token() 228 if not tok or tok.value != "\n": 229 self.t_NEWLINE = None 230 print "Couldn't determine token for newlines" 231 else: 232 self.t_NEWLINE = tok.type 233 234 self.t_WS = (self.t_SPACE, self.t_NEWLINE) 235 236 # Check for other characters used by the preprocessor 237 chars = [ '<','>','#','##','\\','(',')',',','.'] 238 for c in chars: 239 self.lexer.input(c) 240 tok = self.lexer.token() 241 if not tok or tok.value != c: 242 print "Unable to lex '%s' required for preprocessor" % c 243 244 # ---------------------------------------------------------------------- 245 # add_path() 246 # 247 # Adds a search path to the preprocessor. 248 # ---------------------------------------------------------------------- 249 250 def add_path(self,path): 251 self.path.append(path) 252 253 # ---------------------------------------------------------------------- 254 # group_lines() 255 # 256 # Given an input string, this function splits it into lines. Trailing whitespace 257 # is removed. Any line ending with \ is grouped with the next line. This 258 # function forms the lowest level of the preprocessor---grouping into text into 259 # a line-by-line format. 260 # ---------------------------------------------------------------------- 261 262 def group_lines(self,input): 263 lex = self.lexer.clone() 264 lines = [x.rstrip() for x in input.splitlines()] 265 for i in xrange(len(lines)): 266 j = i+1 267 while lines[i].endswith('\\') and (j < len(lines)): 268 lines[i] = lines[i][:-1]+lines[j] 269 lines[j] = "" 270 j += 1 271 272 input = "\n".join(lines) 273 lex.input(input) 274 lex.lineno = 1 275 276 current_line = [] 277 while True: 278 tok = lex.token() 279 if not tok: 280 break 281 current_line.append(tok) 282 if tok.type in self.t_WS and '\n' in tok.value: 283 yield current_line 284 current_line = [] 285 286 if current_line: 287 yield current_line 288 289 # ---------------------------------------------------------------------- 290 # tokenstrip() 291 # 292 # Remove leading/trailing whitespace tokens from a token list 293 # ---------------------------------------------------------------------- 294 295 def tokenstrip(self,tokens): 296 i = 0 297 while i < len(tokens) and tokens[i].type in self.t_WS: 298 i += 1 299 del tokens[:i] 300 i = len(tokens)-1 301 while i >= 0 and tokens[i].type in self.t_WS: 302 i -= 1 303 del tokens[i+1:] 304 return tokens 305 306 307 # ---------------------------------------------------------------------- 308 # collect_args() 309 # 310 # Collects comma separated arguments from a list of tokens. The arguments 311 # must be enclosed in parenthesis. Returns a tuple (tokencount,args,positions) 312 # where tokencount is the number of tokens consumed, args is a list of arguments, 313 # and positions is a list of integers containing the starting index of each 314 # argument. Each argument is represented by a list of tokens. 315 # 316 # When collecting arguments, leading and trailing whitespace is removed 317 # from each argument. 318 # 319 # This function properly handles nested parenthesis and commas---these do not 320 # define new arguments. 321 # ---------------------------------------------------------------------- 322 323 def collect_args(self,tokenlist): 324 args = [] 325 positions = [] 326 current_arg = [] 327 nesting = 1 328 tokenlen = len(tokenlist) 329 330 # Search for the opening '('. 331 i = 0 332 while (i < tokenlen) and (tokenlist[i].type in self.t_WS): 333 i += 1 334 335 if (i < tokenlen) and (tokenlist[i].value == '('): 336 positions.append(i+1) 337 else: 338 self.error(self.source,tokenlist[0].lineno,"Missing '(' in macro arguments") 339 return 0, [], [] 340 341 i += 1 342 343 while i < tokenlen: 344 t = tokenlist[i] 345 if t.value == '(': 346 current_arg.append(t) 347 nesting += 1 348 elif t.value == ')': 349 nesting -= 1 350 if nesting == 0: 351 if current_arg: 352 args.append(self.tokenstrip(current_arg)) 353 positions.append(i) 354 return i+1,args,positions 355 current_arg.append(t) 356 elif t.value == ',' and nesting == 1: 357 args.append(self.tokenstrip(current_arg)) 358 positions.append(i+1) 359 current_arg = [] 360 else: 361 current_arg.append(t) 362 i += 1 363 364 # Missing end argument 365 self.error(self.source,tokenlist[-1].lineno,"Missing ')' in macro arguments") 366 return 0, [],[] 367 368 # ---------------------------------------------------------------------- 369 # macro_prescan() 370 # 371 # Examine the macro value (token sequence) and identify patch points 372 # This is used to speed up macro expansion later on---we'll know 373 # right away where to apply patches to the value to form the expansion 374 # ---------------------------------------------------------------------- 375 376 def macro_prescan(self,macro): 377 macro.patch = [] # Standard macro arguments 378 macro.str_patch = [] # String conversion expansion 379 macro.var_comma_patch = [] # Variadic macro comma patch 380 i = 0 381 while i < len(macro.value): 382 if macro.value[i].type == self.t_ID and macro.value[i].value in macro.arglist: 383 argnum = macro.arglist.index(macro.value[i].value) 384 # Conversion of argument to a string 385 if i > 0 and macro.value[i-1].value == '#': 386 macro.value[i] = copy.copy(macro.value[i]) 387 macro.value[i].type = self.t_STRING 388 del macro.value[i-1] 389 macro.str_patch.append((argnum,i-1)) 390 continue 391 # Concatenation 392 elif (i > 0 and macro.value[i-1].value == '##'): 393 macro.patch.append(('c',argnum,i-1)) 394 del macro.value[i-1] 395 continue 396 elif ((i+1) < len(macro.value) and macro.value[i+1].value == '##'): 397 macro.patch.append(('c',argnum,i)) 398 i += 1 399 continue 400 # Standard expansion 401 else: 402 macro.patch.append(('e',argnum,i)) 403 elif macro.value[i].value == '##': 404 if macro.variadic and (i > 0) and (macro.value[i-1].value == ',') and \ 405 ((i+1) < len(macro.value)) and (macro.value[i+1].type == self.t_ID) and \ 406 (macro.value[i+1].value == macro.vararg): 407 macro.var_comma_patch.append(i-1) 408 i += 1 409 macro.patch.sort(key=lambda x: x[2],reverse=True) 410 411 # ---------------------------------------------------------------------- 412 # macro_expand_args() 413 # 414 # Given a Macro and list of arguments (each a token list), this method 415 # returns an expanded version of a macro. The return value is a token sequence 416 # representing the replacement macro tokens 417 # ---------------------------------------------------------------------- 418 419 def macro_expand_args(self,macro,args): 420 # Make a copy of the macro token sequence 421 rep = [copy.copy(_x) for _x in macro.value] 422 423 # Make string expansion patches. These do not alter the length of the replacement sequence 424 425 str_expansion = {} 426 for argnum, i in macro.str_patch: 427 if argnum not in str_expansion: 428 str_expansion[argnum] = ('"%s"' % "".join([x.value for x in args[argnum]])).replace("\\","\\\\") 429 rep[i] = copy.copy(rep[i]) 430 rep[i].value = str_expansion[argnum] 431 432 # Make the variadic macro comma patch. If the variadic macro argument is empty, we get rid 433 comma_patch = False 434 if macro.variadic and not args[-1]: 435 for i in macro.var_comma_patch: 436 rep[i] = None 437 comma_patch = True 438 439 # Make all other patches. The order of these matters. It is assumed that the patch list 440 # has been sorted in reverse order of patch location since replacements will cause the 441 # size of the replacement sequence to expand from the patch point. 442 443 expanded = { } 444 for ptype, argnum, i in macro.patch: 445 # Concatenation. Argument is left unexpanded 446 if ptype == 'c': 447 rep[i:i+1] = args[argnum] 448 # Normal expansion. Argument is macro expanded first 449 elif ptype == 'e': 450 if argnum not in expanded: 451 expanded[argnum] = self.expand_macros(args[argnum]) 452 rep[i:i+1] = expanded[argnum] 453 454 # Get rid of removed comma if necessary 455 if comma_patch: 456 rep = [_i for _i in rep if _i] 457 458 return rep 459 460 461 # ---------------------------------------------------------------------- 462 # expand_macros() 463 # 464 # Given a list of tokens, this function performs macro expansion. 465 # The expanded argument is a dictionary that contains macros already 466 # expanded. This is used to prevent infinite recursion. 467 # ---------------------------------------------------------------------- 468 469 def expand_macros(self,tokens,expanded=None): 470 if expanded is None: 471 expanded = {} 472 i = 0 473 while i < len(tokens): 474 t = tokens[i] 475 if t.type == self.t_ID: 476 if t.value in self.macros and t.value not in expanded: 477 # Yes, we found a macro match 478 expanded[t.value] = True 479 480 m = self.macros[t.value] 481 if not m.arglist: 482 # A simple macro 483 ex = self.expand_macros([copy.copy(_x) for _x in m.value],expanded) 484 for e in ex: 485 e.lineno = t.lineno 486 tokens[i:i+1] = ex 487 i += len(ex) 488 else: 489 # A macro with arguments 490 j = i + 1 491 while j < len(tokens) and tokens[j].type in self.t_WS: 492 j += 1 493 if tokens[j].value == '(': 494 tokcount,args,positions = self.collect_args(tokens[j:]) 495 if not m.variadic and len(args) != len(m.arglist): 496 self.error(self.source,t.lineno,"Macro %s requires %d arguments" % (t.value,len(m.arglist))) 497 i = j + tokcount 498 elif m.variadic and len(args) < len(m.arglist)-1: 499 if len(m.arglist) > 2: 500 self.error(self.source,t.lineno,"Macro %s must have at least %d arguments" % (t.value, len(m.arglist)-1)) 501 else: 502 self.error(self.source,t.lineno,"Macro %s must have at least %d argument" % (t.value, len(m.arglist)-1)) 503 i = j + tokcount 504 else: 505 if m.variadic: 506 if len(args) == len(m.arglist)-1: 507 args.append([]) 508 else: 509 args[len(m.arglist)-1] = tokens[j+positions[len(m.arglist)-1]:j+tokcount-1] 510 del args[len(m.arglist):] 511 512 # Get macro replacement text 513 rep = self.macro_expand_args(m,args) 514 rep = self.expand_macros(rep,expanded) 515 for r in rep: 516 r.lineno = t.lineno 517 tokens[i:j+tokcount] = rep 518 i += len(rep) 519 del expanded[t.value] 520 continue 521 elif t.value == '__LINE__': 522 t.type = self.t_INTEGER 523 t.value = self.t_INTEGER_TYPE(t.lineno) 524 525 i += 1 526 return tokens 527 528 # ---------------------------------------------------------------------- 529 # evalexpr() 530 # 531 # Evaluate an expression token sequence for the purposes of evaluating 532 # integral expressions. 533 # ---------------------------------------------------------------------- 534 535 def evalexpr(self,tokens): 536 # tokens = tokenize(line) 537 # Search for defined macros 538 i = 0 539 while i < len(tokens): 540 if tokens[i].type == self.t_ID and tokens[i].value == 'defined': 541 j = i + 1 542 needparen = False 543 result = "0L" 544 while j < len(tokens): 545 if tokens[j].type in self.t_WS: 546 j += 1 547 continue 548 elif tokens[j].type == self.t_ID: 549 if tokens[j].value in self.macros: 550 result = "1L" 551 else: 552 result = "0L" 553 if not needparen: break 554 elif tokens[j].value == '(': 555 needparen = True 556 elif tokens[j].value == ')': 557 break 558 else: 559 self.error(self.source,tokens[i].lineno,"Malformed defined()") 560 j += 1 561 tokens[i].type = self.t_INTEGER 562 tokens[i].value = self.t_INTEGER_TYPE(result) 563 del tokens[i+1:j+1] 564 i += 1 565 tokens = self.expand_macros(tokens) 566 for i,t in enumerate(tokens): 567 if t.type == self.t_ID: 568 tokens[i] = copy.copy(t) 569 tokens[i].type = self.t_INTEGER 570 tokens[i].value = self.t_INTEGER_TYPE("0L") 571 elif t.type == self.t_INTEGER: 572 tokens[i] = copy.copy(t) 573 # Strip off any trailing suffixes 574 tokens[i].value = str(tokens[i].value) 575 while tokens[i].value[-1] not in "0123456789abcdefABCDEF": 576 tokens[i].value = tokens[i].value[:-1] 577 578 expr = "".join([str(x.value) for x in tokens]) 579 expr = expr.replace("&&"," and ") 580 expr = expr.replace("||"," or ") 581 expr = expr.replace("!"," not ") 582 try: 583 result = eval(expr) 584 except StandardError: 585 self.error(self.source,tokens[0].lineno,"Couldn't evaluate expression") 586 result = 0 587 return result 588 589 # ---------------------------------------------------------------------- 590 # parsegen() 591 # 592 # Parse an input string/ 593 # ---------------------------------------------------------------------- 594 def parsegen(self,input,source=None): 595 596 # Replace trigraph sequences 597 t = trigraph(input) 598 lines = self.group_lines(t) 599 600 if not source: 601 source = "" 602 603 self.define("__FILE__ \"%s\"" % source) 604 605 self.source = source 606 chunk = [] 607 enable = True 608 iftrigger = False 609 ifstack = [] 610 611 for x in lines: 612 for i,tok in enumerate(x): 613 if tok.type not in self.t_WS: break 614 if tok.value == '#': 615 # Preprocessor directive 616 617 for tok in x: 618 if tok in self.t_WS and '\n' in tok.value: 619 chunk.append(tok) 620 621 dirtokens = self.tokenstrip(x[i+1:]) 622 if dirtokens: 623 name = dirtokens[0].value 624 args = self.tokenstrip(dirtokens[1:]) 625 else: 626 name = "" 627 args = [] 628 629 if name == 'define': 630 if enable: 631 for tok in self.expand_macros(chunk): 632 yield tok 633 chunk = [] 634 self.define(args) 635 elif name == 'include': 636 if enable: 637 for tok in self.expand_macros(chunk): 638 yield tok 639 chunk = [] 640 oldfile = self.macros['__FILE__'] 641 for tok in self.include(args): 642 yield tok 643 self.macros['__FILE__'] = oldfile 644 self.source = source 645 elif name == 'undef': 646 if enable: 647 for tok in self.expand_macros(chunk): 648 yield tok 649 chunk = [] 650 self.undef(args) 651 elif name == 'ifdef': 652 ifstack.append((enable,iftrigger)) 653 if enable: 654 if not args[0].value in self.macros: 655 enable = False 656 iftrigger = False 657 else: 658 iftrigger = True 659 elif name == 'ifndef': 660 ifstack.append((enable,iftrigger)) 661 if enable: 662 if args[0].value in self.macros: 663 enable = False 664 iftrigger = False 665 else: 666 iftrigger = True 667 elif name == 'if': 668 ifstack.append((enable,iftrigger)) 669 if enable: 670 result = self.evalexpr(args) 671 if not result: 672 enable = False 673 iftrigger = False 674 else: 675 iftrigger = True 676 elif name == 'elif': 677 if ifstack: 678 if ifstack[-1][0]: # We only pay attention if outer "if" allows this 679 if enable: # If already true, we flip enable False 680 enable = False 681 elif not iftrigger: # If False, but not triggered yet, we'll check expression 682 result = self.evalexpr(args) 683 if result: 684 enable = True 685 iftrigger = True 686 else: 687 self.error(self.source,dirtokens[0].lineno,"Misplaced #elif") 688 689 elif name == 'else': 690 if ifstack: 691 if ifstack[-1][0]: 692 if enable: 693 enable = False 694 elif not iftrigger: 695 enable = True 696 iftrigger = True 697 else: 698 self.error(self.source,dirtokens[0].lineno,"Misplaced #else") 699 700 elif name == 'endif': 701 if ifstack: 702 enable,iftrigger = ifstack.pop() 703 else: 704 self.error(self.source,dirtokens[0].lineno,"Misplaced #endif") 705 else: 706 # Unknown preprocessor directive 707 pass 708 709 else: 710 # Normal text 711 if enable: 712 chunk.extend(x) 713 714 for tok in self.expand_macros(chunk): 715 yield tok 716 chunk = [] 717 718 # ---------------------------------------------------------------------- 719 # include() 720 # 721 # Implementation of file-inclusion 722 # ---------------------------------------------------------------------- 723 724 def include(self,tokens): 725 # Try to extract the filename and then process an include file 726 if not tokens: 727 return 728 if tokens: 729 if tokens[0].value != '<' and tokens[0].type != self.t_STRING: 730 tokens = self.expand_macros(tokens) 731 732 if tokens[0].value == '<': 733 # Include <...> 734 i = 1 735 while i < len(tokens): 736 if tokens[i].value == '>': 737 break 738 i += 1 739 else: 740 print "Malformed #include <...>" 741 return 742 filename = "".join([x.value for x in tokens[1:i]]) 743 path = self.path + [""] + self.temp_path 744 elif tokens[0].type == self.t_STRING: 745 filename = tokens[0].value[1:-1] 746 path = self.temp_path + [""] + self.path 747 else: 748 print "Malformed #include statement" 749 return 750 for p in path: 751 iname = os.path.join(p,filename) 752 try: 753 data = open(iname,"r").read() 754 dname = os.path.dirname(iname) 755 if dname: 756 self.temp_path.insert(0,dname) 757 for tok in self.parsegen(data,filename): 758 yield tok 759 if dname: 760 del self.temp_path[0] 761 break 762 except IOError,e: 763 pass 764 else: 765 print "Couldn't find '%s'" % filename 766 767 # ---------------------------------------------------------------------- 768 # define() 769 # 770 # Define a new macro 771 # ---------------------------------------------------------------------- 772 773 def define(self,tokens): 774 if isinstance(tokens,(str,unicode)): 775 tokens = self.tokenize(tokens) 776 777 linetok = tokens 778 try: 779 name = linetok[0] 780 if len(linetok) > 1: 781 mtype = linetok[1] 782 else: 783 mtype = None 784 if not mtype: 785 m = Macro(name.value,[]) 786 self.macros[name.value] = m 787 elif mtype.type in self.t_WS: 788 # A normal macro 789 m = Macro(name.value,self.tokenstrip(linetok[2:])) 790 self.macros[name.value] = m 791 elif mtype.value == '(': 792 # A macro with arguments 793 tokcount, args, positions = self.collect_args(linetok[1:]) 794 variadic = False 795 for a in args: 796 if variadic: 797 print "No more arguments may follow a variadic argument" 798 break 799 astr = "".join([str(_i.value) for _i in a]) 800 if astr == "...": 801 variadic = True 802 a[0].type = self.t_ID 803 a[0].value = '__VA_ARGS__' 804 variadic = True 805 del a[1:] 806 continue 807 elif astr[-3:] == "..." and a[0].type == self.t_ID: 808 variadic = True 809 del a[1:] 810 # If, for some reason, "." is part of the identifier, strip off the name for the purposes 811 # of macro expansion 812 if a[0].value[-3:] == '...': 813 a[0].value = a[0].value[:-3] 814 continue 815 if len(a) > 1 or a[0].type != self.t_ID: 816 print "Invalid macro argument" 817 break 818 else: 819 mvalue = self.tokenstrip(linetok[1+tokcount:]) 820 i = 0 821 while i < len(mvalue): 822 if i+1 < len(mvalue): 823 if mvalue[i].type in self.t_WS and mvalue[i+1].value == '##': 824 del mvalue[i] 825 continue 826 elif mvalue[i].value == '##' and mvalue[i+1].type in self.t_WS: 827 del mvalue[i+1] 828 i += 1 829 m = Macro(name.value,mvalue,[x[0].value for x in args],variadic) 830 self.macro_prescan(m) 831 self.macros[name.value] = m 832 else: 833 print "Bad macro definition" 834 except LookupError: 835 print "Bad macro definition" 836 837 # ---------------------------------------------------------------------- 838 # undef() 839 # 840 # Undefine a macro 841 # ---------------------------------------------------------------------- 842 843 def undef(self,tokens): 844 id = tokens[0].value 845 try: 846 del self.macros[id] 847 except LookupError: 848 pass 849 850 # ---------------------------------------------------------------------- 851 # parse() 852 # 853 # Parse input text. 854 # ---------------------------------------------------------------------- 855 def parse(self,input,source=None,ignore={}): 856 self.ignore = ignore 857 self.parser = self.parsegen(input,source) 858 859 # ---------------------------------------------------------------------- 860 # token() 861 # 862 # Method to return individual tokens 863 # ---------------------------------------------------------------------- 864 def token(self): 865 try: 866 while True: 867 tok = self.parser.next() 868 if tok.type not in self.ignore: return tok 869 except StopIteration: 870 self.parser = None 871 return None 872 873if __name__ == '__main__': 874 import ply.lex as lex 875 lexer = lex.lex() 876 877 # Run a preprocessor 878 import sys 879 f = open(sys.argv[1]) 880 input = f.read() 881 882 p = Preprocessor(lexer) 883 p.parse(input,sys.argv[1]) 884 while True: 885 tok = p.token() 886 if not tok: break 887 print p.source, tok 888 889 890 891 892 893 894 895 896 897 898 899