1# lexer for yacc-grammars
2#
3# Author: David Beazley (dave@dabeaz.com)
4# Date  : October 2, 2006
5
6import sys
7sys.path.append("../..")
8
9from ply import *
10
11tokens = (
12    'LITERAL','SECTION','TOKEN','LEFT','RIGHT','PREC','START','TYPE','NONASSOC','UNION','CODE',
13    'ID','QLITERAL','NUMBER',
14)
15
16states = (('code','exclusive'),)
17
18literals = [ ';', ',', '<', '>', '|',':' ]
19t_ignore = ' \t'
20
21t_TOKEN     = r'%token'
22t_LEFT      = r'%left'
23t_RIGHT     = r'%right'
24t_NONASSOC  = r'%nonassoc'
25t_PREC      = r'%prec'
26t_START     = r'%start'
27t_TYPE      = r'%type'
28t_UNION     = r'%union'
29t_ID        = r'[a-zA-Z_][a-zA-Z_0-9]*'
30t_QLITERAL  = r'''(?P<quote>['"]).*?(?P=quote)'''
31t_NUMBER    = r'\d+'
32
33def t_SECTION(t):
34    r'%%'
35    if getattr(t.lexer,"lastsection",0):
36         t.value = t.lexer.lexdata[t.lexpos+2:]
37         t.lexer.lexpos = len(t.lexer.lexdata)
38    else:
39         t.lexer.lastsection = 0
40    return t
41
42# Comments
43def t_ccomment(t):
44    r'/\*(.|\n)*?\*/'
45    t.lexer.lineno += t.value.count('\n')
46
47t_ignore_cppcomment = r'//.*'
48
49def t_LITERAL(t):
50   r'%\{(.|\n)*?%\}'
51   t.lexer.lineno += t.value.count("\n")
52   return t
53
54def t_NEWLINE(t):
55   r'\n'
56   t.lexer.lineno += 1
57
58def t_code(t):
59   r'\{'
60   t.lexer.codestart = t.lexpos
61   t.lexer.level = 1
62   t.lexer.begin('code')
63
64def t_code_ignore_string(t):
65    r'\"([^\\\n]|(\\.))*?\"'
66
67def t_code_ignore_char(t):
68    r'\'([^\\\n]|(\\.))*?\''
69
70def t_code_ignore_comment(t):
71   r'/\*(.|\n)*?\*/'
72
73def t_code_ignore_cppcom(t):
74   r'//.*'
75
76def t_code_lbrace(t):
77    r'\{'
78    t.lexer.level += 1
79
80def t_code_rbrace(t):
81    r'\}'
82    t.lexer.level -= 1
83    if t.lexer.level == 0:
84         t.type = 'CODE'
85         t.value = t.lexer.lexdata[t.lexer.codestart:t.lexpos+1]
86         t.lexer.begin('INITIAL')
87         t.lexer.lineno += t.value.count('\n')
88         return t
89
90t_code_ignore_nonspace   = r'[^\s\}\'\"\{]+'
91t_code_ignore_whitespace = r'\s+'
92t_code_ignore = ""
93
94def t_code_error(t):
95    raise RuntimeError
96
97def t_error(t):
98    print "%d: Illegal character '%s'" % (t.lexer.lineno, t.value[0])
99    print t.value
100    t.lexer.skip(1)
101
102lex.lex()
103
104if __name__ == '__main__':
105    lex.runmain()
106
107
108
109
110
111
112
113