generator/cpp/tokenize.py

18 """Tokenize C++ source code."""
70     start contains the index of the first char of the token in the source
71     end contains the index of the last char of the token in the source
89 def _GetString(source, start, i):
90     i = source.find('"', i+1)
91     while source[i-1] == '\\':
95         while source[j] == '\\':
101         i = source.find('"', i+1)
105 def _GetChar(source, start, i):
107     i = source.find("'", i+1)
108     while source[i-1] == '\\':
110         if (i - 2) > start and source[i-2] == '\\':
112         i = source.find("'", i+1)
119 def GetTokens(source):
123       source: string of C++ source code.
126       Token that represents the next token in the source.
139     end = len(source)
142         while i < end and source[i].isspace():
149         c = source[i]
152             while source[i] in valid_identifier_chars:
156             if (source[i] == "'" and (i - start) == 1 and
157                 source[start:i] in 'uUL'):
160                 i = _GetChar(source, start, i)
161             elif source[i] == "'" and source[start:i] in _STR_PREFIXES:
163                 i = _GetString(source, start, i)
164         elif c == '/' and source[i+1] == '/':    # Find // comments.
165             i = source.find('\n', i)
169         elif c == '/' and source[i+1] == '*':    # Find /* comments. */
170             i = source.find('*/', i) + 2
175             new_ch = source[i]
185             if c == '.' and source[i].isdigit():
188                 while source[i] in int_or_float_digits:
192                     if suffix == source[i:i+1].lower():
197             if c == '0' and source[i+1] in 'xX':
200                 while source[i] in hex_digits:
203                 while source[i] in int_or_float_digits2:
208                 if suffix == source[i:i+size].lower():
213             i = _GetString(source, start, i)
216             i = _GetChar(source, start, i)
219             got_if = source[i:i+3] == '#if' and source[i+3:i+4].isspace()
222             elif source[i:i+6] == '#endif':
229                 i1 = source.find('\n', i)
230                 i2 = source.find('//', i)
231                 i3 = source.find('/*', i)
232                 i4 = source.find('"', i)
238                 if source[i] == '"':
239                     i = source.find('"', i+1) + 1
243                 if not (i == i1 and source[i-1] == '\\'):
245                         condition = source[start+4:i].lstrip()
264                              ('?', i, c, source[i-10:i+10]))
270         yield Token(token_type, source[start:i], start, i)
277             source = utils.ReadFile(filename)
278             if source is None:
281             for token in GetTokens(source):