Lines Matching defs:token
154 # The original lex token stream contains WS and NEWLINE characters.
158 # "must_indent" is True if the token must be indented from the
176 for token in tokens:
177 token.at_line_start = at_line_start
179 if token.type == "COLON":
182 token.must_indent = False
184 elif token.type == "NEWLINE":
188 token.must_indent = False
190 elif token.type == "WS":
191 assert token.at_line_start == True
193 token.must_indent = False
196 # A real token; only indent after COLON NEWLINE
198 token.must_indent = True
200 token.must_indent = False
204 yield token
227 token = None
230 for token in tokens:
232 ## print "Process", token,
233 ## if token.at_line_start:
235 ## if token.must_indent:
243 if token.type == "WS":
245 depth = len(token.value)
250 if token.type == "NEWLINE":
252 if prev_was_ws or token.at_line_start:
256 yield token
259 # then it must be a real token (not WS, not NEWLINE)
263 if token.must_indent:
269 yield INDENT(token.lineno)
271 elif token.at_line_start:
285 yield DEDENT(token.lineno)
288 yield token
294 assert token is not None
296 yield DEDENT(token.lineno)
302 token = None
303 tokens = iter(lexer.token, None)
305 for token in indentation_filter(tokens):
306 yield token
310 if token is not None:
311 lineno = token.lineno
324 def token(self):