Lines Matching refs:tokens
47 tokens = (
125 # Don't generate newline tokens when inside of parenthesis, eg
155 # WS will only occur before any other tokens on a line.
157 # I have three filters. One tags tokens by adding two attributes.
172 def track_tokens_filter(lexer, tokens):
176 for token in tokens:
224 def indentation_filter(tokens):
230 for token in tokens:
247 # WS tokens are never passed to the parser
303 tokens = iter(lexer.token, None)
304 tokens = track_tokens_filter(lexer, tokens)
305 for token in indentation_filter(tokens):
330 ########## Parser (tokens -> AST) ######