12632Sstever@eecs.umich.edu# testlex.py 22632Sstever@eecs.umich.edu 36498Snate@binkert.orgimport unittest 46498Snate@binkert.orgtry: 56498Snate@binkert.org import StringIO 66498Snate@binkert.orgexcept ImportError: 76498Snate@binkert.org import io as StringIO 82632Sstever@eecs.umich.edu 96498Snate@binkert.orgimport sys 106498Snate@binkert.orgsys.path.insert(0,"..") 116498Snate@binkert.orgsys.tracebacklimit = 0 122632Sstever@eecs.umich.edu 136498Snate@binkert.orgimport ply.lex 142632Sstever@eecs.umich.edu 156498Snate@binkert.orgdef check_expected(result,expected): 166498Snate@binkert.org if sys.version_info[0] >= 3: 176498Snate@binkert.org if isinstance(result,str): 186498Snate@binkert.org result = result.encode('ascii') 196498Snate@binkert.org if isinstance(expected,str): 206498Snate@binkert.org expected = expected.encode('ascii') 216498Snate@binkert.org resultlines = result.splitlines() 226498Snate@binkert.org expectedlines = expected.splitlines() 232632Sstever@eecs.umich.edu 242632Sstever@eecs.umich.edu 256498Snate@binkert.org if len(resultlines) != len(expectedlines): 266498Snate@binkert.org return False 272632Sstever@eecs.umich.edu 286498Snate@binkert.org for rline,eline in zip(resultlines,expectedlines): 296498Snate@binkert.org if not rline.endswith(eline): 306498Snate@binkert.org return False 316498Snate@binkert.org return True 322632Sstever@eecs.umich.edu 336498Snate@binkert.orgdef run_import(module): 346498Snate@binkert.org code = "import "+module 356498Snate@binkert.org exec(code) 366498Snate@binkert.org del sys.modules[module] 376498Snate@binkert.org 386498Snate@binkert.org# Tests related to errors and warnings when building lexers 396498Snate@binkert.orgclass LexErrorWarningTests(unittest.TestCase): 406498Snate@binkert.org def setUp(self): 416498Snate@binkert.org sys.stderr = StringIO.StringIO() 426498Snate@binkert.org sys.stdout = StringIO.StringIO() 436498Snate@binkert.org def tearDown(self): 446498Snate@binkert.org sys.stderr = sys.__stderr__ 456498Snate@binkert.org sys.stdout = sys.__stdout__ 466498Snate@binkert.org def test_lex_doc1(self): 476498Snate@binkert.org self.assertRaises(SyntaxError,run_import,"lex_doc1") 486498Snate@binkert.org result = sys.stderr.getvalue() 496498Snate@binkert.org self.assert_(check_expected(result, 506498Snate@binkert.org "lex_doc1.py:18: No regular expression defined for rule 't_NUMBER'\n")) 516498Snate@binkert.org def test_lex_dup1(self): 526498Snate@binkert.org self.assertRaises(SyntaxError,run_import,"lex_dup1") 536498Snate@binkert.org result = sys.stderr.getvalue() 546498Snate@binkert.org self.assert_(check_expected(result, 556498Snate@binkert.org "lex_dup1.py:20: Rule t_NUMBER redefined. Previously defined on line 18\n" )) 566498Snate@binkert.org 576498Snate@binkert.org def test_lex_dup2(self): 586498Snate@binkert.org self.assertRaises(SyntaxError,run_import,"lex_dup2") 596498Snate@binkert.org result = sys.stderr.getvalue() 606498Snate@binkert.org self.assert_(check_expected(result, 616498Snate@binkert.org "lex_dup2.py:22: Rule t_NUMBER redefined. Previously defined on line 18\n" )) 626498Snate@binkert.org 636498Snate@binkert.org def test_lex_dup3(self): 646498Snate@binkert.org self.assertRaises(SyntaxError,run_import,"lex_dup3") 656498Snate@binkert.org result = sys.stderr.getvalue() 666498Snate@binkert.org self.assert_(check_expected(result, 676498Snate@binkert.org "lex_dup3.py:20: Rule t_NUMBER redefined. Previously defined on line 18\n" )) 682632Sstever@eecs.umich.edu 696498Snate@binkert.org def test_lex_empty(self): 706498Snate@binkert.org self.assertRaises(SyntaxError,run_import,"lex_empty") 716498Snate@binkert.org result = sys.stderr.getvalue() 726498Snate@binkert.org self.assert_(check_expected(result, 736498Snate@binkert.org "No rules of the form t_rulename are defined\n" 746498Snate@binkert.org "No rules defined for state 'INITIAL'\n")) 752632Sstever@eecs.umich.edu 766498Snate@binkert.org def test_lex_error1(self): 776498Snate@binkert.org run_import("lex_error1") 786498Snate@binkert.org result = sys.stderr.getvalue() 796498Snate@binkert.org self.assert_(check_expected(result, 806498Snate@binkert.org "No t_error rule is defined\n")) 812632Sstever@eecs.umich.edu 826498Snate@binkert.org def test_lex_error2(self): 836498Snate@binkert.org self.assertRaises(SyntaxError,run_import,"lex_error2") 846498Snate@binkert.org result = sys.stderr.getvalue() 856498Snate@binkert.org self.assert_(check_expected(result, 866498Snate@binkert.org "Rule 't_error' must be defined as a function\n") 876498Snate@binkert.org ) 882632Sstever@eecs.umich.edu 896498Snate@binkert.org def test_lex_error3(self): 906498Snate@binkert.org self.assertRaises(SyntaxError,run_import,"lex_error3") 916498Snate@binkert.org result = sys.stderr.getvalue() 926498Snate@binkert.org self.assert_(check_expected(result, 936498Snate@binkert.org "lex_error3.py:20: Rule 't_error' requires an argument\n")) 942632Sstever@eecs.umich.edu 956498Snate@binkert.org def test_lex_error4(self): 966498Snate@binkert.org self.assertRaises(SyntaxError,run_import,"lex_error4") 976498Snate@binkert.org result = sys.stderr.getvalue() 986498Snate@binkert.org self.assert_(check_expected(result, 996498Snate@binkert.org "lex_error4.py:20: Rule 't_error' has too many arguments\n")) 1002632Sstever@eecs.umich.edu 1016498Snate@binkert.org def test_lex_ignore(self): 1026498Snate@binkert.org self.assertRaises(SyntaxError,run_import,"lex_ignore") 1036498Snate@binkert.org result = sys.stderr.getvalue() 1046498Snate@binkert.org self.assert_(check_expected(result, 1056498Snate@binkert.org "lex_ignore.py:20: Rule 't_ignore' must be defined as a string\n")) 1062632Sstever@eecs.umich.edu 1076498Snate@binkert.org def test_lex_ignore2(self): 1086498Snate@binkert.org run_import("lex_ignore2") 1096498Snate@binkert.org result = sys.stderr.getvalue() 1106498Snate@binkert.org self.assert_(check_expected(result, 1116498Snate@binkert.org "t_ignore contains a literal backslash '\\'\n")) 1122632Sstever@eecs.umich.edu 1136498Snate@binkert.org 1146498Snate@binkert.org def test_lex_re1(self): 1156498Snate@binkert.org self.assertRaises(SyntaxError,run_import,"lex_re1") 1166498Snate@binkert.org result = sys.stderr.getvalue() 1176498Snate@binkert.org self.assert_(check_expected(result, 1186498Snate@binkert.org "Invalid regular expression for rule 't_NUMBER'. unbalanced parenthesis\n")) 1196498Snate@binkert.org 1206498Snate@binkert.org def test_lex_re2(self): 1216498Snate@binkert.org self.assertRaises(SyntaxError,run_import,"lex_re2") 1226498Snate@binkert.org result = sys.stderr.getvalue() 1236498Snate@binkert.org self.assert_(check_expected(result, 1246498Snate@binkert.org "Regular expression for rule 't_PLUS' matches empty string\n")) 1256498Snate@binkert.org 1266498Snate@binkert.org def test_lex_re3(self): 1276498Snate@binkert.org self.assertRaises(SyntaxError,run_import,"lex_re3") 1286498Snate@binkert.org result = sys.stderr.getvalue() 1296498Snate@binkert.org self.assert_(check_expected(result, 1306498Snate@binkert.org "Invalid regular expression for rule 't_POUND'. unbalanced parenthesis\n" 1316498Snate@binkert.org "Make sure '#' in rule 't_POUND' is escaped with '\\#'\n")) 1326498Snate@binkert.org 1336498Snate@binkert.org def test_lex_rule1(self): 1346498Snate@binkert.org self.assertRaises(SyntaxError,run_import,"lex_rule1") 1356498Snate@binkert.org result = sys.stderr.getvalue() 1366498Snate@binkert.org self.assert_(check_expected(result, 1376498Snate@binkert.org "t_NUMBER not defined as a function or string\n")) 1386498Snate@binkert.org 1396498Snate@binkert.org def test_lex_rule2(self): 1406498Snate@binkert.org self.assertRaises(SyntaxError,run_import,"lex_rule2") 1416498Snate@binkert.org result = sys.stderr.getvalue() 1426498Snate@binkert.org self.assert_(check_expected(result, 1436498Snate@binkert.org "lex_rule2.py:18: Rule 't_NUMBER' requires an argument\n")) 1446498Snate@binkert.org 1456498Snate@binkert.org def test_lex_rule3(self): 1466498Snate@binkert.org self.assertRaises(SyntaxError,run_import,"lex_rule3") 1476498Snate@binkert.org result = sys.stderr.getvalue() 1486498Snate@binkert.org self.assert_(check_expected(result, 1496498Snate@binkert.org "lex_rule3.py:18: Rule 't_NUMBER' has too many arguments\n")) 1506498Snate@binkert.org 1516498Snate@binkert.org 1526498Snate@binkert.org def test_lex_state1(self): 1536498Snate@binkert.org self.assertRaises(SyntaxError,run_import,"lex_state1") 1546498Snate@binkert.org result = sys.stderr.getvalue() 1556498Snate@binkert.org self.assert_(check_expected(result, 1566498Snate@binkert.org "states must be defined as a tuple or list\n")) 1576498Snate@binkert.org 1586498Snate@binkert.org def test_lex_state2(self): 1596498Snate@binkert.org self.assertRaises(SyntaxError,run_import,"lex_state2") 1606498Snate@binkert.org result = sys.stderr.getvalue() 1616498Snate@binkert.org self.assert_(check_expected(result, 1626498Snate@binkert.org "Invalid state specifier 'comment'. Must be a tuple (statename,'exclusive|inclusive')\n" 1636498Snate@binkert.org "Invalid state specifier 'example'. Must be a tuple (statename,'exclusive|inclusive')\n")) 1646498Snate@binkert.org 1656498Snate@binkert.org def test_lex_state3(self): 1666498Snate@binkert.org self.assertRaises(SyntaxError,run_import,"lex_state3") 1676498Snate@binkert.org result = sys.stderr.getvalue() 1686498Snate@binkert.org self.assert_(check_expected(result, 1696498Snate@binkert.org "State name 1 must be a string\n" 1706498Snate@binkert.org "No rules defined for state 'example'\n")) 1716498Snate@binkert.org 1726498Snate@binkert.org def test_lex_state4(self): 1736498Snate@binkert.org self.assertRaises(SyntaxError,run_import,"lex_state4") 1746498Snate@binkert.org result = sys.stderr.getvalue() 1756498Snate@binkert.org self.assert_(check_expected(result, 1766498Snate@binkert.org "State type for state comment must be 'inclusive' or 'exclusive'\n")) 1776498Snate@binkert.org 1786498Snate@binkert.org 1796498Snate@binkert.org def test_lex_state5(self): 1806498Snate@binkert.org self.assertRaises(SyntaxError,run_import,"lex_state5") 1816498Snate@binkert.org result = sys.stderr.getvalue() 1826498Snate@binkert.org self.assert_(check_expected(result, 1836498Snate@binkert.org "State 'comment' already defined\n")) 1846498Snate@binkert.org 1856498Snate@binkert.org def test_lex_state_noerror(self): 1866498Snate@binkert.org run_import("lex_state_noerror") 1876498Snate@binkert.org result = sys.stderr.getvalue() 1886498Snate@binkert.org self.assert_(check_expected(result, 1896498Snate@binkert.org "No error rule is defined for exclusive state 'comment'\n")) 1906498Snate@binkert.org 1916498Snate@binkert.org def test_lex_state_norule(self): 1926498Snate@binkert.org self.assertRaises(SyntaxError,run_import,"lex_state_norule") 1936498Snate@binkert.org result = sys.stderr.getvalue() 1946498Snate@binkert.org self.assert_(check_expected(result, 1956498Snate@binkert.org "No rules defined for state 'example'\n")) 1966498Snate@binkert.org 1976498Snate@binkert.org def test_lex_token1(self): 1986498Snate@binkert.org self.assertRaises(SyntaxError,run_import,"lex_token1") 1996498Snate@binkert.org result = sys.stderr.getvalue() 2006498Snate@binkert.org self.assert_(check_expected(result, 2016498Snate@binkert.org "No token list is defined\n" 2026498Snate@binkert.org "Rule 't_NUMBER' defined for an unspecified token NUMBER\n" 2036498Snate@binkert.org "Rule 't_PLUS' defined for an unspecified token PLUS\n" 2046498Snate@binkert.org "Rule 't_MINUS' defined for an unspecified token MINUS\n" 2056498Snate@binkert.org)) 2066498Snate@binkert.org 2076498Snate@binkert.org def test_lex_token2(self): 2086498Snate@binkert.org self.assertRaises(SyntaxError,run_import,"lex_token2") 2096498Snate@binkert.org result = sys.stderr.getvalue() 2106498Snate@binkert.org self.assert_(check_expected(result, 2116498Snate@binkert.org "tokens must be a list or tuple\n" 2126498Snate@binkert.org "Rule 't_NUMBER' defined for an unspecified token NUMBER\n" 2136498Snate@binkert.org "Rule 't_PLUS' defined for an unspecified token PLUS\n" 2146498Snate@binkert.org "Rule 't_MINUS' defined for an unspecified token MINUS\n" 2156498Snate@binkert.org)) 2166498Snate@binkert.org 2176498Snate@binkert.org def test_lex_token3(self): 2186498Snate@binkert.org self.assertRaises(SyntaxError,run_import,"lex_token3") 2196498Snate@binkert.org result = sys.stderr.getvalue() 2206498Snate@binkert.org self.assert_(check_expected(result, 2216498Snate@binkert.org "Rule 't_MINUS' defined for an unspecified token MINUS\n")) 2226498Snate@binkert.org 2236498Snate@binkert.org 2246498Snate@binkert.org def test_lex_token4(self): 2256498Snate@binkert.org self.assertRaises(SyntaxError,run_import,"lex_token4") 2266498Snate@binkert.org result = sys.stderr.getvalue() 2276498Snate@binkert.org self.assert_(check_expected(result, 2286498Snate@binkert.org "Bad token name '-'\n")) 2296498Snate@binkert.org 2306498Snate@binkert.org 2316498Snate@binkert.org def test_lex_token5(self): 2326498Snate@binkert.org try: 2336498Snate@binkert.org run_import("lex_token5") 2346498Snate@binkert.org except ply.lex.LexError: 2356498Snate@binkert.org e = sys.exc_info()[1] 2366498Snate@binkert.org self.assert_(check_expected(str(e),"lex_token5.py:19: Rule 't_NUMBER' returned an unknown token type 'NUM'")) 2376498Snate@binkert.org 2386498Snate@binkert.org def test_lex_token_dup(self): 2396498Snate@binkert.org run_import("lex_token_dup") 2406498Snate@binkert.org result = sys.stderr.getvalue() 2416498Snate@binkert.org self.assert_(check_expected(result, 2426498Snate@binkert.org "Token 'MINUS' multiply defined\n")) 2436498Snate@binkert.org 2446498Snate@binkert.org 2456498Snate@binkert.org def test_lex_literal1(self): 2466498Snate@binkert.org self.assertRaises(SyntaxError,run_import,"lex_literal1") 2476498Snate@binkert.org result = sys.stderr.getvalue() 2486498Snate@binkert.org self.assert_(check_expected(result, 2496498Snate@binkert.org "Invalid literal '**'. Must be a single character\n")) 2506498Snate@binkert.org 2516498Snate@binkert.org def test_lex_literal2(self): 2526498Snate@binkert.org self.assertRaises(SyntaxError,run_import,"lex_literal2") 2536498Snate@binkert.org result = sys.stderr.getvalue() 2546498Snate@binkert.org self.assert_(check_expected(result, 2556498Snate@binkert.org "Invalid literals specification. literals must be a sequence of characters\n")) 2566498Snate@binkert.org 2576498Snate@binkert.orgimport os 2586498Snate@binkert.orgimport subprocess 2596498Snate@binkert.orgimport shutil 2606498Snate@binkert.org 2616498Snate@binkert.org# Tests related to various build options associated with lexers 2626498Snate@binkert.orgclass LexBuildOptionTests(unittest.TestCase): 2636498Snate@binkert.org def setUp(self): 2646498Snate@binkert.org sys.stderr = StringIO.StringIO() 2656498Snate@binkert.org sys.stdout = StringIO.StringIO() 2666498Snate@binkert.org def tearDown(self): 2676498Snate@binkert.org sys.stderr = sys.__stderr__ 2686498Snate@binkert.org sys.stdout = sys.__stdout__ 2696498Snate@binkert.org try: 2706498Snate@binkert.org shutil.rmtree("lexdir") 2716498Snate@binkert.org except OSError: 2726498Snate@binkert.org pass 2736498Snate@binkert.org 2746498Snate@binkert.org def test_lex_module(self): 2756498Snate@binkert.org run_import("lex_module") 2766498Snate@binkert.org result = sys.stdout.getvalue() 2776498Snate@binkert.org self.assert_(check_expected(result, 2786498Snate@binkert.org "(NUMBER,3,1,0)\n" 2796498Snate@binkert.org "(PLUS,'+',1,1)\n" 2806498Snate@binkert.org "(NUMBER,4,1,2)\n")) 2816498Snate@binkert.org 2826498Snate@binkert.org def test_lex_object(self): 2836498Snate@binkert.org run_import("lex_object") 2846498Snate@binkert.org result = sys.stdout.getvalue() 2856498Snate@binkert.org self.assert_(check_expected(result, 2866498Snate@binkert.org "(NUMBER,3,1,0)\n" 2876498Snate@binkert.org "(PLUS,'+',1,1)\n" 2886498Snate@binkert.org "(NUMBER,4,1,2)\n")) 2896498Snate@binkert.org 2906498Snate@binkert.org def test_lex_closure(self): 2916498Snate@binkert.org run_import("lex_closure") 2926498Snate@binkert.org result = sys.stdout.getvalue() 2936498Snate@binkert.org self.assert_(check_expected(result, 2946498Snate@binkert.org "(NUMBER,3,1,0)\n" 2956498Snate@binkert.org "(PLUS,'+',1,1)\n" 2966498Snate@binkert.org "(NUMBER,4,1,2)\n")) 2976498Snate@binkert.org def test_lex_optimize(self): 2986498Snate@binkert.org try: 2996498Snate@binkert.org os.remove("lextab.py") 3006498Snate@binkert.org except OSError: 3016498Snate@binkert.org pass 3026498Snate@binkert.org try: 3036498Snate@binkert.org os.remove("lextab.pyc") 3046498Snate@binkert.org except OSError: 3056498Snate@binkert.org pass 3066498Snate@binkert.org try: 3076498Snate@binkert.org os.remove("lextab.pyo") 3086498Snate@binkert.org except OSError: 3096498Snate@binkert.org pass 3106498Snate@binkert.org run_import("lex_optimize") 3116498Snate@binkert.org 3126498Snate@binkert.org result = sys.stdout.getvalue() 3136498Snate@binkert.org self.assert_(check_expected(result, 3146498Snate@binkert.org "(NUMBER,3,1,0)\n" 3156498Snate@binkert.org "(PLUS,'+',1,1)\n" 3166498Snate@binkert.org "(NUMBER,4,1,2)\n")) 3176498Snate@binkert.org self.assert_(os.path.exists("lextab.py")) 3186498Snate@binkert.org 3196498Snate@binkert.org 3206498Snate@binkert.org p = subprocess.Popen([sys.executable,'-O','lex_optimize.py'], 3216498Snate@binkert.org stdout=subprocess.PIPE) 3226498Snate@binkert.org result = p.stdout.read() 3236498Snate@binkert.org 3246498Snate@binkert.org self.assert_(check_expected(result, 3256498Snate@binkert.org "(NUMBER,3,1,0)\n" 3266498Snate@binkert.org "(PLUS,'+',1,1)\n" 3276498Snate@binkert.org "(NUMBER,4,1,2)\n")) 3286498Snate@binkert.org self.assert_(os.path.exists("lextab.pyo")) 3296498Snate@binkert.org 3306498Snate@binkert.org os.remove("lextab.pyo") 3316498Snate@binkert.org p = subprocess.Popen([sys.executable,'-OO','lex_optimize.py'], 3326498Snate@binkert.org stdout=subprocess.PIPE) 3336498Snate@binkert.org result = p.stdout.read() 3346498Snate@binkert.org self.assert_(check_expected(result, 3356498Snate@binkert.org "(NUMBER,3,1,0)\n" 3366498Snate@binkert.org "(PLUS,'+',1,1)\n" 3376498Snate@binkert.org "(NUMBER,4,1,2)\n")) 3386498Snate@binkert.org self.assert_(os.path.exists("lextab.pyo")) 3396498Snate@binkert.org try: 3406498Snate@binkert.org os.remove("lextab.py") 3416498Snate@binkert.org except OSError: 3426498Snate@binkert.org pass 3436498Snate@binkert.org try: 3446498Snate@binkert.org os.remove("lextab.pyc") 3456498Snate@binkert.org except OSError: 3466498Snate@binkert.org pass 3476498Snate@binkert.org try: 3486498Snate@binkert.org os.remove("lextab.pyo") 3496498Snate@binkert.org except OSError: 3506498Snate@binkert.org pass 3516498Snate@binkert.org 3526498Snate@binkert.org def test_lex_optimize2(self): 3536498Snate@binkert.org try: 3546498Snate@binkert.org os.remove("opt2tab.py") 3556498Snate@binkert.org except OSError: 3566498Snate@binkert.org pass 3576498Snate@binkert.org try: 3586498Snate@binkert.org os.remove("opt2tab.pyc") 3596498Snate@binkert.org except OSError: 3606498Snate@binkert.org pass 3616498Snate@binkert.org try: 3626498Snate@binkert.org os.remove("opt2tab.pyo") 3636498Snate@binkert.org except OSError: 3646498Snate@binkert.org pass 3656498Snate@binkert.org run_import("lex_optimize2") 3666498Snate@binkert.org result = sys.stdout.getvalue() 3676498Snate@binkert.org self.assert_(check_expected(result, 3686498Snate@binkert.org "(NUMBER,3,1,0)\n" 3696498Snate@binkert.org "(PLUS,'+',1,1)\n" 3706498Snate@binkert.org "(NUMBER,4,1,2)\n")) 3716498Snate@binkert.org self.assert_(os.path.exists("opt2tab.py")) 3726498Snate@binkert.org 3736498Snate@binkert.org p = subprocess.Popen([sys.executable,'-O','lex_optimize2.py'], 3746498Snate@binkert.org stdout=subprocess.PIPE) 3756498Snate@binkert.org result = p.stdout.read() 3766498Snate@binkert.org self.assert_(check_expected(result, 3776498Snate@binkert.org "(NUMBER,3,1,0)\n" 3786498Snate@binkert.org "(PLUS,'+',1,1)\n" 3796498Snate@binkert.org "(NUMBER,4,1,2)\n")) 3806498Snate@binkert.org self.assert_(os.path.exists("opt2tab.pyo")) 3816498Snate@binkert.org os.remove("opt2tab.pyo") 3826498Snate@binkert.org p = subprocess.Popen([sys.executable,'-OO','lex_optimize2.py'], 3836498Snate@binkert.org stdout=subprocess.PIPE) 3846498Snate@binkert.org result = p.stdout.read() 3856498Snate@binkert.org self.assert_(check_expected(result, 3866498Snate@binkert.org "(NUMBER,3,1,0)\n" 3876498Snate@binkert.org "(PLUS,'+',1,1)\n" 3886498Snate@binkert.org "(NUMBER,4,1,2)\n")) 3896498Snate@binkert.org self.assert_(os.path.exists("opt2tab.pyo")) 3906498Snate@binkert.org try: 3916498Snate@binkert.org os.remove("opt2tab.py") 3926498Snate@binkert.org except OSError: 3936498Snate@binkert.org pass 3946498Snate@binkert.org try: 3956498Snate@binkert.org os.remove("opt2tab.pyc") 3966498Snate@binkert.org except OSError: 3976498Snate@binkert.org pass 3986498Snate@binkert.org try: 3996498Snate@binkert.org os.remove("opt2tab.pyo") 4006498Snate@binkert.org except OSError: 4016498Snate@binkert.org pass 4026498Snate@binkert.org 4036498Snate@binkert.org def test_lex_optimize3(self): 4046498Snate@binkert.org try: 4056498Snate@binkert.org shutil.rmtree("lexdir") 4066498Snate@binkert.org except OSError: 4076498Snate@binkert.org pass 4086498Snate@binkert.org 4096498Snate@binkert.org os.mkdir("lexdir") 4106498Snate@binkert.org os.mkdir("lexdir/sub") 4116498Snate@binkert.org open("lexdir/__init__.py","w").write("") 4126498Snate@binkert.org open("lexdir/sub/__init__.py","w").write("") 4136498Snate@binkert.org run_import("lex_optimize3") 4146498Snate@binkert.org result = sys.stdout.getvalue() 4156498Snate@binkert.org self.assert_(check_expected(result, 4166498Snate@binkert.org "(NUMBER,3,1,0)\n" 4176498Snate@binkert.org "(PLUS,'+',1,1)\n" 4186498Snate@binkert.org "(NUMBER,4,1,2)\n")) 4196498Snate@binkert.org self.assert_(os.path.exists("lexdir/sub/calctab.py")) 4206498Snate@binkert.org 4216498Snate@binkert.org p = subprocess.Popen([sys.executable,'-O','lex_optimize3.py'], 4226498Snate@binkert.org stdout=subprocess.PIPE) 4236498Snate@binkert.org result = p.stdout.read() 4246498Snate@binkert.org self.assert_(check_expected(result, 4256498Snate@binkert.org "(NUMBER,3,1,0)\n" 4266498Snate@binkert.org "(PLUS,'+',1,1)\n" 4276498Snate@binkert.org "(NUMBER,4,1,2)\n")) 4286498Snate@binkert.org self.assert_(os.path.exists("lexdir/sub/calctab.pyo")) 4296498Snate@binkert.org os.remove("lexdir/sub/calctab.pyo") 4306498Snate@binkert.org p = subprocess.Popen([sys.executable,'-OO','lex_optimize3.py'], 4316498Snate@binkert.org stdout=subprocess.PIPE) 4326498Snate@binkert.org result = p.stdout.read() 4336498Snate@binkert.org self.assert_(check_expected(result, 4346498Snate@binkert.org "(NUMBER,3,1,0)\n" 4356498Snate@binkert.org "(PLUS,'+',1,1)\n" 4366498Snate@binkert.org "(NUMBER,4,1,2)\n")) 4376498Snate@binkert.org self.assert_(os.path.exists("lexdir/sub/calctab.pyo")) 4386498Snate@binkert.org try: 4396498Snate@binkert.org shutil.rmtree("lexdir") 4406498Snate@binkert.org except OSError: 4416498Snate@binkert.org pass 4426498Snate@binkert.org 4436498Snate@binkert.org def test_lex_opt_alias(self): 4446498Snate@binkert.org try: 4456498Snate@binkert.org os.remove("aliastab.py") 4466498Snate@binkert.org except OSError: 4476498Snate@binkert.org pass 4486498Snate@binkert.org try: 4496498Snate@binkert.org os.remove("aliastab.pyc") 4506498Snate@binkert.org except OSError: 4516498Snate@binkert.org pass 4526498Snate@binkert.org try: 4536498Snate@binkert.org os.remove("aliastab.pyo") 4546498Snate@binkert.org except OSError: 4556498Snate@binkert.org pass 4566498Snate@binkert.org run_import("lex_opt_alias") 4576498Snate@binkert.org result = sys.stdout.getvalue() 4586498Snate@binkert.org self.assert_(check_expected(result, 4596498Snate@binkert.org "(NUMBER,3,1,0)\n" 4606498Snate@binkert.org "(+,'+',1,1)\n" 4616498Snate@binkert.org "(NUMBER,4,1,2)\n")) 4626498Snate@binkert.org self.assert_(os.path.exists("aliastab.py")) 4636498Snate@binkert.org 4646498Snate@binkert.org p = subprocess.Popen([sys.executable,'-O','lex_opt_alias.py'], 4656498Snate@binkert.org stdout=subprocess.PIPE) 4666498Snate@binkert.org result = p.stdout.read() 4676498Snate@binkert.org self.assert_(check_expected(result, 4686498Snate@binkert.org "(NUMBER,3,1,0)\n" 4696498Snate@binkert.org "(+,'+',1,1)\n" 4706498Snate@binkert.org "(NUMBER,4,1,2)\n")) 4716498Snate@binkert.org self.assert_(os.path.exists("aliastab.pyo")) 4726498Snate@binkert.org os.remove("aliastab.pyo") 4736498Snate@binkert.org p = subprocess.Popen([sys.executable,'-OO','lex_opt_alias.py'], 4746498Snate@binkert.org stdout=subprocess.PIPE) 4756498Snate@binkert.org result = p.stdout.read() 4766498Snate@binkert.org self.assert_(check_expected(result, 4776498Snate@binkert.org "(NUMBER,3,1,0)\n" 4786498Snate@binkert.org "(+,'+',1,1)\n" 4796498Snate@binkert.org "(NUMBER,4,1,2)\n")) 4806498Snate@binkert.org self.assert_(os.path.exists("aliastab.pyo")) 4816498Snate@binkert.org try: 4826498Snate@binkert.org os.remove("aliastab.py") 4836498Snate@binkert.org except OSError: 4846498Snate@binkert.org pass 4856498Snate@binkert.org try: 4866498Snate@binkert.org os.remove("aliastab.pyc") 4876498Snate@binkert.org except OSError: 4886498Snate@binkert.org pass 4896498Snate@binkert.org try: 4906498Snate@binkert.org os.remove("aliastab.pyo") 4916498Snate@binkert.org except OSError: 4926498Snate@binkert.org pass 4936498Snate@binkert.org 4946498Snate@binkert.org def test_lex_many_tokens(self): 4956498Snate@binkert.org try: 4966498Snate@binkert.org os.remove("manytab.py") 4976498Snate@binkert.org except OSError: 4986498Snate@binkert.org pass 4996498Snate@binkert.org try: 5006498Snate@binkert.org os.remove("manytab.pyc") 5016498Snate@binkert.org except OSError: 5026498Snate@binkert.org pass 5036498Snate@binkert.org try: 5046498Snate@binkert.org os.remove("manytab.pyo") 5056498Snate@binkert.org except OSError: 5066498Snate@binkert.org pass 5076498Snate@binkert.org run_import("lex_many_tokens") 5086498Snate@binkert.org result = sys.stdout.getvalue() 5096498Snate@binkert.org self.assert_(check_expected(result, 5106498Snate@binkert.org "(TOK34,'TOK34:',1,0)\n" 5116498Snate@binkert.org "(TOK143,'TOK143:',1,7)\n" 5126498Snate@binkert.org "(TOK269,'TOK269:',1,15)\n" 5136498Snate@binkert.org "(TOK372,'TOK372:',1,23)\n" 5146498Snate@binkert.org "(TOK452,'TOK452:',1,31)\n" 5156498Snate@binkert.org "(TOK561,'TOK561:',1,39)\n" 5166498Snate@binkert.org "(TOK999,'TOK999:',1,47)\n" 5176498Snate@binkert.org )) 5186498Snate@binkert.org 5196498Snate@binkert.org self.assert_(os.path.exists("manytab.py")) 5206498Snate@binkert.org 5216498Snate@binkert.org p = subprocess.Popen([sys.executable,'-O','lex_many_tokens.py'], 5226498Snate@binkert.org stdout=subprocess.PIPE) 5236498Snate@binkert.org result = p.stdout.read() 5246498Snate@binkert.org self.assert_(check_expected(result, 5256498Snate@binkert.org "(TOK34,'TOK34:',1,0)\n" 5266498Snate@binkert.org "(TOK143,'TOK143:',1,7)\n" 5276498Snate@binkert.org "(TOK269,'TOK269:',1,15)\n" 5286498Snate@binkert.org "(TOK372,'TOK372:',1,23)\n" 5296498Snate@binkert.org "(TOK452,'TOK452:',1,31)\n" 5306498Snate@binkert.org "(TOK561,'TOK561:',1,39)\n" 5316498Snate@binkert.org "(TOK999,'TOK999:',1,47)\n" 5326498Snate@binkert.org )) 5336498Snate@binkert.org 5346498Snate@binkert.org self.assert_(os.path.exists("manytab.pyo")) 5356498Snate@binkert.org os.remove("manytab.pyo") 5366498Snate@binkert.org try: 5376498Snate@binkert.org os.remove("manytab.py") 5386498Snate@binkert.org except OSError: 5396498Snate@binkert.org pass 5406498Snate@binkert.org try: 5416498Snate@binkert.org os.remove("manytab.pyc") 5426498Snate@binkert.org except OSError: 5436498Snate@binkert.org pass 5446498Snate@binkert.org try: 5456498Snate@binkert.org os.remove("manytab.pyo") 5466498Snate@binkert.org except OSError: 5476498Snate@binkert.org pass 5486498Snate@binkert.org 5496498Snate@binkert.org# Tests related to run-time behavior of lexers 5506498Snate@binkert.orgclass LexRunTests(unittest.TestCase): 5516498Snate@binkert.org def setUp(self): 5526498Snate@binkert.org sys.stderr = StringIO.StringIO() 5536498Snate@binkert.org sys.stdout = StringIO.StringIO() 5546498Snate@binkert.org def tearDown(self): 5556498Snate@binkert.org sys.stderr = sys.__stderr__ 5566498Snate@binkert.org sys.stdout = sys.__stdout__ 5576498Snate@binkert.org 5586498Snate@binkert.org def test_lex_hedit(self): 5596498Snate@binkert.org run_import("lex_hedit") 5606498Snate@binkert.org result = sys.stdout.getvalue() 5616498Snate@binkert.org self.assert_(check_expected(result, 5626498Snate@binkert.org "(H_EDIT_DESCRIPTOR,'abc',1,0)\n" 5636498Snate@binkert.org "(H_EDIT_DESCRIPTOR,'abcdefghij',1,6)\n" 5646498Snate@binkert.org "(H_EDIT_DESCRIPTOR,'xy',1,20)\n")) 5656498Snate@binkert.org 5666498Snate@binkert.org def test_lex_state_try(self): 5676498Snate@binkert.org run_import("lex_state_try") 5686498Snate@binkert.org result = sys.stdout.getvalue() 5696498Snate@binkert.org self.assert_(check_expected(result, 5706498Snate@binkert.org "(NUMBER,'3',1,0)\n" 5716498Snate@binkert.org "(PLUS,'+',1,2)\n" 5726498Snate@binkert.org "(NUMBER,'4',1,4)\n" 5736498Snate@binkert.org "Entering comment state\n" 5746498Snate@binkert.org "comment body LexToken(body_part,'This is a comment */',1,9)\n" 5756498Snate@binkert.org "(PLUS,'+',1,30)\n" 5766498Snate@binkert.org "(NUMBER,'10',1,32)\n" 5776498Snate@binkert.org )) 5786498Snate@binkert.org 5796498Snate@binkert.org 5806498Snate@binkert.org 5816498Snate@binkert.orgunittest.main() 582