diff --git a/pytest/test_grammar.py b/pytest/test_grammar.py index d6f2fc21..ebcdaab2 100644 --- a/pytest/test_grammar.py +++ b/pytest/test_grammar.py @@ -1,9 +1,18 @@ -import pytest +import pytest, re from uncompyle6 import PYTHON_VERSION, PYTHON3, IS_PYPY # , PYTHON_VERSION from uncompyle6.parser import get_python_parser from uncompyle6.scanner import get_scanner def test_grammar(): + + def check_tokens(tokens, opcode_set): + remain_tokens = set(tokens) - opcode_set + remain_tokens = set([re.sub('_\d+$','', t) for t in remain_tokens]) + remain_tokens = set([re.sub('_CONT$','', t) for t in remain_tokens]) + remain_tokens = set(remain_tokens) - opcode_set + assert remain_tokens == set([]), \ + "Remaining tokens %s\n====\n%s" % (remain_tokens, p.dumpGrammar()) + p = get_python_parser(PYTHON_VERSION, is_pypy=IS_PYPY) lhs, rhs, tokens, right_recursive = p.checkSets() expect_lhs = set(['expr1024', 'pos_arg']) @@ -21,16 +30,15 @@ def test_grammar(): assert unused_rhs == set(rhs) assert expect_right_recursive == right_recursive s = get_scanner(PYTHON_VERSION, IS_PYPY) - if PYTHON_VERSION == 2.7: - opcode_set = set(s.opc.opname).union(set( + ignore_set = set( """JUMP_BACK CONTINUE RETURN_END_IF COME_FROM LOAD_GENEXPR LOAD_ASSERT LOAD_SETCOMP LOAD_DICTCOMP LAMBDA_MARKER RETURN_LAST - """.split())) - remain_tokens = set(tokens) - opcode_set - import re - remain_tokens = set([re.sub('_\d+$','', t) for t in remain_tokens]) - remain_tokens = set([re.sub('_CONT$','', t) for t in remain_tokens]) - remain_tokens = set(remain_tokens) - opcode_set - assert remain_tokens == set([]), \ - "Remaining tokens %s\n====\n%s" % (remain_tokens, p.dumpGrammar()) + """.split()) + if 2.6 <= PYTHON_VERSION <= 2.7: + opcode_set = set(s.opc.opname).union(ignore_set) + check_tokens(tokens, opcode_set) + elif PYTHON_VERSION == 3.4: + ignore_set.add('LOAD_CLASSNAME') + opcode_set = set(s.opc.opname).union(ignore_set) + check_tokens(tokens, opcode_set) diff --git a/uncompyle6/parser.py b/uncompyle6/parser.py index defae5fc..d17d8190 100644 --- a/uncompyle6/parser.py +++ b/uncompyle6/parser.py @@ -236,17 +236,12 @@ class PythonParser(GenericASTBuilder): stmt ::= augassign2 augassign1 ::= expr expr inplace_op designator augassign1 ::= expr expr inplace_op ROT_THREE STORE_SUBSCR - augassign1 ::= expr expr inplace_op ROT_TWO STORE_SLICE+0 - augassign1 ::= expr expr inplace_op ROT_THREE STORE_SLICE+1 - augassign1 ::= expr expr inplace_op ROT_THREE STORE_SLICE+2 - augassign1 ::= expr expr inplace_op ROT_FOUR STORE_SLICE+3 augassign2 ::= expr DUP_TOP LOAD_ATTR expr inplace_op ROT_TWO STORE_ATTR inplace_op ::= INPLACE_ADD inplace_op ::= INPLACE_SUBTRACT inplace_op ::= INPLACE_MULTIPLY - inplace_op ::= INPLACE_DIVIDE inplace_op ::= INPLACE_TRUE_DIVIDE inplace_op ::= INPLACE_FLOOR_DIVIDE inplace_op ::= INPLACE_MODULO @@ -405,15 +400,10 @@ class PythonParser(GenericASTBuilder): expr ::= unary_expr expr ::= call_function expr ::= unary_not - expr ::= unary_convert expr ::= binary_subscr expr ::= binary_subscr2 expr ::= load_attr expr ::= get_iter - expr ::= slice0 - expr ::= slice1 - expr ::= slice2 - expr ::= slice3 expr ::= buildslice2 expr ::= buildslice3 expr ::= yield @@ -428,7 +418,6 @@ class PythonParser(GenericASTBuilder): binary_op ::= BINARY_OR binary_op ::= BINARY_XOR binary_op ::= BINARY_SUBTRACT - binary_op ::= BINARY_DIVIDE binary_op ::= BINARY_TRUE_DIVIDE binary_op ::= BINARY_FLOOR_DIVIDE binary_op ::= BINARY_MODULO @@ -442,21 +431,11 @@ class PythonParser(GenericASTBuilder): unary_op ::= UNARY_INVERT unary_not ::= expr UNARY_NOT - unary_convert ::= expr UNARY_CONVERT binary_subscr ::= expr expr BINARY_SUBSCR - binary_subscr2 ::= expr expr DUP_TOPX_2 BINARY_SUBSCR load_attr ::= expr LOAD_ATTR get_iter ::= expr GET_ITER - slice0 ::= expr SLICE+0 - slice0 ::= expr DUP_TOP SLICE+0 - slice1 ::= expr expr SLICE+1 - slice1 ::= expr expr DUP_TOPX_2 SLICE+1 - slice2 ::= expr expr SLICE+2 - slice2 ::= expr expr DUP_TOPX_2 SLICE+2 - slice3 ::= expr expr expr SLICE+3 - slice3 ::= expr expr expr DUP_TOPX_3 SLICE+3 buildslice3 ::= expr expr expr BUILD_SLICE_3 buildslice2 ::= expr expr BUILD_SLICE_2 @@ -465,12 +444,6 @@ class PythonParser(GenericASTBuilder): _mklambda ::= load_closure mklambda _mklambda ::= mklambda - # Note: Python < 2.7 doesn't have *POP* or this. Remove from here? - # FIXME: segregate 2.7+ - - or ::= expr JUMP_IF_TRUE_OR_POP expr COME_FROM - and ::= expr JUMP_IF_FALSE_OR_POP expr COME_FROM - or ::= expr jmp_true expr come_from_opt and ::= expr jmp_false expr come_from_opt and2 ::= _jump jmp_false COME_FROM expr COME_FROM @@ -489,14 +462,6 @@ class PythonParser(GenericASTBuilder): ret_expr_or_cond ::= ret_cond ret_expr_or_cond ::= ret_cond_not - # Note: Python < 2.7 doesn't have *POP* or this. Remove from here? - # FIXME: segregate 2.7+ - - ret_and ::= expr JUMP_IF_FALSE_OR_POP ret_expr_or_cond COME_FROM - ret_or ::= expr JUMP_IF_TRUE_OR_POP ret_expr_or_cond COME_FROM - ret_cond ::= expr POP_JUMP_IF_FALSE expr RETURN_END_IF ret_expr_or_cond - ret_cond_not ::= expr POP_JUMP_IF_TRUE expr RETURN_END_IF ret_expr_or_cond - stmt ::= return_lambda stmt ::= conditional_lambda @@ -508,15 +473,9 @@ class PythonParser(GenericASTBuilder): compare ::= expr expr COMPARE_OP cmp_list ::= expr cmp_list1 ROT_TWO POP_TOP _come_from - cmp_list1 ::= expr DUP_TOP ROT_THREE - COMPARE_OP JUMP_IF_FALSE_OR_POP - cmp_list1 COME_FROM cmp_list1 ::= expr DUP_TOP ROT_THREE COMPARE_OP jmp_false cmp_list1 _come_from - cmp_list1 ::= expr DUP_TOP ROT_THREE - COMPARE_OP JUMP_IF_FALSE_OR_POP - cmp_list2 COME_FROM cmp_list1 ::= expr DUP_TOP ROT_THREE COMPARE_OP jmp_false cmp_list2 _come_from @@ -561,10 +520,6 @@ class PythonParser(GenericASTBuilder): designator ::= STORE_GLOBAL designator ::= STORE_DEREF designator ::= expr STORE_ATTR - designator ::= expr STORE_SLICE+0 - designator ::= expr expr STORE_SLICE+1 - designator ::= expr expr STORE_SLICE+2 - designator ::= expr expr expr STORE_SLICE+3 designator ::= store_subscr store_subscr ::= expr expr STORE_SUBSCR designator ::= unpack diff --git a/uncompyle6/parsers/parse2.py b/uncompyle6/parsers/parse2.py index 33708d38..cec08167 100644 --- a/uncompyle6/parsers/parse2.py +++ b/uncompyle6/parsers/parse2.py @@ -43,6 +43,9 @@ class Python2Parser(PythonParser): def p_stmt2(self, args): """ while1stmt ::= SETUP_LOOP l_stmts_opt JUMP_BACK POP_BLOCK COME_FROM + exec_stmt ::= expr exprlist DUP_TOP EXEC_STMT + exec_stmt ::= expr exprlist EXEC_STMT + """ def p_print_to(self, args): @@ -89,8 +92,6 @@ class Python2Parser(PythonParser): raise_stmt3 ::= expr expr expr RAISE_VARARGS_3 stmt ::= exec_stmt - exec_stmt ::= expr exprlist DUP_TOP EXEC_STMT - exec_stmt ::= expr exprlist EXEC_STMT stmt ::= assert stmt ::= assert2 @@ -218,8 +219,13 @@ class Python2Parser(PythonParser): def p_expr2(self, args): - ''' + """ expr ::= LOAD_LOCALS + expr ::= slice0 + expr ::= slice1 + expr ::= slice2 + expr ::= slice3 + expr ::= unary_convert slice0 ::= expr SLICE+0 slice0 ::= expr DUP_TOP SLICE+0 @@ -229,10 +235,38 @@ class Python2Parser(PythonParser): slice2 ::= expr expr DUP_TOPX_2 SLICE+2 slice3 ::= expr expr expr SLICE+3 slice3 ::= expr expr expr DUP_TOPX_3 SLICE+3 + unary_convert ::= expr UNARY_CONVERT # In Python 3, DUP_TOPX_2 is DUP_TOP_TWO binary_subscr2 ::= expr expr DUP_TOPX_2 BINARY_SUBSCR - ''' + """ + + def p_slice2(self, args): + """ + designator ::= expr STORE_SLICE+0 + designator ::= expr expr STORE_SLICE+1 + designator ::= expr expr STORE_SLICE+2 + designator ::= expr expr expr STORE_SLICE+3 + augassign1 ::= expr expr inplace_op ROT_TWO STORE_SLICE+0 + augassign1 ::= expr expr inplace_op ROT_THREE STORE_SLICE+1 + augassign1 ::= expr expr inplace_op ROT_THREE STORE_SLICE+2 + augassign1 ::= expr expr inplace_op ROT_FOUR STORE_SLICE+3 + slice0 ::= expr SLICE+0 + slice0 ::= expr DUP_TOP SLICE+0 + slice1 ::= expr expr SLICE+1 + slice1 ::= expr expr DUP_TOPX_2 SLICE+1 + slice2 ::= expr expr SLICE+2 + slice2 ::= expr expr DUP_TOPX_2 SLICE+2 + slice3 ::= expr expr expr SLICE+3 + slice3 ::= expr expr expr DUP_TOPX_3 SLICE+3 + """ + + def p_op2(self, args): + """ + inplace_op ::= INPLACE_DIVIDE + binary_op ::= BINARY_DIVIDE + binary_subscr2 ::= expr expr DUP_TOPX_2 BINARY_SUBSCR + """ def add_custom_rules(self, tokens, customize): ''' diff --git a/uncompyle6/parsers/parse3.py b/uncompyle6/parsers/parse3.py index 00be5b6d..926704b9 100644 --- a/uncompyle6/parsers/parse3.py +++ b/uncompyle6/parsers/parse3.py @@ -90,10 +90,6 @@ class Python3Parser(PythonParser): raise_stmt2 ::= expr expr RAISE_VARARGS_2 raise_stmt3 ::= expr expr expr RAISE_VARARGS_3 - stmt ::= exec_stmt - exec_stmt ::= expr exprlist DUP_TOP EXEC_STMT - exec_stmt ::= expr exprlist EXEC_STMT - stmt ::= assert stmt ::= assert2 stmt ::= ifstmt @@ -115,10 +111,6 @@ class Python3Parser(PythonParser): del_stmt ::= DELETE_FAST del_stmt ::= DELETE_NAME del_stmt ::= DELETE_GLOBAL - del_stmt ::= expr DELETE_SLICE+0 - del_stmt ::= expr expr DELETE_SLICE+1 - del_stmt ::= expr expr DELETE_SLICE+2 - del_stmt ::= expr expr expr DELETE_SLICE+3 del_stmt ::= delete_subscr delete_subscr ::= expr expr DELETE_SUBSCR del_stmt ::= expr DELETE_ATTR diff --git a/uncompyle6/parsers/parse34.py b/uncompyle6/parsers/parse34.py index 69be3540..1ef1f7b8 100644 --- a/uncompyle6/parsers/parse34.py +++ b/uncompyle6/parsers/parse34.py @@ -57,7 +57,7 @@ if __name__ == '__main__': s = get_scanner(PYTHON_VERSION, IS_PYPY) opcode_set = set(s.opc.opname).union(set( """JUMP_BACK CONTINUE RETURN_END_IF COME_FROM - LOAD_GENEXPR LOAD_ASSERT LOAD_SETCOMP LOAD_DICTCOMP + LOAD_GENEXPR LOAD_ASSERT LOAD_SETCOMP LOAD_DICTCOMP LOAD_CLASSNAME LAMBDA_MARKER RETURN_LAST """.split())) remain_tokens = set(tokens) - opcode_set