diff --git a/test/Makefile b/test/Makefile index 1443d6bd..8b6db98a 100644 --- a/test/Makefile +++ b/test/Makefile @@ -8,6 +8,7 @@ NATIVE_CHECK = check-$(PYTHON_VERSION) # Set COMPILE='--compile' to force compilation before check COMPILE ?= +COVER_DIR=../tmp/grammar-cover # Run short tests check-short: @@ -101,6 +102,36 @@ check-bytecode-2.4: check-bytecode-2.5: $(PYTHON) test_pythonlib.py --bytecode-2.5 +#: Get grammar coverage for Python 2.5 +grammar-coverage-2.5: + SPARK_PARSER_COVERAGE=/tmp/spark-grammar-25.cover $(PYTHON) test_pythonlib.py --bytecode-2.5 + SPARK_PARSER_COVERAGE=/tmp/spark-grammar-25.cover $(PYTHON) test_pyenvlib.py --2.5.6 + +#: Get grammar coverage for Python 2.6 +grammar-coverage-2.6: + SPARK_PARSER_COVERAGE=/tmp/spark-grammar-26.cover $(PYTHON) test_pythonlib.py --bytecode-2.6 + SPARK_PARSER_COVERAGE=/tmp/spark-grammar-26.cover $(PYTHON) test_pyenvlib.py --2.6.9 + +#: Get grammar coverage for Python 2.7 +grammar-coverage-2.7: + SPARK_PARSER_COVERAGE=/tmp/spark-grammar-27.cover $(PYTHON) test_pythonlib.py --bytecode-2.7 + SPARK_PARSER_COVERAGE=/tmp/spark-grammar-27.cover $(PYTHON) test_pyenvlib.py --2.7.13 + +#: Get grammar coverage for Python 3.4 +grammar-coverage-3.4: + rm $(COVER_DIR)/spark-grammar-34.cover || /bin/true + SPARK_PARSER_COVERAGE=$(COVER_DIR)/spark-grammar-34.cover $(PYTHON) test_pythonlib.py --bytecode-3.4 + SPARK_PARSER_COVERAGE=$(COVER_DIR)/spark-grammar-34.cover $(PYTHON) test_pyenvlib.py --3.4.2 + +#: Check deparsing Python 2.6 +check-bytecode-2.6: + $(PYTHON) test_pythonlib.py --bytecode-2.6 --weak-verify + +#: Check deparsing Python 2.7 +check-bytecode-2.7: + $(PYTHON) test_pythonlib.py --bytecode-2.7 --verify + +>>>>>>> master #: Check deparsing Python 3.0 check-bytecode-3.0: $(PYTHON) test_pythonlib.py --bytecode-3.0 diff --git a/test/bytecode_2.6/05_with.pyc b/test/bytecode_2.6/05_with.pyc index fadb2815..e136d6ab 100644 Binary files a/test/bytecode_2.6/05_with.pyc and b/test/bytecode_2.6/05_with.pyc differ diff --git a/uncompyle6/parser.py b/uncompyle6/parser.py index f07c5ede..d91bb8f4 100644 --- a/uncompyle6/parser.py +++ b/uncompyle6/parser.py @@ -231,6 +231,8 @@ class PythonParser(GenericASTBuilder): suite_stmts ::= continue_stmts suite_stmts_opt ::= suite_stmts + + # passtmt is needed for semantic actions to add "pass" suite_stmts_opt ::= passstmt else_suite ::= suite_stmts @@ -239,7 +241,6 @@ class PythonParser(GenericASTBuilder): else_suitec ::= return_stmts stmt ::= assert - stmt ::= assert2 stmt ::= classdef stmt ::= call_stmt @@ -469,7 +470,6 @@ class PythonParser(GenericASTBuilder): expr ::= cmp expr ::= mapexpr expr ::= and - expr ::= and2 expr ::= or expr ::= unary_expr expr ::= call_function @@ -511,13 +511,8 @@ class PythonParser(GenericASTBuilder): yield ::= expr YIELD_VALUE - _mklambda ::= load_closure mklambda _mklambda ::= mklambda - # "and" where the first part of the and is true, - # so there is only the 2nd part to evaluate - and2 ::= _jump jmp_false COME_FROM expr COME_FROM - expr ::= conditional conditional ::= expr jmp_false expr JUMP_FORWARD expr COME_FROM conditional ::= expr jmp_false expr JUMP_ABSOLUTE expr diff --git a/uncompyle6/parsers/parse2.py b/uncompyle6/parsers/parse2.py index 1edd0e6e..bc71cb57 100644 --- a/uncompyle6/parsers/parse2.py +++ b/uncompyle6/parsers/parse2.py @@ -1,4 +1,4 @@ -# Copyright (c) 2015-2016 Rocky Bernstein +# Copyright (c) 2015-2017 Rocky Bernstein # Copyright (c) 2000-2002 by hartmut Goebel # Copyright (c) 1999 John Aycock """ @@ -80,6 +80,7 @@ class Python2Parser(PythonParser): continue_stmts ::= lastl_stmt continue_stmt continue_stmts ::= continue_stmt + stmt ::= assert2 stmt ::= raise_stmt0 stmt ::= raise_stmt1 stmt ::= raise_stmt2 @@ -100,7 +101,8 @@ class Python2Parser(PythonParser): delete_subscr ::= expr expr DELETE_SUBSCR del_stmt ::= expr DELETE_ATTR - kwarg ::= LOAD_CONST expr + _mklambda ::= load_closure mklambda + kwarg ::= LOAD_CONST expr classdef ::= buildclass designator diff --git a/uncompyle6/parsers/parse23.py b/uncompyle6/parsers/parse23.py index abbd41c7..546ce153 100644 --- a/uncompyle6/parsers/parse23.py +++ b/uncompyle6/parsers/parse23.py @@ -42,6 +42,11 @@ class Python23Parser(Python24Parser): lc_body ::= LOAD_FAST expr CALL_FUNCTION_1 POP_TOP lc_body ::= LOAD_NAME expr LIST_APPEND lc_body ::= LOAD_FAST expr LIST_APPEND + + # "and" where the first part of the and is true, + # so there is only the 2nd part to evaluate + expr ::= and2 + and2 ::= _jump jmp_false COME_FROM expr COME_FROM ''' def add_custom_rules(self, tokens, customize): diff --git a/uncompyle6/parsers/parse25.py b/uncompyle6/parsers/parse25.py index ef3343db..9c539c29 100644 --- a/uncompyle6/parsers/parse25.py +++ b/uncompyle6/parsers/parse25.py @@ -48,11 +48,13 @@ class Python25Parser(Python26Parser): def add_custom_rules(self, tokens, customize): # grammar rules inherited from Python 2.6 self.remove_rules(""" - setupwith ::= DUP_TOP LOAD_ATTR ROT_TWO LOAD_ATTR CALL_FUNCTION_0 POP_TOP - withstmt ::= expr setupwith SETUP_FINALLY suite_stmts_opt - POP_BLOCK LOAD_CONST COME_FROM WITH_CLEANUP END_FINALLY + setupwith ::= DUP_TOP LOAD_ATTR ROT_TWO LOAD_ATTR CALL_FUNCTION_0 POP_TOP + withstmt ::= expr setupwith SETUP_FINALLY suite_stmts_opt + POP_BLOCK LOAD_CONST COME_FROM WITH_CLEANUP END_FINALLY withasstmt ::= expr setupwithas designator suite_stmts_opt POP_BLOCK LOAD_CONST COME_FROM WITH_CLEANUP END_FINALLY + whilestmt ::= SETUP_LOOP testexpr return_stmts come_froms POP_TOP bp_come_from + assert2 ::= assert_expr jmp_true LOAD_ASSERT expr CALL_FUNCTION_1 RAISE_VARARGS_1 """) super(Python25Parser, self).add_custom_rules(tokens, customize) if self.version == 2.5: diff --git a/uncompyle6/parsers/parse3.py b/uncompyle6/parsers/parse3.py index 80ef71a6..393caa54 100644 --- a/uncompyle6/parsers/parse3.py +++ b/uncompyle6/parsers/parse3.py @@ -124,10 +124,6 @@ class Python3Parser(PythonParser): classdefdeco1 ::= expr classdefdeco2 CALL_FUNCTION_1 assert ::= assert_expr jmp_true LOAD_ASSERT RAISE_VARARGS_1 COME_FROM - assert2 ::= assert_expr jmp_true LOAD_ASSERT expr CALL_FUNCTION_1 - RAISE_VARARGS_1 COME_FROM - assert2 ::= assert_expr jmp_true LOAD_ASSERT expr - RAISE_VARARGS_2 COME_FROM assert_expr ::= expr assert_expr ::= assert_expr_or @@ -281,8 +277,8 @@ class Python3Parser(PythonParser): for_block ::= l_stmts_opt opt_come_from_loop JUMP_BACK for_block ::= l_stmts iflaststmtl ::= testexpr c_stmts_opt - iflaststmt ::= testexpr c_stmts_opt34 - c_stmts_opt34 ::= JUMP_BACK JUMP_ABSOLUTE c_stmts_opt + iflaststmt ::= testexpr c_stmts_opt33 + c_stmts_opt33 ::= JUMP_BACK JUMP_ABSOLUTE c_stmts_opt """ def p_def_annotations3(self, args): @@ -436,13 +432,6 @@ class Python3Parser(PythonParser): return_stmt_lambda LAMBDA_MARKER conditional_lambda ::= expr jmp_false expr return_if_lambda return_stmt_lambda LAMBDA_MARKER - - - expr ::= LOAD_CLASSNAME - - # Python 3.4+ - expr ::= LOAD_CLASSDEREF - """ @staticmethod @@ -491,7 +480,8 @@ class Python3Parser(PythonParser): self.add_unique_rule(rule, opname, token.attr, customize) return - def custom_classfunc_rule(self, opname, token, customize): + def custom_classfunc_rule(self, opname, token, customize, + seen_LOAD_BUILD_CLASS): """ call_function ::= expr {expr}^n CALL_FUNCTION_n call_function ::= expr {expr}^n CALL_FUNCTION_VAR_n @@ -550,9 +540,10 @@ class Python3Parser(PythonParser): self.add_unique_rule(rule, token.kind, uniq_param, customize) self.add_unique_rule('expr ::= async_call_function', token.kind, uniq_param, customize) - rule = ('classdefdeco2 ::= LOAD_BUILD_CLASS mkfunc %s%s_%d' - % (('expr ' * (args_pos-1)), opname, args_pos)) - self.add_unique_rule(rule, token.kind, uniq_param, customize) + if seen_LOAD_BUILD_CLASS: + rule = ('classdefdeco2 ::= LOAD_BUILD_CLASS mkfunc %s%s_%d' + % (('expr ' * (args_pos-1)), opname, args_pos)) + self.add_unique_rule(rule, token.kind, uniq_param, customize) def add_make_function_rule(self, rule, opname, attr, customize): """Python 3.3 added a an addtional LOAD_CONST before MAKE_FUNCTION and @@ -628,10 +619,18 @@ class Python3Parser(PythonParser): load_attr ::= expr LOOKUP_METHOD call_function ::= expr CALL_METHOD """ + seen_LOAD_BUILD_CLASS = False + seen_LOAD_DICTCOMP = False + seen_LOAD_LISTCOMP = False + + # Loop over instructions adding custom grammar rules based on + # a specific instruction seen. + for i, token in enumerate(tokens): opname = token.kind opname_base = opname[:opname.rfind('_')] + # The order listed is roughly alphabetic by instruction opcode. if opname == 'PyPy': self.addRule(""" stmt ::= assign3_pypy @@ -640,21 +639,11 @@ class Python3Parser(PythonParser): assign2_pypy ::= expr expr designator designator """, nop_func) continue - elif (opname in ('CALL_FUNCTION', 'CALL_FUNCTION_VAR', - 'CALL_FUNCTION_VAR_KW', 'CALL_FUNCTION_EX_KW') - or opname.startswith('CALL_FUNCTION_KW')): - self.custom_classfunc_rule(opname, token, customize) - elif opname == 'LOAD_DICTCOMP': - rule_pat = ("dictcomp ::= LOAD_DICTCOMP %sMAKE_FUNCTION_0 expr " - "GET_ITER CALL_FUNCTION_1") - self.add_make_function_rule(rule_pat, opname, token.attr, customize) - elif opname == 'LOAD_SETCOMP': - # Should this be generalized and put under MAKE_FUNCTION? - rule_pat = ("setcomp ::= LOAD_SETCOMP %sMAKE_FUNCTION_0 expr " - "GET_ITER CALL_FUNCTION_1") - self.add_make_function_rule(rule_pat, opname, token.attr, customize) - elif opname == 'LOAD_BUILD_CLASS': - self.custom_build_class_rule(opname, i, token, tokens, customize) + elif opname_base == 'BUILD_CONST_KEY_MAP': + # This is in 3.6+ + kvlist_n = 'expr ' * (token.attr) + rule = "mapexpr ::= %sLOAD_CONST %s" % (kvlist_n, opname) + self.add_unique_rule(rule, opname, token.attr, customize) elif opname.startswith('BUILD_LIST_UNPACK'): v = token.attr rule = ('build_list_unpack ::= ' + 'expr1024 ' * int(v//1024) + @@ -663,51 +652,6 @@ class Python3Parser(PythonParser): self.add_unique_rule(rule, opname, token.attr, customize) rule = 'expr ::= build_list_unpack' self.add_unique_rule(rule, opname, token.attr, customize) - elif opname.startswith('BUILD_TUPLE_UNPACK_WITH_CALL'): - v = token.attr - rule = ('build_tuple_unpack_with_call ::= ' + 'expr1024 ' * int(v//1024) + - 'expr32 ' * int((v//32) % 32) + - 'expr ' * (v % 32) + opname) - self.add_unique_rule(rule, opname, token.attr, customize) - elif opname.startswith('BUILD_MAP_UNPACK_WITH_CALL'): - v = token.attr - rule = ('build_map_unpack_with_call ::= ' + 'expr1024 ' * int(v//1024) + - 'expr32 ' * int((v//32) % 32) + - 'expr ' * (v % 32) + opname) - self.add_unique_rule(rule, opname, token.attr, customize) - elif opname_base in ('BUILD_LIST', 'BUILD_TUPLE', 'BUILD_SET'): - v = token.attr - rule = ('build_list ::= ' + 'expr1024 ' * int(v//1024) + - 'expr32 ' * int((v//32) % 32) + - 'expr ' * (v % 32) + opname) - self.add_unique_rule(rule, opname, token.attr, customize) - if opname_base == 'BUILD_TUPLE': - rule = ('load_closure ::= %s%s' % (('LOAD_CLOSURE ' * v), opname)) - self.add_unique_rule(rule, opname, token.attr, customize) - rule = ('build_tuple ::= ' + 'expr1024 ' * int(v//1024) + - 'expr32 ' * int((v//32) % 32) + - 'expr ' * (v % 32) + opname) - self.add_unique_rule(rule, opname, token.attr, customize) - elif opname == 'LOOKUP_METHOD': - # A PyPy speciality - DRY with parse2 - self.add_unique_rule("load_attr ::= expr LOOKUP_METHOD", - opname, token.attr, customize) - continue - elif opname == 'JUMP_IF_NOT_DEBUG': - v = token.attr - self.add_unique_rule( - "stmt ::= assert_pypy", opname, v, customize) - self.add_unique_rule( - "stmt ::= assert2_pypy", opname_base, v, customize) - self.add_unique_rule( - "assert_pypy ::= JUMP_IF_NOT_DEBUG assert_expr jmp_true " - "LOAD_ASSERT RAISE_VARARGS_1 COME_FROM", - opname, token.attr, customize) - self.add_unique_rule( - "assert2_pypy ::= JUMP_IF_NOT_DEBUG assert_expr jmp_true " - "LOAD_ASSERT expr CALL_FUNCTION_1 RAISE_VARARGS_1 COME_FROM", - opname_base, v, customize) - continue elif opname_base == 'BUILD_MAP': kvlist_n = "kvlist_%s" % token.attr if opname == 'BUILD_MAP_n': @@ -743,20 +687,137 @@ class Python3Parser(PythonParser): self.add_unique_rule(rule, opname, token.attr, customize) rule = "mapexpr ::= %s %s" % (opname, kvlist_n) self.add_unique_rule(rule, opname, token.attr, customize) - elif opname_base == 'BUILD_CONST_KEY_MAP': - # This is in 3.6+ - kvlist_n = 'expr ' * (token.attr) - rule = "mapexpr ::= %sLOAD_CONST %s" % (kvlist_n, opname) + elif opname.startswith('BUILD_MAP_UNPACK_WITH_CALL'): + v = token.attr + rule = ('build_map_unpack_with_call ::= ' + 'expr1024 ' * int(v//1024) + + 'expr32 ' * int((v//32) % 32) + + 'expr ' * (v % 32) + opname) self.add_unique_rule(rule, opname, token.attr, customize) - elif opname_base in ('UNPACK_EX',): - before_count, after_count = token.attr - rule = 'unpack ::= ' + opname + ' designator' * (before_count + after_count + 1) + elif opname_base in ('BUILD_LIST', 'BUILD_TUPLE', 'BUILD_SET'): + v = token.attr + rule = ('build_list ::= ' + 'expr1024 ' * int(v//1024) + + 'expr32 ' * int((v//32) % 32) + + 'expr ' * (v % 32) + opname) self.add_unique_rule(rule, opname, token.attr, customize) - elif opname_base in ('UNPACK_TUPLE', 'UNPACK_SEQUENCE'): - rule = 'unpack ::= ' + opname + ' designator' * token.attr + if opname_base == 'BUILD_TUPLE': + rule = ('load_closure ::= %s%s' % (('LOAD_CLOSURE ' * v), opname)) + self.add_unique_rule(rule, opname, token.attr, customize) + elif opname.startswith('BUILD_TUPLE_UNPACK_WITH_CALL'): + v = token.attr + rule = ('build_tuple_unpack_with_call ::= ' + 'expr1024 ' * int(v//1024) + + 'expr32 ' * int((v//32) % 32) + + 'expr ' * (v % 32) + opname) self.add_unique_rule(rule, opname, token.attr, customize) - elif opname_base == 'UNPACK_LIST': - rule = 'unpack_list ::= ' + opname + ' designator' * token.attr + elif (opname in ('CALL_FUNCTION', 'CALL_FUNCTION_VAR', + 'CALL_FUNCTION_VAR_KW', 'CALL_FUNCTION_EX_KW') + or opname.startswith('CALL_FUNCTION_KW')): + self.custom_classfunc_rule(opname, token, customize, seen_LOAD_BUILD_CLASS) + elif opname_base == 'CALL_METHOD': + # PyPy only - DRY with parse2 + + # FIXME: The below argument parsing will be wrong when PyPy gets to 3.6 + args_pos = (token.attr & 0xff) # positional parameters + args_kw = (token.attr >> 8) & 0xff # keyword parameters + + # number of apply equiv arguments: + nak = ( len(opname_base)-len('CALL_METHOD') ) // 3 + rule = ('call_function ::= expr ' + + ('pos_arg ' * args_pos) + + ('kwarg ' * args_kw) + + 'expr ' * nak + opname) + self.add_unique_rule(rule, opname, token.attr, customize) + elif opname == 'JUMP_IF_NOT_DEBUG': + v = token.attr + self.add_unique_rule( + "stmt ::= assert_pypy", opname, v, customize) + self.add_unique_rule( + "stmt ::= assert2_pypy", opname_base, v, customize) + self.add_unique_rule( + "assert_pypy ::= JUMP_IF_NOT_DEBUG assert_expr jmp_true " + "LOAD_ASSERT RAISE_VARARGS_1 COME_FROM", + opname, token.attr, customize) + self.add_unique_rule( + "assert2_pypy ::= JUMP_IF_NOT_DEBUG assert_expr jmp_true " + "LOAD_ASSERT expr CALL_FUNCTION_1 RAISE_VARARGS_1 COME_FROM", + opname_base, v, customize) + continue + elif opname == 'LOAD_BUILD_CLASS': + seen_LOAD_BUILD_CLASS = True + self.custom_build_class_rule(opname, i, token, tokens, customize) + elif opname == 'LOAD_CLASSDEREF': + # Python 3.4+ + self.add_unique_rule("expr ::= LOAD_CLASSDEREF", + opname, token.attr, customize) + continue + elif opname == 'LOAD_CLASSNAME': + self.add_unique_rule("expr ::= LOAD_CLASSNAME", + opname, token.attr, customize) + continue + elif opname == 'LOAD_DICTCOMP': + seen_LOAD_DICTCOMP = True + rule_pat = ("dictcomp ::= LOAD_DICTCOMP %sMAKE_FUNCTION_0 expr " + "GET_ITER CALL_FUNCTION_1") + self.add_make_function_rule(rule_pat, opname, token.attr, customize) + elif opname == 'LOAD_LISTCOMP': + seen_LOAD_LISTCOMP = True + continue + elif opname == 'LOAD_SETCOMP': + # Should this be generalized and put under MAKE_FUNCTION? + rule_pat = ("setcomp ::= LOAD_SETCOMP %sMAKE_FUNCTION_0 expr " + "GET_ITER CALL_FUNCTION_1") + self.add_make_function_rule(rule_pat, opname, token.attr, customize) + elif opname == 'LOOKUP_METHOD': + # A PyPy speciality - DRY with parse2 + self.add_unique_rule("load_attr ::= expr LOOKUP_METHOD", + opname, token.attr, customize) + continue + elif opname.startswith('MAKE_CLOSURE'): + # DRY with MAKE_FUNCTION + # Note: this probably doesn't handle kwargs proprerly + args_pos, args_kw, annotate_args = token.attr + + rule_pat = ("genexpr ::= %sload_closure load_genexpr %%s%s expr " + "GET_ITER CALL_FUNCTION_1" % ('pos_arg '* args_pos, opname)) + self.add_make_function_rule(rule_pat, opname, token.attr, customize) + rule_pat = ('mklambda ::= %sload_closure LOAD_LAMBDA %%s%s' % + ('pos_arg '* args_pos, opname)) + self.add_make_function_rule(rule_pat, opname, token.attr, customize) + rule_pat = ('listcomp ::= %sload_closure LOAD_LISTCOMP %%s%s expr ' + 'GET_ITER CALL_FUNCTION_1' % ('pos_arg ' * args_pos, opname)) + self.add_make_function_rule(rule_pat, opname, token.attr, customize) + rule_pat = ('setcomp ::= %sload_closure LOAD_SETCOMP %%s%s expr ' + 'GET_ITER CALL_FUNCTION_1' % ('pos_arg ' * args_pos, opname)) + self.add_make_function_rule(rule_pat, opname, token.attr, customize) + + if seen_LOAD_DICTCOMP: + self.add_unique_rule('dictcomp ::= %sload_closure LOAD_DICTCOMP %s ' + 'expr GET_ITER CALL_FUNCTION_1' % + ('pos_arg '* args_pos, opname), + opname, token.attr, customize) + + # FIXME: kwarg processing is missing here. + # Note order of kwargs and pos args changed between 3.3-3.4 + if self.version <= 3.2: + rule = ('mkfunc ::= kwargs %sload_closure LOAD_CONST kwargs %s' + % ('expr ' * args_pos, opname)) + elif self.version == 3.3: + rule = ('mkfunc ::= kwargs %sload_closure LOAD_CONST LOAD_CONST %s' + % ('expr ' * args_pos, opname)) + elif self.version >= 3.4: + rule = ('mkfunc ::= %skwargs load_closure LOAD_CONST LOAD_CONST %s' + % ('expr ' * args_pos, opname)) + + self.add_unique_rule(rule, opname, token.attr, customize) + rule = ('mkfunc ::= %sload_closure load_genexpr %s' + % ('pos_arg ' * args_pos, opname)) + self.add_unique_rule(rule, opname, token.attr, customize) + + if self.version < 3.4: + rule = ('mkfunc ::= %sload_closure LOAD_CONST %s' + % ('expr ' * args_pos, opname)) + self.add_unique_rule(rule, opname, token.attr, customize) + + pass elif opname_base.startswith('MAKE_FUNCTION'): # DRY with MAKE_CLOSURE if self.version >= 3.6: @@ -775,9 +836,10 @@ class Python3Parser(PythonParser): ('kwarg '* args_kw), opname)) self.add_make_function_rule(rule_pat, opname, token.attr, customize) - rule_pat = ("listcomp ::= %sLOAD_LISTCOMP %%s%s expr " - "GET_ITER CALL_FUNCTION_1" % ('expr ' * args_pos, opname)) - self.add_make_function_rule(rule_pat, opname, token.attr, customize) + if seen_LOAD_LISTCOMP: + rule_pat = ("listcomp ::= %sLOAD_LISTCOMP %%s%s expr " + "GET_ITER CALL_FUNCTION_1" % ('expr ' * args_pos, opname)) + self.add_make_function_rule(rule_pat, opname, token.attr, customize) continue if self.version < 3.6: args_pos, args_kw, annotate_args = token.attr @@ -840,66 +902,15 @@ class Python3Parser(PythonParser): (('pos_arg ' * (args_pos)), ('call_function ' * (annotate_args-1)), opname)) self.add_unique_rule(rule, opname, token.attr, customize) - elif opname_base == 'CALL_METHOD': - # PyPy only - DRY with parse2 - - # FIXME: The below argument parsing will be wrong when PyPy gets to 3.6 - args_pos = (token.attr & 0xff) # positional parameters - args_kw = (token.attr >> 8) & 0xff # keyword parameters - - # number of apply equiv arguments: - nak = ( len(opname_base)-len('CALL_METHOD') ) // 3 - rule = ('call_function ::= expr ' + - ('pos_arg ' * args_pos) + - ('kwarg ' * args_kw) + - 'expr ' * nak + opname) + elif opname_base in ('UNPACK_EX',): + before_count, after_count = token.attr + rule = 'unpack ::= ' + opname + ' designator' * (before_count + after_count + 1) self.add_unique_rule(rule, opname, token.attr, customize) - elif opname.startswith('MAKE_CLOSURE'): - # DRY with MAKE_FUNCTION - # Note: this probably doesn't handle kwargs proprerly - args_pos, args_kw, annotate_args = token.attr - - rule_pat = ("genexpr ::= %sload_closure load_genexpr %%s%s expr " - "GET_ITER CALL_FUNCTION_1" % ('pos_arg '* args_pos, opname)) - self.add_make_function_rule(rule_pat, opname, token.attr, customize) - rule_pat = ('mklambda ::= %sload_closure LOAD_LAMBDA %%s%s' % - ('pos_arg '* args_pos, opname)) - self.add_make_function_rule(rule_pat, opname, token.attr, customize) - rule_pat = ('listcomp ::= %sload_closure LOAD_LISTCOMP %%s%s expr ' - 'GET_ITER CALL_FUNCTION_1' % ('pos_arg ' * args_pos, opname)) - self.add_make_function_rule(rule_pat, opname, token.attr, customize) - rule_pat = ('setcomp ::= %sload_closure LOAD_SETCOMP %%s%s expr ' - 'GET_ITER CALL_FUNCTION_1' % ('pos_arg ' * args_pos, opname)) - self.add_make_function_rule(rule_pat, opname, token.attr, customize) - - self.add_unique_rule('dictcomp ::= %sload_closure LOAD_DICTCOMP %s ' - 'expr GET_ITER CALL_FUNCTION_1' % - ('pos_arg '* args_pos, opname), - opname, token.attr, customize) - - # FIXME: kwarg processing is missing here. - # Note order of kwargs and pos args changed between 3.3-3.4 - if self.version <= 3.2: - rule = ('mkfunc ::= kwargs %sload_closure LOAD_CONST kwargs %s' - % ('expr ' * args_pos, opname)) - elif self.version == 3.3: - rule = ('mkfunc ::= kwargs %sload_closure LOAD_CONST LOAD_CONST %s' - % ('expr ' * args_pos, opname)) - elif self.version >= 3.4: - rule = ('mkfunc ::= %skwargs load_closure LOAD_CONST LOAD_CONST %s' - % ('expr ' * args_pos, opname)) - + elif opname_base in ('UNPACK_TUPLE', 'UNPACK_SEQUENCE'): + rule = 'unpack ::= ' + opname + ' designator' * token.attr self.add_unique_rule(rule, opname, token.attr, customize) - rule = ('mkfunc ::= %sload_closure load_genexpr %s' - % ('pos_arg ' * args_pos, opname)) - self.add_unique_rule(rule, opname, token.attr, customize) - - if self.version < 3.4: - rule = ('mkfunc ::= %sload_closure LOAD_CONST %s' - % ('expr ' * args_pos, opname)) - self.add_unique_rule(rule, opname, token.attr, customize) - - pass + elif opname_base == 'UNPACK_LIST': + rule = 'unpack_list ::= ' + opname + ' designator' * token.attr self.check_reduce['augassign1'] = 'AST' self.check_reduce['augassign2'] = 'AST' self.check_reduce['while1stmt'] = 'noAST' diff --git a/uncompyle6/parsers/parse33.py b/uncompyle6/parsers/parse33.py index 8df4dd34..d08101b2 100644 --- a/uncompyle6/parsers/parse33.py +++ b/uncompyle6/parsers/parse33.py @@ -17,9 +17,6 @@ class Python33Parser(Python32Parser): # We do the grammar hackery below for semantics # actions that want c_stmts_opt at index 1 - iflaststmt ::= testexpr c_stmts_opt33 - c_stmts_opt33 ::= JUMP_BACK JUMP_ABSOLUTE c_stmts_opt - whileTruestmt ::= SETUP_LOOP l_stmts JUMP_ABSOLUTE JUMP_BACK COME_FROM_LOOP diff --git a/uncompyle6/parsers/parse34.py b/uncompyle6/parsers/parse34.py index bba5c29d..d55e56ce 100644 --- a/uncompyle6/parsers/parse34.py +++ b/uncompyle6/parsers/parse34.py @@ -1,4 +1,4 @@ -# Copyright (c) 2016 Rocky Bernstein +# Copyright (c) 2017 Rocky Bernstein """ spark grammar differences over Python 3.3 for Python 3.4 """ @@ -22,6 +22,18 @@ class Python34Parser(Python33Parser): # Is this 3.4 only? yield_from ::= expr GET_ITER LOAD_CONST YIELD_FROM """ + + def add_custom_rules(self, tokens, customize): + self.remove_rules(""" + whileTruestmt ::= SETUP_LOOP l_stmts JUMP_ABSOLUTE JUMP_BACK COME_FROM_LOOP + whileTruestmt ::= SETUP_LOOP l_stmts_opt JUMP_BACK NOP COME_FROM_LOOP + whileTruestmt ::= SETUP_LOOP l_stmts_opt JUMP_BACK POP_BLOCK NOP COME_FROM_LOOP + whilestmt ::= SETUP_LOOP testexpr l_stmts_opt JUMP_BACK + POP_BLOCK JUMP_ABSOLUTE COME_FROM_LOOP + """) + super(Python34Parser, self).add_custom_rules(tokens, customize) + return + class Python34ParserSingle(Python34Parser, PythonParserSingle): pass diff --git a/uncompyle6/parsers/parse36.py b/uncompyle6/parsers/parse36.py index b6c6bb67..1a8ffd53 100644 --- a/uncompyle6/parsers/parse36.py +++ b/uncompyle6/parsers/parse36.py @@ -97,8 +97,7 @@ class Python36Parser(Python35Parser): """ % (fstring_expr_or_str_n, fstring_expr_or_str_n, "fstring_expr_or_str " * v) self.add_unique_doc_rules(rules_str, customize) - def custom_classfunc_rule(self, opname, token, customize): - + def custom_classfunc_rule(self, opname, token, customize, seen_LOAD_BUILD_CLASS): if opname.startswith('CALL_FUNCTION_KW'): values = 'expr ' * token.attr rule = 'call_function ::= expr kwargs_only_36 {token.kind}'.format(**locals()) @@ -106,7 +105,8 @@ class Python36Parser(Python35Parser): rule = 'kwargs_only_36 ::= {values} LOAD_CONST'.format(**locals()) self.add_unique_rule(rule, token.kind, token.attr, customize) else: - super(Python36Parser, self).custom_classfunc_rule(opname, token, customize) + super(Python36Parser, self).custom_classfunc_rule(opname, token, + customize, seen_LOAD_BUILD_CLASS) class Python36ParserSingle(Python36Parser, PythonParserSingle): diff --git a/uncompyle6/semantics/consts.py b/uncompyle6/semantics/consts.py index 1fef9cb8..e9fc7274 100644 --- a/uncompyle6/semantics/consts.py +++ b/uncompyle6/semantics/consts.py @@ -67,27 +67,27 @@ TABLE_R0 = { } TABLE_DIRECT = { - 'BINARY_ADD': ( '+' ,), - 'BINARY_SUBTRACT': ( '-' ,), - 'BINARY_MULTIPLY': ( '*' ,), - 'BINARY_DIVIDE': ( '/' ,), - 'BINARY_MATRIX_MULTIPLY': ( '@' ,), - 'BINARY_TRUE_DIVIDE': ( '/' ,), # Not in <= 2.1 - 'BINARY_FLOOR_DIVIDE': ( '//' ,), - 'BINARY_MODULO': ( '%%',), - 'BINARY_POWER': ( '**',), - 'BINARY_LSHIFT': ( '<<',), - 'BINARY_RSHIFT': ( '>>',), - 'BINARY_AND': ( '&' ,), - 'BINARY_OR': ( '|' ,), - 'BINARY_XOR': ( '^' ,), - 'INPLACE_ADD': ( '+=' ,), - 'INPLACE_SUBTRACT': ( '-=' ,), - 'INPLACE_MULTIPLY': ( '*=' ,), - 'INPLACE_MATRIX_MULTIPLY': ( '@=' ,), - 'INPLACE_DIVIDE': ( '/=' ,), - 'INPLACE_TRUE_DIVIDE': ( '/=' ,), # Not in <= 2.1; 2.6 generates INPLACE_DIVIDE only? - 'INPLACE_FLOOR_DIVIDE': ( '//=' ,), + 'BINARY_ADD': ( '+' ,), + 'BINARY_SUBTRACT': ( '-' ,), + 'BINARY_MULTIPLY': ( '*' ,), + 'BINARY_DIVIDE': ( '/' ,), + 'BINARY_MATRIX_MULTIPLY': ( '@' ,), + 'BINARY_TRUE_DIVIDE': ( '/' ,), # Not in <= 2.1 + 'BINARY_FLOOR_DIVIDE': ( '//' ,), + 'BINARY_MODULO': ( '%%',), + 'BINARY_POWER': ( '**',), + 'BINARY_LSHIFT': ( '<<',), + 'BINARY_RSHIFT': ( '>>',), + 'BINARY_AND': ( '&' ,), + 'BINARY_OR': ( '|' ,), + 'BINARY_XOR': ( '^' ,), + 'INPLACE_ADD': ( '+=' ,), + 'INPLACE_SUBTRACT': ( '-=' ,), + 'INPLACE_MULTIPLY': ( '*=' ,), + 'INPLACE_MATRIX_MULTIPLY': ( '@=' ,), + 'INPLACE_DIVIDE': ( '/=' ,), + 'INPLACE_TRUE_DIVIDE': ( '/=' ,), # Not in <= 2.1; 2.6 generates INPLACE_DIVIDE only? + 'INPLACE_FLOOR_DIVIDE': ( '//=' ,), 'INPLACE_MODULO': ( '%%=',), 'INPLACE_POWER': ( '**=',), 'INPLACE_LSHIFT': ( '<<=',), @@ -118,9 +118,11 @@ TABLE_DIRECT = { (0, 'expr'), (1, 100) ), 'slice2': ( '%c[:%p]', - 0, (1, 100) ), + (0, 'expr'), + (1, 100) ), 'slice3': ( '%c[%p:%p]', - 0, (1, 100), (2, 100) ), + (0, 'expr'), + (1, 100), (2, 100) ), 'IMPORT_FROM': ( '%{pattr}', ), 'load_attr': ( '%c.%[1]{pattr}', 0), @@ -135,8 +137,12 @@ TABLE_DIRECT = { 'DELETE_NAME': ( '%|del %{pattr}\n', ), 'DELETE_GLOBAL': ( '%|del %{pattr}\n', ), 'delete_subscr': ( '%|del %c[%c]\n', 0, 1,), - 'binary_subscr': ( '%c[%p]', 0, (1, 100)), - 'binary_subscr2': ( '%c[%p]', 0, (1, 100)), + 'binary_subscr': ( '%c[%p]', + (0, 'expr'), + (1, 100) ), + 'binary_subscr2': ( '%c[%p]', + (0, 'expr'), + (1, 100) ), 'store_subscr': ( '%c[%c]', 0, 1), 'STORE_FAST': ( '%{pattr}', ), 'STORE_NAME': ( '%{pattr}', ),