From 6de57249edd2a9a48b83a15633465d9e7fb695e1 Mon Sep 17 00:00:00 2001 From: rocky Date: Wed, 1 Jan 2020 01:42:00 -0500 Subject: [PATCH] Start 3.6+ var type annotations and decompyle3 merge... Although overall an improvement, some new breakage has occurred and should be fixed. --- pytest/test_grammar.py | 2 +- test/bytecode_3.7_run/02_var_annotate.pyc | Bin 0 -> 354 bytes test/bytecode_3.8_run/02_var_annotate.pyc | Bin 0 -> 358 bytes uncompyle6/parsers/parse37.py | 297 ++++++++++++---------- uncompyle6/parsers/parse37base.py | 110 +++++++- uncompyle6/semantics/customize37.py | 182 ++++++++----- uncompyle6/semantics/transform.py | 24 +- 7 files changed, 403 insertions(+), 212 deletions(-) create mode 100644 test/bytecode_3.7_run/02_var_annotate.pyc create mode 100644 test/bytecode_3.8_run/02_var_annotate.pyc diff --git a/pytest/test_grammar.py b/pytest/test_grammar.py index bf66b226..f20cfacd 100644 --- a/pytest/test_grammar.py +++ b/pytest/test_grammar.py @@ -35,7 +35,7 @@ def test_grammar(): expect_right_recursive = set([("designList", ("store", "DUP_TOP", "designList"))]) - if PYTHON_VERSION <= 3.7: + if PYTHON_VERSION <= 3.6: unused_rhs.add("call") if PYTHON_VERSION > 2.6: diff --git a/test/bytecode_3.7_run/02_var_annotate.pyc b/test/bytecode_3.7_run/02_var_annotate.pyc new file mode 100644 index 0000000000000000000000000000000000000000..4f82614d55ba6c3bee50f1577310c8fe652f079a GIT binary patch literal 354 zcmXwyzfQw25XNoC`B$Y>LM*%hQWlCp>J%ZU)D5v9mJCEGKBCYhj_jl&jJypC59F08 zufTvf2lb@;zVGaxee3yrMzDU9=;;gn+YX0vFx+74SCmI0h@cgzX-X*(EM?#T7fwP1 zPhIdZy9ohT@EuEi@b#(-J}5B3c1*8r7M@@QOGBXGof71IxQ;$3iXc+hHs^;%p%M3rjcFVm9o)Urab$?DVt7l zn~rX^24ycdN=c;xJp3eTfi}s{nJ(+Lf=tU^J;QR_7gww0#bvh7)!}D2ZwC`3wG@2? U*LJ@?Mqwm!I1OloZS{oy0S$0e9RL6T literal 0 HcmV?d00001 diff --git a/test/bytecode_3.8_run/02_var_annotate.pyc b/test/bytecode_3.8_run/02_var_annotate.pyc new file mode 100644 index 0000000000000000000000000000000000000000..d9b37d99e341662c5ec586dbc89c1911c89707e6 GIT binary patch literal 358 zcmYjK%}T^D5KhwcZ|S-qUVH(e7grSaR78cni5J;R50GQNPq2ceAyDv+3Fdvwhr()`d73i8J&D5+?~#}q zep~8pWM~Y18QqCrM>_HTE}j{$8``od)Nf=kd@E&T7Fm`zjqLKSl#R|Z<=G!j*>s9q zb#SdUDEqipN-7oL@SUgy(ga^8x~$s@GA(=c2+M6>T&$L7=h;42f1Ba79ZZnaQuGyE T+WYztfsx4JG@uc-#UuIyHgZ%j literal 0 HcmV?d00001 diff --git a/uncompyle6/parsers/parse37.py b/uncompyle6/parsers/parse37.py index 8f8682d2..22bfe67e 100644 --- a/uncompyle6/parsers/parse37.py +++ b/uncompyle6/parsers/parse37.py @@ -1,4 +1,4 @@ -# Copyright (c) 2017-2019 Rocky Bernstein +# Copyright (c) 2017-2020 Rocky Bernstein # # This program is free software: you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by @@ -22,7 +22,6 @@ from spark_parser import DEFAULT_DEBUG as PARSER_DEFAULT_DEBUG from uncompyle6.parsers.parse37base import Python37BaseParser class Python37Parser(Python37BaseParser): - def __init__(self, debug_parser=PARSER_DEFAULT_DEBUG): super(Python37Parser, self).__init__(debug_parser) self.customized = {} @@ -328,6 +327,26 @@ class Python37Parser(Python37BaseParser): attributes ::= LOAD_ATTR+ """ + def p_import37(self, args): + """ + stmt ::= import37 + + # Where does the POP_TOP really belong? + import37 ::= import POP_TOP + + attributes ::= IMPORT_FROM ROT_TWO POP_TOP IMPORT_FROM + attributes ::= attributes ROT_TWO POP_TOP IMPORT_FROM + + # The 3.7base scanner adds IMPORT_NAME_ATTR + alias ::= IMPORT_NAME_ATTR IMPORT_FROM store + alias ::= IMPORT_NAME_ATTR attributes store + alias ::= IMPORT_NAME_ATTR store + import_from ::= LOAD_CONST LOAD_CONST IMPORT_NAME_ATTR importlist POP_TOP + + expr ::= attribute37 + attribute37 ::= expr LOAD_METHOD + """ + def p_list_comprehension(self, args): """ expr ::= list_comp @@ -501,108 +520,12 @@ class Python37Parser(Python37BaseParser): iflaststmt ::= testexpr c_stmts_opt JUMP_FORWARD """ - def p_36misc(self, args): + def p_37async(self, args): """ - sstmt ::= sstmt RETURN_LAST - - # 3.6 redoes how return_closure works. FIXME: Isolate to LOAD_CLOSURE - return_closure ::= LOAD_CLOSURE DUP_TOP STORE_NAME RETURN_VALUE RETURN_LAST - - for_block ::= l_stmts_opt come_from_loops JUMP_BACK - come_from_loops ::= COME_FROM_LOOP* - - whilestmt ::= setup_loop testexpr l_stmts_opt - JUMP_BACK come_froms POP_BLOCK COME_FROM_LOOP - whilestmt ::= setup_loop testexpr l_stmts_opt - come_froms JUMP_BACK come_froms POP_BLOCK COME_FROM_LOOP - - # 3.6 due to jump optimization, we sometimes add RETURN_END_IF where - # RETURN_VALUE is meant. Specifcally this can happen in - # ifelsestmt -> ...else_suite _. suite_stmts... (last) stmt - return ::= ret_expr RETURN_END_IF - return ::= ret_expr RETURN_VALUE COME_FROM - return_stmt_lambda ::= ret_expr RETURN_VALUE_LAMBDA COME_FROM - - # A COME_FROM is dropped off because of JUMP-to-JUMP optimization - and ::= expr jmp_false expr - and ::= expr jmp_false expr jmp_false - - jf_cf ::= JUMP_FORWARD COME_FROM - cf_jf_else ::= come_froms JUMP_FORWARD ELSE - - conditional ::= expr jmp_false expr jf_cf expr COME_FROM - - async_for_stmt ::= setup_loop expr - GET_AITER - LOAD_CONST YIELD_FROM SETUP_EXCEPT GET_ANEXT LOAD_CONST - YIELD_FROM - store - POP_BLOCK JUMP_FORWARD COME_FROM_EXCEPT DUP_TOP - LOAD_GLOBAL COMPARE_OP POP_JUMP_IF_FALSE - POP_TOP POP_TOP POP_TOP POP_EXCEPT POP_BLOCK - JUMP_ABSOLUTE END_FINALLY COME_FROM - for_block POP_BLOCK - COME_FROM_LOOP - - # Adds a COME_FROM_ASYNC_WITH over 3.5 - # FIXME: remove corresponding rule for 3.5? - - except_suite ::= c_stmts_opt COME_FROM POP_EXCEPT jump_except COME_FROM - - jb_cfs ::= come_from_opt JUMP_BACK come_froms - ifelsestmtl ::= testexpr c_stmts_opt jb_cfs else_suitel - ifelsestmtl ::= testexpr c_stmts_opt cf_jf_else else_suitel - - # In 3.6+, A sequence of statements ending in a RETURN can cause - # JUMP_FORWARD END_FINALLY to be omitted from try middle - - except_return ::= POP_TOP POP_TOP POP_TOP returns - except_handler ::= JUMP_FORWARD COME_FROM_EXCEPT except_return - - # Try middle following a returns - except_handler36 ::= COME_FROM_EXCEPT except_stmts END_FINALLY - - stmt ::= try_except36 - try_except36 ::= SETUP_EXCEPT returns except_handler36 - opt_come_from_except - try_except36 ::= SETUP_EXCEPT suite_stmts - try_except36 ::= SETUP_EXCEPT suite_stmts_opt POP_BLOCK - except_handler36 come_from_opt - - # 3.6 omits END_FINALLY sometimes - except_handler36 ::= COME_FROM_EXCEPT except_stmts - except_handler36 ::= JUMP_FORWARD COME_FROM_EXCEPT except_stmts - except_handler ::= jmp_abs COME_FROM_EXCEPT except_stmts - - stmt ::= tryfinally36 - tryfinally36 ::= SETUP_FINALLY returns - COME_FROM_FINALLY suite_stmts - tryfinally36 ::= SETUP_FINALLY returns - COME_FROM_FINALLY suite_stmts_opt END_FINALLY - except_suite_finalize ::= SETUP_FINALLY returns - COME_FROM_FINALLY suite_stmts_opt END_FINALLY _jump - - stmt ::= tryfinally_return_stmt - tryfinally_return_stmt ::= SETUP_FINALLY suite_stmts_opt POP_BLOCK LOAD_CONST - COME_FROM_FINALLY - - compare_chained2 ::= expr COMPARE_OP come_froms JUMP_FORWARD - """ - - def p_37misc(self, args): - """ - stmt ::= import37 stmt ::= async_for_stmt37 stmt ::= async_for_stmt stmt ::= async_forelse_stmt - # Where does the POP_TOP really belong? - import37 ::= import POP_TOP - alias ::= IMPORT_NAME_ATTR IMPORT_FROM store - alias ::= IMPORT_NAME_ATTR attributes store - alias ::= IMPORT_NAME_ATTR store - import_from ::= LOAD_CONST LOAD_CONST IMPORT_NAME_ATTR importlist POP_TOP - async_for_stmt ::= setup_loop expr GET_AITER SETUP_EXCEPT GET_ANEXT LOAD_CONST @@ -641,19 +564,10 @@ class Python37Parser(Python37BaseParser): COME_FROM POP_TOP POP_TOP POP_TOP POP_EXCEPT POP_TOP POP_BLOCK else_suite COME_FROM_LOOP + """ - attributes ::= IMPORT_FROM ROT_TWO POP_TOP IMPORT_FROM - attributes ::= attributes ROT_TWO POP_TOP IMPORT_FROM - - attribute37 ::= expr LOAD_METHOD - expr ::= attribute37 - - # long except clauses in a loop can sometimes cause a JUMP_BACK to turn into a - # JUMP_FORWARD to a JUMP_BACK. And when this happens there is an additional - # ELSE added to the except_suite. With better flow control perhaps we can - # sort this out better. - except_suite ::= c_stmts_opt POP_EXCEPT jump_except ELSE - + def p_37chained(self, args): + """ testtrue ::= compare_chained37 testfalse ::= compare_chained37_false @@ -694,7 +608,10 @@ class Python37Parser(Python37BaseParser): compare_chained2a_false_37 ELSE compare_chained2c_37 ::= expr DUP_TOP ROT_THREE COMPARE_OP come_from_opt POP_JUMP_IF_FALSE compare_chained2a_false_37 + """ + def p_37conditionals(self, args): + """ jf_cfs ::= JUMP_FORWARD _come_froms ifelsestmt ::= testexpr c_stmts_opt jf_cfs else_suite opt_come_from_except @@ -768,6 +685,28 @@ class Python37Parser(Python37BaseParser): comp_iter ::= comp_body """ + def p_expr3(self, args): + """ + expr ::= conditionalnot + conditionalnot ::= expr jmp_true expr jump_forward_else expr COME_FROM + + # a JUMP_FORWARD to another JUMP_FORWARD can get turned into + # a JUMP_ABSOLUTE with no COME_FROM + conditional ::= expr jmp_false expr jump_absolute_else expr + + # if_expr_true are for conditions which always evaluate true + # There is dead or non-optional remnants of the condition code though, + # and we use that to match on to reconstruct the source more accurately + expr ::= if_expr_true + if_expr_true ::= expr JUMP_FORWARD expr COME_FROM + """ + + def p_generator_exp3(self, args): + """ + load_genexpr ::= LOAD_GENEXPR + load_genexpr ::= BUILD_TUPLE_1 LOAD_GENEXPR LOAD_STR + """ + def p_grammar(self, args): """ sstmt ::= stmt @@ -826,6 +765,7 @@ class Python37Parser(Python37BaseParser): iflaststmtl ::= testexpr c_stmts JUMP_BACK POP_BLOCK # These are used to keep parse tree indices the same + jump_forward_else ::= JUMP_FORWARD jump_forward_else ::= JUMP_FORWARD ELSE jump_forward_else ::= JUMP_FORWARD COME_FROM jump_absolute_else ::= JUMP_ABSOLUTE ELSE @@ -950,6 +890,7 @@ class Python37Parser(Python37BaseParser): ret_cond ::= expr POP_JUMP_IF_FALSE expr RETURN_END_IF COME_FROM ret_expr_or_cond jitop_come_from ::= JUMP_IF_TRUE_OR_POP COME_FROM + jifop_come_from ::= JUMP_IF_FALSE_OR_POP COME_FROM or ::= and jitop_come_from expr COME_FROM or ::= expr JUMP_IF_TRUE_OR_POP expr COME_FROM or ::= expr JUMP_IF_TRUE expr COME_FROM @@ -963,11 +904,13 @@ class Python37Parser(Python37BaseParser): testfalse ::= or jmp_false COME_FROM or ::= expr jmp_true expr - - - and ::= expr JUMP_IF_FALSE_OR_POP expr COME_FROM + and ::= expr JUMP_IF_FALSE_OR_POP expr come_from_opt + and ::= expr jifop_come_from expr and ::= expr JUMP_IF_FALSE expr COME_FROM + pjit_come_from ::= POP_JUMP_IF_TRUE COME_FROM + or ::= expr pjit_come_from expr + ## FIXME: Is the below needed or is it covered above?? and ::= expr jmp_false expr COME_FROM or ::= expr jmp_true expr COME_FROM @@ -983,6 +926,8 @@ class Python37Parser(Python37BaseParser): """ stmt ::= if_expr_lambda stmt ::= conditional_not_lambda + stmt ::= ifstmtl + if_expr_lambda ::= expr jmp_false expr return_if_lambda return_stmt_lambda LAMBDA_MARKER conditional_not_lambda @@ -997,6 +942,10 @@ class Python37Parser(Python37BaseParser): stmt ::= whileTruestmt ifelsestmt ::= testexpr c_stmts_opt JUMP_FORWARD else_suite _come_froms + + _ifstmts_jumpl ::= c_stmts JUMP_BACK + _ifstmts_jumpl ::= _ifstmts_jump + ifstmtl ::= testexpr _ifstmts_jumpl """ def p_loop_stmt3(self, args): @@ -1019,12 +968,20 @@ class Python37Parser(Python37BaseParser): whilestmt ::= setup_loop testexpr l_stmts_opt COME_FROM JUMP_BACK POP_BLOCK COME_FROM_LOOP + whilestmt ::= setup_loop testexpr l_stmts_opt JUMP_BACK POP_BLOCK COME_FROM_LOOP whilestmt ::= setup_loop testexpr returns POP_BLOCK COME_FROM_LOOP + # We can be missing a COME_FROM_LOOP if the "while" statement is nested inside an if/else + # so after the POP_BLOCK we have a JUMP_FORWARD which forms the "else" portion of the "if" + # This is undoubtedly some sort of JUMP optimization going on. + + whilestmt ::= setup_loop testexpr l_stmts_opt JUMP_BACK come_froms + POP_BLOCK + while1elsestmt ::= setup_loop l_stmts JUMP_BACK else_suitel @@ -1032,11 +989,12 @@ class Python37Parser(Python37BaseParser): else_suitel COME_FROM_LOOP whileTruestmt ::= setup_loop l_stmts_opt JUMP_BACK POP_BLOCK - COME_FROM_LOOP + _come_froms # FIXME: Python 3.? starts adding branch optimization? Put this starting there. while1stmt ::= setup_loop l_stmts COME_FROM_LOOP + while1stmt ::= setup_loop l_stmts COME_FROM_LOOP JUMP_BACK POP_BLOCK COME_FROM_LOOP while1stmt ::= setup_loop l_stmts COME_FROM JUMP_BACK COME_FROM_LOOP while1elsestmt ::= setup_loop l_stmts JUMP_BACK @@ -1047,26 +1005,105 @@ class Python37Parser(Python37BaseParser): COME_FROM_LOOP """ - def p_generator_exp3(self, args): + def p_36misc(self, args): """ - load_genexpr ::= LOAD_GENEXPR - load_genexpr ::= BUILD_TUPLE_1 LOAD_GENEXPR LOAD_STR + sstmt ::= sstmt RETURN_LAST + + # 3.6 redoes how return_closure works. FIXME: Isolate to LOAD_CLOSURE + return_closure ::= LOAD_CLOSURE DUP_TOP STORE_NAME RETURN_VALUE RETURN_LAST + + for_block ::= l_stmts_opt come_from_loops JUMP_BACK + come_from_loops ::= COME_FROM_LOOP* + + whilestmt ::= setup_loop testexpr l_stmts_opt + JUMP_BACK come_froms POP_BLOCK COME_FROM_LOOP + whilestmt ::= setup_loop testexpr l_stmts_opt + come_froms JUMP_BACK come_froms POP_BLOCK COME_FROM_LOOP + + # 3.6 due to jump optimization, we sometimes add RETURN_END_IF where + # RETURN_VALUE is meant. Specifcally this can happen in + # ifelsestmt -> ...else_suite _. suite_stmts... (last) stmt + return ::= ret_expr RETURN_END_IF + return ::= ret_expr RETURN_VALUE COME_FROM + return_stmt_lambda ::= ret_expr RETURN_VALUE_LAMBDA COME_FROM + + # A COME_FROM is dropped off because of JUMP-to-JUMP optimization + and ::= expr jmp_false expr + and ::= expr jmp_false expr jmp_false + + jf_cf ::= JUMP_FORWARD COME_FROM + cf_jf_else ::= come_froms JUMP_FORWARD ELSE + + conditional ::= expr jmp_false expr jf_cf expr COME_FROM + + async_for_stmt ::= setup_loop expr + GET_AITER + LOAD_CONST YIELD_FROM SETUP_EXCEPT GET_ANEXT LOAD_CONST + YIELD_FROM + store + POP_BLOCK JUMP_FORWARD COME_FROM_EXCEPT DUP_TOP + LOAD_GLOBAL COMPARE_OP POP_JUMP_IF_FALSE + POP_TOP POP_TOP POP_TOP POP_EXCEPT POP_BLOCK + JUMP_ABSOLUTE END_FINALLY COME_FROM + for_block POP_BLOCK + COME_FROM_LOOP + + # Adds a COME_FROM_ASYNC_WITH over 3.5 + # FIXME: remove corresponding rule for 3.5? + + except_suite ::= c_stmts_opt COME_FROM POP_EXCEPT jump_except COME_FROM + + jb_cfs ::= come_from_opt JUMP_BACK come_froms + ifelsestmtl ::= testexpr c_stmts_opt jb_cfs else_suitel + ifelsestmtl ::= testexpr c_stmts_opt cf_jf_else else_suitel + + # In 3.6+, A sequence of statements ending in a RETURN can cause + # JUMP_FORWARD END_FINALLY to be omitted from try middle + + except_return ::= POP_TOP POP_TOP POP_TOP returns + except_handler ::= JUMP_FORWARD COME_FROM_EXCEPT except_return + + # Try middle following a returns + except_handler36 ::= COME_FROM_EXCEPT except_stmts END_FINALLY + + stmt ::= try_except36 + try_except36 ::= SETUP_EXCEPT returns except_handler36 + opt_come_from_except + try_except36 ::= SETUP_EXCEPT suite_stmts + try_except36 ::= SETUP_EXCEPT suite_stmts_opt POP_BLOCK + except_handler36 come_from_opt + + # 3.6 omits END_FINALLY sometimes + except_handler36 ::= COME_FROM_EXCEPT except_stmts + except_handler36 ::= JUMP_FORWARD COME_FROM_EXCEPT except_stmts + except_handler ::= jmp_abs COME_FROM_EXCEPT except_stmts + + stmt ::= tryfinally36 + tryfinally36 ::= SETUP_FINALLY returns + COME_FROM_FINALLY suite_stmts + tryfinally36 ::= SETUP_FINALLY returns + COME_FROM_FINALLY suite_stmts_opt END_FINALLY + except_suite_finalize ::= SETUP_FINALLY returns + COME_FROM_FINALLY suite_stmts_opt END_FINALLY _jump + + stmt ::= tryfinally_return_stmt + tryfinally_return_stmt ::= SETUP_FINALLY suite_stmts_opt POP_BLOCK LOAD_CONST + COME_FROM_FINALLY + + compare_chained2 ::= expr COMPARE_OP come_froms JUMP_FORWARD """ - def p_expr3(self, args): + def p_37misc(self, args): """ - expr ::= conditionalnot - conditionalnot ::= expr jmp_true expr jump_forward_else expr COME_FROM + # long except clauses in a loop can sometimes cause a JUMP_BACK to turn into a + # JUMP_FORWARD to a JUMP_BACK. And when this happens there is an additional + # ELSE added to the except_suite. With better flow control perhaps we can + # sort this out better. + except_suite ::= c_stmts_opt POP_EXCEPT jump_except ELSE - # a JUMP_FORWARD to another JUMP_FORWARD can get turned into - # a JUMP_ABSOLUTE with no COME_FROM - conditional ::= expr jmp_false expr jump_absolute_else expr - - # if_expr_true are for conditions which always evaluate true - # There is dead or non-optional remnants of the condition code though, - # and we use that to match on to reconstruct the source more accurately - expr ::= if_expr_true - if_expr_true ::= expr JUMP_FORWARD expr COME_FROM + # FIXME: the below is to work around test_grammar expecting a "call" to be + # on the LHS because it is also somewhere on in a rule. + call ::= expr CALL_METHOD_0 """ def customize_grammar_rules(self, tokens, customize): diff --git a/uncompyle6/parsers/parse37base.py b/uncompyle6/parsers/parse37base.py index 0305a821..7481c303 100644 --- a/uncompyle6/parsers/parse37base.py +++ b/uncompyle6/parsers/parse37base.py @@ -1,4 +1,4 @@ -# Copyright (c) 2016-2017, 2019 Rocky Bernstein +# Copyright (c) 2016-2017, 2019-2020 Rocky Bernstein """ Python 3.7 base code. We keep non-custom-generated grammar rules out of this file. """ @@ -581,6 +581,18 @@ class Python37BaseParser(PythonParser): elif opname == "LOAD_LISTCOMP": self.add_unique_rule("expr ::= listcomp", opname, token.attr, customize) custom_ops_processed.add(opname) + elif opname == "LOAD_NAME": + if token.attr == "__annotations__" and "SETUP_ANNOTATIONS" in self.seen_ops: + token.kind = "LOAD_ANNOTATION" + self.addRule( + """ + stmt ::= SETUP_ANNOTATIONS + stmt ::= ann_assign + ann_assign ::= expr LOAD_ANNOTATION LOAD_STR STORE_SUBSCR + """, + nop_func, + ) + pass elif opname == "LOAD_SETCOMP": # Should this be generalized and put under MAKE_FUNCTION? if has_get_iter_call_function1: @@ -1103,8 +1115,13 @@ class Python37BaseParser(PythonParser): # FIXME: This is a cheap test. Should we do something with an AST like we # do with "and"? # "or"s with constants like this will have "COME_FROM" at the end - return tokens[last] in ("LOAD_ASSERT", "LOAD_STR", "LOAD_CODE", "LOAD_CONST", - "RAISE_VARARGS_1") + return tokens[last] in ( + "LOAD_ASSERT", + "LOAD_STR", + "LOAD_CODE", + "LOAD_CONST", + "RAISE_VARARGS_1", + ) elif lhs == "while1elsestmt": if last == n: @@ -1143,7 +1160,7 @@ class Python37BaseParser(PythonParser): for i in range(cfl - 1, first, -1): if tokens[i] != "POP_BLOCK": break - if tokens[i].kind not in ("JUMP_BACK", "RETURN_VALUE"): + if tokens[i].kind not in ("JUMP_BACK", "RETURN_VALUE", "RAISE_VARARGS_1"): if not tokens[i].kind.startswith("COME_FROM"): return True @@ -1156,9 +1173,8 @@ class Python37BaseParser(PythonParser): last -= 1 offset = tokens[last].off2int() assert tokens[first] == "SETUP_LOOP" - if offset != tokens[first].attr: - return True - return False + # SETUP_LOOP location must jump either to the last token or the token after the last one + return tokens[first].attr not in (offset, offset + 2) elif lhs == "_ifstmts_jump" and len(rule[1]) > 1 and ast: come_froms = ast[-1] # Make sure all of the "come froms" offset at the @@ -1192,6 +1208,10 @@ class Python37BaseParser(PythonParser): return False if isinstance(come_froms, Token): + if tokens[pop_jump_index].attr < tokens[pop_jump_index].offset and ast[0] != "pass": + # This is a jump backwards to a loop. All bets are off here when there the + # unless statement is "pass" which has no instructions associated with it. + return False return ( come_froms.attr is not None and tokens[pop_jump_index].offset > come_froms.attr @@ -1210,7 +1230,7 @@ class Python37BaseParser(PythonParser): if last == n: last -= 1 pass - if (tokens[last].attr and isinstance(tokens[last].attr, int)): + if tokens[last].attr and isinstance(tokens[last].attr, int): return tokens[first].offset < tokens[last].attr pass @@ -1225,7 +1245,14 @@ class Python37BaseParser(PythonParser): for i in range(first, l): t = tokens[i] if t.kind == "POP_JUMP_IF_FALSE": - if t.attr > last_offset: + pjif_target = t.attr + if pjif_target > last_offset: + # In come cases, where we have long bytecode, a + # "POP_JUMP_IF_FALSE" offset might be too + # large for the instruction; so instead it + # jumps to a JUMP_FORWARD. Allow that here. + if tokens[l] == "JUMP_FORWARD": + return tokens[l].attr != pjif_target return True pass pass @@ -1244,7 +1271,11 @@ class Python37BaseParser(PythonParser): if last == n: last -= 1 jmp_target = test[1][0].attr - if tokens[first].off2int() <= jmp_target < tokens[last].off2int(): + if ( + tokens[first].off2int() + <= jmp_target + < tokens[last].off2int() + ): return True # jmp_target less than tokens[first] is okay - is to a loop # jmp_target equal tokens[last] is also okay: normal non-optimized non-loop jump @@ -1279,7 +1310,11 @@ class Python37BaseParser(PythonParser): # jmp_target less than tokens[first] is okay - is to a loop # jmp_target equal tokens[last] is also okay: normal non-optimized non-loop jump - if (last + 1) < n and tokens[last - 1] != "JUMP_BACK" and tokens[last + 1] == "COME_FROM_LOOP": + if ( + (last + 1) < n + and tokens[last - 1] != "JUMP_BACK" + and tokens[last + 1] == "COME_FROM_LOOP" + ): # iflastsmtl is not at the end of a loop, but jumped outside of loop. No good. # FIXME: check that tokens[last] == "POP_BLOCK"? Or allow for it not to appear? return True @@ -1323,6 +1358,36 @@ class Python37BaseParser(PythonParser): "_come_froms", ), ), + ( + "ifelsestmt", + ( + "testexpr", + "c_stmts_opt", + "jump_forward_else", + "else_suite", + '\\e__come_froms' + ), + ), + ( + "ifelsestmt", + ( + "testexpr", + "c_stmts_opt", + "jf_cfs", + "else_suite", + '\\e_opt_come_from_except', + ), + ), + ( + "ifelsestmt", + ( + "testexpr", + "c_stmts_opt", + "come_froms", + "else_suite", + 'come_froms', + ), + ), ( "ifelsestmt", ( @@ -1345,7 +1410,8 @@ class Python37BaseParser(PythonParser): if come_froms == "opt_come_from_except" and len(come_froms) > 0: come_froms = come_froms[0] if not isinstance(come_froms, Token): - return tokens[first].offset > come_froms[-1].attr + if len(come_froms): + return tokens[first].offset > come_froms[-1].attr elif tokens[first].offset > come_froms.attr: return True @@ -1363,16 +1429,34 @@ class Python37BaseParser(PythonParser): # Check that the condition portion of the "if" # jumps to the "else" part. - # Compare with parse30.py of uncompyle6 if testexpr[0] in ("testtrue", "testfalse"): test = testexpr[0] + + else_suite = ast[3] + assert else_suite == "else_suite" + if len(test) > 1 and test[1].kind.startswith("jmp_"): if last == n: last -= 1 jmp = test[1] jmp_target = jmp[0].attr + + # FIXME: the jump inside "else" check below should be added. + # + # add this until we can find out what's wrong with + # not being able to parse: + # if a and b or c: + # x = 1 + # else: + # x = 2 + + # FIXME: add this + # if jmp_target < else_suite.first_child().off2int(): + # return True + if tokens[first].off2int() > jmp_target: return True + return (jmp_target > tokens[last].off2int()) and tokens[ last ] != "JUMP_FORWARD" diff --git a/uncompyle6/semantics/customize37.py b/uncompyle6/semantics/customize37.py index a73cfcce..c50927a2 100644 --- a/uncompyle6/semantics/customize37.py +++ b/uncompyle6/semantics/customize37.py @@ -1,4 +1,4 @@ -# Copyright (c) 2019 by Rocky Bernstein +# Copyright (c) 2019-2020 by Rocky Bernstein # # This program is free software: you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by @@ -22,70 +22,118 @@ def customize_for_version37(self, version): # Python 3.7+ changes ####################### - PRECEDENCE['attribute37'] = 2 - PRECEDENCE['if_exp_37a'] = 28 - PRECEDENCE['if_exp_37b'] = 28 + PRECEDENCE["attribute37"] = 2 + PRECEDENCE["call_ex"] = 1 + PRECEDENCE["call_ex_kw"] = 1 + PRECEDENCE["call_ex_kw2"] = 1 + PRECEDENCE["call_ex_kw3"] = 1 + PRECEDENCE["call_ex_kw4"] = 1 + PRECEDENCE["call_kw"] = 0 + PRECEDENCE["call_kw36"] = 1 + PRECEDENCE["formatted_value1"] = 100 + PRECEDENCE["if_exp_37a"] = 28 + PRECEDENCE["if_exp_37b"] = 28 + PRECEDENCE["unmap_dict"] = 0 - TABLE_DIRECT.update({ - 'and_not': ( '%c and not %c', - (0, 'expr'), (2, 'expr') ), - 'async_forelse_stmt': ( - '%|async for %c in %c:\n%+%c%-%|else:\n%+%c%-\n\n', - (7, 'store'), (1, 'expr'), (17, 'for_block'), (25, 'else_suite') ), - 'async_for_stmt': ( - '%|async for %c in %c:\n%+%c%-\n\n', - (7, 'store'), (1, 'expr'), (17, 'for_block')), - 'async_for_stmt37': ( - '%|async for %c in %c:\n%+%c%-%-\n\n', - (7, 'store'), (1, 'expr'), (16, 'for_block') ), - 'attribute37': ( '%c.%[1]{pattr}', 0 ), - 'compare_chained1a_37': ( - ' %[3]{pattr.replace("-", " ")} %p %p', - (0, 19), (-4, 19)), - 'compare_chained1_false_37': ( - ' %[3]{pattr.replace("-", " ")} %p %p', - (0, 19), (-4, 19)), - 'compare_chained2_false_37': ( - ' %[3]{pattr.replace("-", " ")} %p %p', - (0, 19), (-5, 19)), - 'compare_chained1b_37': ( - ' %[3]{pattr.replace("-", " ")} %p %p', - (0, 19), (-4, 19)), - 'compare_chained1c_37': ( - ' %[3]{pattr.replace("-", " ")} %p %p', - (0, 19), (-2, 19)), - 'compare_chained2a_37': ( - '%[1]{pattr.replace("-", " ")} %p', - (0, 19) ), - 'compare_chained2b_37': ( - '%[1]{pattr.replace("-", " ")} %p', - (0, 19) ), - 'compare_chained2a_false_37': ( - '%[1]{pattr.replace("-", " ")} %p', - (0, 19 ) ), - 'compare_chained2c_37': ( - '%[3]{pattr.replace("-", " ")} %p %p', (0, 19), (6, 19) ), - 'if_exp_37a': ( '%p if %p else %p', (1, 'expr', 27), (0, 27), (4, 'expr', 27) ), - 'if_exp_37b': ( '%p if %p else %p', (2, 'expr', 27), (0, 'expr', 27), (5, 'expr', 27) ), - 'list_if37': ( " if %p%c", (0, 27), 1 ), - 'testfalse_not_or': ( "not %c or %c", - (0, "expr"), - (2, "expr") ), - 'testfalse_not_and': ( "not (%c)", 0 ), - - }) - - def n_import_from(node): - relative_path_index = 0 - if node[relative_path_index].pattr > 0: - node[2].pattr = ("." * node[relative_path_index].pattr) + node[2].pattr - if isinstance(node[1].pattr, tuple): - imports = node[1].pattr - for pattr in imports: - node[1].pattr = pattr - self.default(node) - return - self.default(node) - - self.n_import_from = n_import_from - self.n_import_from_star = n_import_from + TABLE_DIRECT.update( + { + "ann_assign": ( + "%|%[2]{attr}: %c\n", 0, + ), + "ann_assign_init": ( + "%|%[2]{attr}: %c = %c\n", 0, 1, + ), + "async_for_stmt": ( + "%|async for %c in %c:\n%+%c%-\n\n", + (7, "store"), + (1, "expr"), + (17, "for_block"), + ), + "async_for_stmt36": ( + "%|async for %c in %c:\n%+%c%-%-\n\n", + (9, "store"), + (1, "expr"), + (18, "for_block"), + ), + "async_for_stmt37": ( + "%|async for %c in %c:\n%+%c%-%-\n\n", + (7, "store"), + (1, "expr"), + (16, "for_block"), + ), + "and_not": ("%c and not %c", (0, "expr"), (2, "expr")), + "async_with_stmt": ("%|async with %c:\n%+%c%-", (0, "expr"), 7), + "async_with_as_stmt": ( + "%|async with %c as %c:\n%+%c%-", + (0, "expr"), + (6, "store"), + 7, + ), + "async_forelse_stmt": ( + "%|async for %c in %c:\n%+%c%-%|else:\n%+%c%-\n\n", + (7, "store"), + (1, "expr"), + (17, "for_block"), + (25, "else_suite"), + ), + "attribute37": ("%c.%[1]{pattr}", 0), + "await_expr": ("await %c", 0), + "await_stmt": ("%|%c\n", 0), + "call_ex": ("%c(%p)", (0, "expr"), (1, 100)), + "compare_chained1a_37": ( + ' %[3]{pattr.replace("-", " ")} %p %p', + (0, 19), + (-4, 19), + ), + "compare_chained1_false_37": ( + ' %[3]{pattr.replace("-", " ")} %p %p', + (0, 19), + (-4, 19), + ), + "compare_chained2_false_37": ( + ' %[3]{pattr.replace("-", " ")} %p %p', + (0, 19), + (-5, 19), + ), + "compare_chained1b_37": ( + ' %[3]{pattr.replace("-", " ")} %p %p', + (0, 19), + (-4, 19), + ), + "compare_chained1c_37": ( + ' %[3]{pattr.replace("-", " ")} %p %p', + (0, 19), + (-2, 19), + ), + "compare_chained2a_37": ('%[1]{pattr.replace("-", " ")} %p', (0, 19)), + "compare_chained2b_37": ('%[1]{pattr.replace("-", " ")} %p', (0, 19)), + "compare_chained2a_false_37": ('%[1]{pattr.replace("-", " ")} %p', (0, 19)), + "compare_chained2c_37": ( + '%[3]{pattr.replace("-", " ")} %p %p', + (0, 19), + (6, 19), + ), + "except_return": ("%|except:\n%+%c%-", 3), + "if_exp_37a": ( + "%p if %p else %p", + (1, "expr", 27), + (0, 27), + (4, "expr", 27), + ), + "if_exp_37b": ( + "%p if %p else %p", + (2, "expr", 27), + (0, "expr", 27), + (5, "expr", 27), + ), + "ifstmtl": ("%|if %c:\n%+%c%-", (0, "testexpr"), (1, "_ifstmts_jumpl")), + 'list_if37': ( " if %p%c", (0, 27), 1 ), + "testfalse_not_or": ("not %c or %c", (0, "expr"), (2, "expr")), + "testfalse_not_and": ("not (%c)", 0), + "try_except36": ("%|try:\n%+%c%-%c\n\n", 1, -2), + "tryfinally36": ("%|try:\n%+%c%-%|finally:\n%+%c%-\n\n", (1, "returns"), 3), + "unmap_dict": ("{**%C}", (0, -1, ", **")), + "unpack_list": ("*%c", (0, "list")), + "yield_from": ("yield from %c", (0, "expr")), + } + ) diff --git a/uncompyle6/semantics/transform.py b/uncompyle6/semantics/transform.py index 7edde504..07ac9c2a 100644 --- a/uncompyle6/semantics/transform.py +++ b/uncompyle6/semantics/transform.py @@ -1,4 +1,4 @@ -# Copyright (c) 2019 by Rocky Bernstein +# Copyright (c) 2019-2020 by Rocky Bernstein # This program is free software: you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by @@ -296,6 +296,28 @@ class TreeTransform(GenericASTTraversal, object): list_for_node.transformed_by = ("n_list_for",) return list_for_node + def n_stmts(self, node): + if node.first_child() == "SETUP_ANNOTATIONS": + prev = node[0][0][0] + new_stmts = [node[0]] + for i, sstmt in enumerate(node[1:]): + ann_assign = sstmt[0][0] + if (sstmt[0] == "stmt" and ann_assign == "ann_assign" and prev == "assign"): + annotate_var = ann_assign[-2] + if annotate_var.attr == prev[-1][0].attr: + del new_stmts[-1] + sstmt[0][0] = SyntaxTree( + "ann_assign_init", + [ann_assign[0], prev[0], annotate_var]) + sstmt[0][0].transformed_by="n_stmts" + pass + pass + new_stmts.append(sstmt) + prev = ann_assign + pass + node.data = new_stmts + return node + def traverse(self, node, is_lambda=False): node = self.preorder(node) return node