Start 3.6+ var type annotations and decompyle3 merge...

Although overall an improvement, some new breakage
has occurred and should be fixed.

pytest/test_grammar.py

@@ -35,7 +35,7 @@ def test_grammar():
 
     expect_right_recursive = set([("designList", ("store", "DUP_TOP", "designList"))])
 
-    if PYTHON_VERSION <= 3.7:
+    if PYTHON_VERSION <= 3.6:
         unused_rhs.add("call")
 
     if PYTHON_VERSION > 2.6:

uncompyle6/parsers/parse37.py

@@ -1,4 +1,4 @@
-#  Copyright (c) 2017-2019 Rocky Bernstein
+#  Copyright (c) 2017-2020 Rocky Bernstein
 #
 #  This program is free software: you can redistribute it and/or modify
 #  it under the terms of the GNU General Public License as published by
@@ -22,7 +22,6 @@ from spark_parser import DEFAULT_DEBUG as PARSER_DEFAULT_DEBUG
 from uncompyle6.parsers.parse37base import Python37BaseParser
 
 class Python37Parser(Python37BaseParser):
-
     def __init__(self, debug_parser=PARSER_DEFAULT_DEBUG):
         super(Python37Parser, self).__init__(debug_parser)
         self.customized = {}
@@ -328,6 +327,26 @@ class Python37Parser(Python37BaseParser):
         attributes   ::= LOAD_ATTR+
         """
 
+    def p_import37(self, args):
+        """
+        stmt     ::= import37
+
+        # Where does the POP_TOP really belong?
+        import37 ::= import POP_TOP
+
+        attributes ::= IMPORT_FROM ROT_TWO POP_TOP IMPORT_FROM
+        attributes ::= attributes ROT_TWO POP_TOP IMPORT_FROM
+
+        # The 3.7base scanner adds IMPORT_NAME_ATTR
+        alias ::= IMPORT_NAME_ATTR IMPORT_FROM store
+        alias ::= IMPORT_NAME_ATTR attributes store
+        alias ::= IMPORT_NAME_ATTR store
+        import_from ::= LOAD_CONST LOAD_CONST IMPORT_NAME_ATTR importlist POP_TOP
+
+        expr          ::= attribute37
+        attribute37   ::= expr LOAD_METHOD
+        """
+
     def p_list_comprehension(self, args):
         """
         expr ::= list_comp
@@ -501,108 +520,12 @@ class Python37Parser(Python37BaseParser):
         iflaststmt ::= testexpr c_stmts_opt JUMP_FORWARD
         """
 
-    def p_36misc(self, args):
+    def p_37async(self, args):
         """
-        sstmt ::= sstmt RETURN_LAST
-
-        # 3.6 redoes how return_closure works. FIXME: Isolate to LOAD_CLOSURE
-        return_closure   ::= LOAD_CLOSURE DUP_TOP STORE_NAME RETURN_VALUE RETURN_LAST
-
-        for_block       ::= l_stmts_opt come_from_loops JUMP_BACK
-        come_from_loops ::= COME_FROM_LOOP*
-
-        whilestmt       ::= setup_loop testexpr l_stmts_opt
-                            JUMP_BACK come_froms POP_BLOCK COME_FROM_LOOP
-        whilestmt       ::= setup_loop testexpr l_stmts_opt
-                            come_froms JUMP_BACK come_froms POP_BLOCK COME_FROM_LOOP
-
-        # 3.6 due to jump optimization, we sometimes add RETURN_END_IF where
-        # RETURN_VALUE is meant. Specifcally this can happen in
-        # ifelsestmt -> ...else_suite _. suite_stmts... (last) stmt
-        return ::= ret_expr RETURN_END_IF
-        return ::= ret_expr RETURN_VALUE COME_FROM
-        return_stmt_lambda ::= ret_expr RETURN_VALUE_LAMBDA COME_FROM
-
-        # A COME_FROM is dropped off because of JUMP-to-JUMP optimization
-        and  ::= expr jmp_false expr
-        and  ::= expr jmp_false expr jmp_false
-
-        jf_cf       ::= JUMP_FORWARD COME_FROM
-        cf_jf_else  ::= come_froms JUMP_FORWARD ELSE
-
-        conditional ::= expr jmp_false expr jf_cf expr COME_FROM
-
-        async_for_stmt     ::= setup_loop expr
-                               GET_AITER
-                               LOAD_CONST YIELD_FROM SETUP_EXCEPT GET_ANEXT LOAD_CONST
-                               YIELD_FROM
-                               store
-                               POP_BLOCK JUMP_FORWARD COME_FROM_EXCEPT DUP_TOP
-                               LOAD_GLOBAL COMPARE_OP POP_JUMP_IF_FALSE
-                               POP_TOP POP_TOP POP_TOP POP_EXCEPT POP_BLOCK
-                               JUMP_ABSOLUTE END_FINALLY COME_FROM
-                               for_block POP_BLOCK
-                               COME_FROM_LOOP
-
-        # Adds a COME_FROM_ASYNC_WITH over 3.5
-        # FIXME: remove corresponding rule for 3.5?
-
-        except_suite ::= c_stmts_opt COME_FROM POP_EXCEPT jump_except COME_FROM
-
-        jb_cfs      ::= come_from_opt JUMP_BACK come_froms
-        ifelsestmtl ::= testexpr c_stmts_opt jb_cfs else_suitel
-        ifelsestmtl ::= testexpr c_stmts_opt cf_jf_else else_suitel
-
-        # In 3.6+, A sequence of statements ending in a RETURN can cause
-        # JUMP_FORWARD END_FINALLY to be omitted from try middle
-
-        except_return    ::= POP_TOP POP_TOP POP_TOP returns
-        except_handler   ::= JUMP_FORWARD COME_FROM_EXCEPT except_return
-
-        # Try middle following a returns
-        except_handler36 ::= COME_FROM_EXCEPT except_stmts END_FINALLY
-
-        stmt             ::= try_except36
-        try_except36     ::= SETUP_EXCEPT returns except_handler36
-                             opt_come_from_except
-        try_except36     ::= SETUP_EXCEPT suite_stmts
-        try_except36     ::= SETUP_EXCEPT suite_stmts_opt POP_BLOCK
-                             except_handler36 come_from_opt
-
-        # 3.6 omits END_FINALLY sometimes
-        except_handler36 ::= COME_FROM_EXCEPT except_stmts
-        except_handler36 ::= JUMP_FORWARD COME_FROM_EXCEPT except_stmts
-        except_handler   ::= jmp_abs COME_FROM_EXCEPT except_stmts
-
-        stmt             ::= tryfinally36
-        tryfinally36     ::= SETUP_FINALLY returns
-                             COME_FROM_FINALLY suite_stmts
-        tryfinally36     ::= SETUP_FINALLY returns
-                             COME_FROM_FINALLY suite_stmts_opt END_FINALLY
-        except_suite_finalize ::= SETUP_FINALLY returns
-                                  COME_FROM_FINALLY suite_stmts_opt END_FINALLY _jump
-
-        stmt ::= tryfinally_return_stmt
-        tryfinally_return_stmt ::= SETUP_FINALLY suite_stmts_opt POP_BLOCK LOAD_CONST
-                                   COME_FROM_FINALLY
-
-        compare_chained2 ::= expr COMPARE_OP come_froms JUMP_FORWARD
-        """
-
-    def p_37misc(self, args):
-        """
-        stmt     ::= import37
         stmt     ::= async_for_stmt37
         stmt     ::= async_for_stmt
         stmt     ::= async_forelse_stmt
 
-        # Where does the POP_TOP really belong?
-        import37 ::= import POP_TOP
-        alias ::= IMPORT_NAME_ATTR IMPORT_FROM store
-        alias ::= IMPORT_NAME_ATTR attributes store
-        alias ::= IMPORT_NAME_ATTR store
-        import_from ::= LOAD_CONST LOAD_CONST IMPORT_NAME_ATTR importlist POP_TOP
-
         async_for_stmt     ::= setup_loop expr
                                GET_AITER
                                SETUP_EXCEPT GET_ANEXT LOAD_CONST
@@ -641,19 +564,10 @@ class Python37Parser(Python37BaseParser):
                                COME_FROM
                                POP_TOP POP_TOP POP_TOP POP_EXCEPT POP_TOP POP_BLOCK
                                else_suite COME_FROM_LOOP
+        """
 
-        attributes ::= IMPORT_FROM ROT_TWO POP_TOP IMPORT_FROM
+    def p_37chained(self, args):
-        attributes ::= attributes ROT_TWO POP_TOP IMPORT_FROM
+        """
-
-        attribute37   ::= expr LOAD_METHOD
-        expr          ::= attribute37
-
-        # long except clauses in a loop can sometimes cause a JUMP_BACK to turn into a
-        # JUMP_FORWARD to a JUMP_BACK. And when this happens there is an additional
-        # ELSE added to the except_suite. With better flow control perhaps we can
-        # sort this out better.
-        except_suite ::= c_stmts_opt POP_EXCEPT jump_except ELSE
-
         testtrue         ::= compare_chained37
         testfalse        ::= compare_chained37_false
 
@@ -694,7 +608,10 @@ class Python37Parser(Python37BaseParser):
                                        compare_chained2a_false_37 ELSE
         compare_chained2c_37       ::= expr DUP_TOP ROT_THREE COMPARE_OP come_from_opt POP_JUMP_IF_FALSE
                                        compare_chained2a_false_37
+        """
 
+    def p_37conditionals(self, args):
+        """
         jf_cfs                     ::= JUMP_FORWARD _come_froms
         ifelsestmt                 ::= testexpr c_stmts_opt jf_cfs else_suite opt_come_from_except
 
@@ -768,6 +685,28 @@ class Python37Parser(Python37BaseParser):
         comp_iter     ::= comp_body
         """
 
+    def p_expr3(self, args):
+        """
+        expr           ::= conditionalnot
+        conditionalnot ::= expr jmp_true  expr jump_forward_else expr COME_FROM
+
+        # a JUMP_FORWARD to another JUMP_FORWARD can get turned into
+        # a JUMP_ABSOLUTE with no COME_FROM
+        conditional    ::= expr jmp_false expr jump_absolute_else expr
+
+        # if_expr_true are for conditions which always evaluate true
+        # There is dead or non-optional remnants of the condition code though,
+        # and we use that to match on to reconstruct the source more accurately
+        expr           ::= if_expr_true
+        if_expr_true   ::= expr JUMP_FORWARD expr COME_FROM
+        """
+
+    def p_generator_exp3(self, args):
+        """
+        load_genexpr ::= LOAD_GENEXPR
+        load_genexpr ::= BUILD_TUPLE_1 LOAD_GENEXPR LOAD_STR
+        """
+
     def p_grammar(self, args):
         """
         sstmt ::= stmt
@@ -826,6 +765,7 @@ class Python37Parser(Python37BaseParser):
         iflaststmtl ::= testexpr c_stmts JUMP_BACK POP_BLOCK
 
         # These are used to keep parse tree indices the same
+        jump_forward_else  ::= JUMP_FORWARD
         jump_forward_else  ::= JUMP_FORWARD ELSE
         jump_forward_else  ::= JUMP_FORWARD COME_FROM
         jump_absolute_else ::= JUMP_ABSOLUTE ELSE
@@ -950,6 +890,7 @@ class Python37Parser(Python37BaseParser):
         ret_cond ::= expr POP_JUMP_IF_FALSE expr RETURN_END_IF COME_FROM ret_expr_or_cond
 
         jitop_come_from ::= JUMP_IF_TRUE_OR_POP COME_FROM
+        jifop_come_from ::= JUMP_IF_FALSE_OR_POP COME_FROM
         or        ::= and jitop_come_from expr COME_FROM
         or        ::= expr JUMP_IF_TRUE_OR_POP expr COME_FROM
         or        ::= expr JUMP_IF_TRUE expr COME_FROM
@@ -963,11 +904,13 @@ class Python37Parser(Python37BaseParser):
         testfalse ::= or jmp_false COME_FROM
         or        ::= expr jmp_true expr
 
-
+        and  ::= expr JUMP_IF_FALSE_OR_POP expr come_from_opt
-
+        and  ::= expr jifop_come_from expr
-        and  ::= expr JUMP_IF_FALSE_OR_POP expr COME_FROM
         and  ::= expr JUMP_IF_FALSE expr COME_FROM
 
+        pjit_come_from ::= POP_JUMP_IF_TRUE COME_FROM
+        or  ::= expr pjit_come_from expr
+
         ## FIXME: Is the below needed or is it covered above??
         and ::= expr jmp_false expr COME_FROM
         or  ::= expr jmp_true  expr COME_FROM
@@ -983,6 +926,8 @@ class Python37Parser(Python37BaseParser):
         """
         stmt               ::= if_expr_lambda
         stmt               ::= conditional_not_lambda
+        stmt               ::= ifstmtl
+
         if_expr_lambda     ::= expr jmp_false expr return_if_lambda
                                return_stmt_lambda LAMBDA_MARKER
         conditional_not_lambda
@@ -997,6 +942,10 @@ class Python37Parser(Python37BaseParser):
 
         stmt ::= whileTruestmt
         ifelsestmt ::= testexpr c_stmts_opt JUMP_FORWARD else_suite _come_froms
+
+        _ifstmts_jumpl     ::= c_stmts JUMP_BACK
+        _ifstmts_jumpl     ::= _ifstmts_jump
+        ifstmtl            ::= testexpr _ifstmts_jumpl
         """
 
     def p_loop_stmt3(self, args):
@@ -1019,12 +968,20 @@ class Python37Parser(Python37BaseParser):
         whilestmt         ::= setup_loop testexpr l_stmts_opt COME_FROM JUMP_BACK POP_BLOCK
                               COME_FROM_LOOP
 
+
         whilestmt         ::= setup_loop testexpr l_stmts_opt JUMP_BACK POP_BLOCK
                               COME_FROM_LOOP
 
         whilestmt         ::= setup_loop testexpr returns          POP_BLOCK
                               COME_FROM_LOOP
 
+        # We can be missing a COME_FROM_LOOP if the "while" statement is nested inside an if/else
+        # so after the POP_BLOCK we have a JUMP_FORWARD which forms the "else" portion of the "if"
+        # This is undoubtedly some sort of JUMP optimization going on.
+
+        whilestmt         ::= setup_loop testexpr l_stmts_opt JUMP_BACK come_froms
+                              POP_BLOCK
+
         while1elsestmt    ::= setup_loop          l_stmts     JUMP_BACK
                               else_suitel
 
@@ -1032,11 +989,12 @@ class Python37Parser(Python37BaseParser):
                               else_suitel COME_FROM_LOOP
 
         whileTruestmt     ::= setup_loop l_stmts_opt          JUMP_BACK POP_BLOCK
-                              COME_FROM_LOOP
+                              _come_froms
 
         # FIXME: Python 3.? starts adding branch optimization? Put this starting there.
 
         while1stmt        ::= setup_loop l_stmts COME_FROM_LOOP
+        while1stmt        ::= setup_loop l_stmts COME_FROM_LOOP JUMP_BACK POP_BLOCK COME_FROM_LOOP
         while1stmt        ::= setup_loop l_stmts COME_FROM JUMP_BACK COME_FROM_LOOP
 
         while1elsestmt    ::= setup_loop l_stmts JUMP_BACK
@@ -1047,26 +1005,105 @@ class Python37Parser(Python37BaseParser):
                               COME_FROM_LOOP
         """
 
-    def p_generator_exp3(self, args):
+    def p_36misc(self, args):
         """
-        load_genexpr ::= LOAD_GENEXPR
+        sstmt ::= sstmt RETURN_LAST
-        load_genexpr ::= BUILD_TUPLE_1 LOAD_GENEXPR LOAD_STR
+
+        # 3.6 redoes how return_closure works. FIXME: Isolate to LOAD_CLOSURE
+        return_closure   ::= LOAD_CLOSURE DUP_TOP STORE_NAME RETURN_VALUE RETURN_LAST
+
+        for_block       ::= l_stmts_opt come_from_loops JUMP_BACK
+        come_from_loops ::= COME_FROM_LOOP*
+
+        whilestmt       ::= setup_loop testexpr l_stmts_opt
+                            JUMP_BACK come_froms POP_BLOCK COME_FROM_LOOP
+        whilestmt       ::= setup_loop testexpr l_stmts_opt
+                            come_froms JUMP_BACK come_froms POP_BLOCK COME_FROM_LOOP
+
+        # 3.6 due to jump optimization, we sometimes add RETURN_END_IF where
+        # RETURN_VALUE is meant. Specifcally this can happen in
+        # ifelsestmt -> ...else_suite _. suite_stmts... (last) stmt
+        return ::= ret_expr RETURN_END_IF
+        return ::= ret_expr RETURN_VALUE COME_FROM
+        return_stmt_lambda ::= ret_expr RETURN_VALUE_LAMBDA COME_FROM
+
+        # A COME_FROM is dropped off because of JUMP-to-JUMP optimization
+        and  ::= expr jmp_false expr
+        and  ::= expr jmp_false expr jmp_false
+
+        jf_cf       ::= JUMP_FORWARD COME_FROM
+        cf_jf_else  ::= come_froms JUMP_FORWARD ELSE
+
+        conditional ::= expr jmp_false expr jf_cf expr COME_FROM
+
+        async_for_stmt     ::= setup_loop expr
+                               GET_AITER
+                               LOAD_CONST YIELD_FROM SETUP_EXCEPT GET_ANEXT LOAD_CONST
+                               YIELD_FROM
+                               store
+                               POP_BLOCK JUMP_FORWARD COME_FROM_EXCEPT DUP_TOP
+                               LOAD_GLOBAL COMPARE_OP POP_JUMP_IF_FALSE
+                               POP_TOP POP_TOP POP_TOP POP_EXCEPT POP_BLOCK
+                               JUMP_ABSOLUTE END_FINALLY COME_FROM
+                               for_block POP_BLOCK
+                               COME_FROM_LOOP
+
+        # Adds a COME_FROM_ASYNC_WITH over 3.5
+        # FIXME: remove corresponding rule for 3.5?
+
+        except_suite ::= c_stmts_opt COME_FROM POP_EXCEPT jump_except COME_FROM
+
+        jb_cfs      ::= come_from_opt JUMP_BACK come_froms
+        ifelsestmtl ::= testexpr c_stmts_opt jb_cfs else_suitel
+        ifelsestmtl ::= testexpr c_stmts_opt cf_jf_else else_suitel
+
+        # In 3.6+, A sequence of statements ending in a RETURN can cause
+        # JUMP_FORWARD END_FINALLY to be omitted from try middle
+
+        except_return    ::= POP_TOP POP_TOP POP_TOP returns
+        except_handler   ::= JUMP_FORWARD COME_FROM_EXCEPT except_return
+
+        # Try middle following a returns
+        except_handler36 ::= COME_FROM_EXCEPT except_stmts END_FINALLY
+
+        stmt             ::= try_except36
+        try_except36     ::= SETUP_EXCEPT returns except_handler36
+                             opt_come_from_except
+        try_except36     ::= SETUP_EXCEPT suite_stmts
+        try_except36     ::= SETUP_EXCEPT suite_stmts_opt POP_BLOCK
+                             except_handler36 come_from_opt
+
+        # 3.6 omits END_FINALLY sometimes
+        except_handler36 ::= COME_FROM_EXCEPT except_stmts
+        except_handler36 ::= JUMP_FORWARD COME_FROM_EXCEPT except_stmts
+        except_handler   ::= jmp_abs COME_FROM_EXCEPT except_stmts
+
+        stmt             ::= tryfinally36
+        tryfinally36     ::= SETUP_FINALLY returns
+                             COME_FROM_FINALLY suite_stmts
+        tryfinally36     ::= SETUP_FINALLY returns
+                             COME_FROM_FINALLY suite_stmts_opt END_FINALLY
+        except_suite_finalize ::= SETUP_FINALLY returns
+                                  COME_FROM_FINALLY suite_stmts_opt END_FINALLY _jump
+
+        stmt ::= tryfinally_return_stmt
+        tryfinally_return_stmt ::= SETUP_FINALLY suite_stmts_opt POP_BLOCK LOAD_CONST
+                                   COME_FROM_FINALLY
+
+        compare_chained2 ::= expr COMPARE_OP come_froms JUMP_FORWARD
         """
 
-    def p_expr3(self, args):
+    def p_37misc(self, args):
         """
-        expr           ::= conditionalnot
+        # long except clauses in a loop can sometimes cause a JUMP_BACK to turn into a
-        conditionalnot ::= expr jmp_true  expr jump_forward_else expr COME_FROM
+        # JUMP_FORWARD to a JUMP_BACK. And when this happens there is an additional
+        # ELSE added to the except_suite. With better flow control perhaps we can
+        # sort this out better.
+        except_suite ::= c_stmts_opt POP_EXCEPT jump_except ELSE
 
-        # a JUMP_FORWARD to another JUMP_FORWARD can get turned into
+        # FIXME: the below is to work around test_grammar expecting a "call" to be
-        # a JUMP_ABSOLUTE with no COME_FROM
+        # on the LHS because it is also somewhere on in a rule.
-        conditional    ::= expr jmp_false expr jump_absolute_else expr
+        call        ::= expr CALL_METHOD_0
-
-        # if_expr_true are for conditions which always evaluate true
-        # There is dead or non-optional remnants of the condition code though,
-        # and we use that to match on to reconstruct the source more accurately
-        expr           ::= if_expr_true
-        if_expr_true   ::= expr JUMP_FORWARD expr COME_FROM
         """
 
     def customize_grammar_rules(self, tokens, customize):

uncompyle6/parsers/parse37base.py

@@ -1,4 +1,4 @@
-#  Copyright (c) 2016-2017, 2019 Rocky Bernstein
+#  Copyright (c) 2016-2017, 2019-2020 Rocky Bernstein
 """
 Python 3.7 base code. We keep non-custom-generated grammar rules out of this file.
 """
@@ -581,6 +581,18 @@ class Python37BaseParser(PythonParser):
             elif opname == "LOAD_LISTCOMP":
                 self.add_unique_rule("expr ::= listcomp", opname, token.attr, customize)
                 custom_ops_processed.add(opname)
+            elif opname == "LOAD_NAME":
+                if token.attr == "__annotations__" and "SETUP_ANNOTATIONS" in self.seen_ops:
+                    token.kind = "LOAD_ANNOTATION"
+                    self.addRule(
+                        """
+                        stmt       ::= SETUP_ANNOTATIONS
+                        stmt       ::= ann_assign
+                        ann_assign ::= expr LOAD_ANNOTATION LOAD_STR STORE_SUBSCR
+                        """,
+                        nop_func,
+                    )
+                    pass
             elif opname == "LOAD_SETCOMP":
                 # Should this be generalized and put under MAKE_FUNCTION?
                 if has_get_iter_call_function1:
@@ -1103,8 +1115,13 @@ class Python37BaseParser(PythonParser):
             # FIXME: This is a cheap test. Should we do something with an AST like we
             # do with "and"?
             # "or"s with constants like this will have "COME_FROM" at the end
-            return tokens[last] in ("LOAD_ASSERT", "LOAD_STR", "LOAD_CODE", "LOAD_CONST",
+            return tokens[last] in (
-                                    "RAISE_VARARGS_1")
+                "LOAD_ASSERT",
+                "LOAD_STR",
+                "LOAD_CODE",
+                "LOAD_CONST",
+                "RAISE_VARARGS_1",
+            )
         elif lhs == "while1elsestmt":
 
             if last == n:
@@ -1143,7 +1160,7 @@ class Python37BaseParser(PythonParser):
             for i in range(cfl - 1, first, -1):
                 if tokens[i] != "POP_BLOCK":
                     break
-            if tokens[i].kind not in ("JUMP_BACK", "RETURN_VALUE"):
+            if tokens[i].kind not in ("JUMP_BACK", "RETURN_VALUE", "RAISE_VARARGS_1"):
                 if not tokens[i].kind.startswith("COME_FROM"):
                     return True
 
@@ -1156,9 +1173,8 @@ class Python37BaseParser(PythonParser):
                 last -= 1
             offset = tokens[last].off2int()
             assert tokens[first] == "SETUP_LOOP"
-            if offset != tokens[first].attr:
+            # SETUP_LOOP location must jump either to the last token or the token after the last one
-                return True
+            return tokens[first].attr not in (offset, offset + 2)
-            return False
         elif lhs == "_ifstmts_jump" and len(rule[1]) > 1 and ast:
             come_froms = ast[-1]
             # Make sure all of the "come froms" offset at the
@@ -1192,6 +1208,10 @@ class Python37BaseParser(PythonParser):
                 return False
 
             if isinstance(come_froms, Token):
+                if tokens[pop_jump_index].attr < tokens[pop_jump_index].offset and ast[0] != "pass":
+                    # This is a jump backwards to a loop. All bets are off here when there the
+                    # unless statement is "pass" which has no instructions associated with it.
+                    return False
                 return (
                     come_froms.attr is not None
                     and tokens[pop_jump_index].offset > come_froms.attr
@@ -1210,7 +1230,7 @@ class Python37BaseParser(PythonParser):
                 if last == n:
                     last -= 1
                     pass
-                if (tokens[last].attr and isinstance(tokens[last].attr, int)):
+                if tokens[last].attr and isinstance(tokens[last].attr, int):
                     return tokens[first].offset < tokens[last].attr
                 pass
 
@@ -1225,7 +1245,14 @@ class Python37BaseParser(PythonParser):
             for i in range(first, l):
                 t = tokens[i]
                 if t.kind == "POP_JUMP_IF_FALSE":
-                    if t.attr > last_offset:
+                    pjif_target = t.attr
+                    if pjif_target > last_offset:
+                        # In come cases, where we have long bytecode, a
+                        # "POP_JUMP_IF_FALSE" offset might be too
+                        # large for the instruction; so instead it
+                        # jumps to a JUMP_FORWARD. Allow that here.
+                        if tokens[l] == "JUMP_FORWARD":
+                            return tokens[l].attr != pjif_target
                         return True
                     pass
                 pass
@@ -1244,7 +1271,11 @@ class Python37BaseParser(PythonParser):
                         if last == n:
                             last -= 1
                         jmp_target = test[1][0].attr
-                        if tokens[first].off2int() <= jmp_target < tokens[last].off2int():
+                        if (
+                            tokens[first].off2int()
+                            <= jmp_target
+                            < tokens[last].off2int()
+                        ):
                             return True
                         # jmp_target less than tokens[first] is okay - is to a loop
                         # jmp_target equal tokens[last] is also okay: normal non-optimized non-loop jump
@@ -1279,7 +1310,11 @@ class Python37BaseParser(PythonParser):
                     # jmp_target less than tokens[first] is okay - is to a loop
                     # jmp_target equal tokens[last] is also okay: normal non-optimized non-loop jump
 
-                    if (last + 1) < n and tokens[last - 1] != "JUMP_BACK" and tokens[last + 1] == "COME_FROM_LOOP":
+                    if (
+                        (last + 1) < n
+                        and tokens[last - 1] != "JUMP_BACK"
+                        and tokens[last + 1] == "COME_FROM_LOOP"
+                    ):
                         # iflastsmtl is not at the end of a loop, but jumped outside of loop. No good.
                         # FIXME: check that tokens[last] == "POP_BLOCK"? Or allow for it not to appear?
                         return True
@@ -1323,6 +1358,36 @@ class Python37BaseParser(PythonParser):
                         "_come_froms",
                     ),
                 ),
+                (
+                    "ifelsestmt",
+                    (
+                        "testexpr",
+                        "c_stmts_opt",
+                        "jump_forward_else",
+                        "else_suite",
+                        '\\e__come_froms'
+                    ),
+                ),
+                (
+                    "ifelsestmt",
+                    (
+                        "testexpr",
+                        "c_stmts_opt",
+                        "jf_cfs",
+                        "else_suite",
+                        '\\e_opt_come_from_except',
+                    ),
+                ),
+                (
+                    "ifelsestmt",
+                    (
+                        "testexpr",
+                        "c_stmts_opt",
+                        "come_froms",
+                        "else_suite",
+                        'come_froms',
+                    ),
+                ),
                 (
                     "ifelsestmt",
                     (
@@ -1345,6 +1410,7 @@ class Python37BaseParser(PythonParser):
             if come_froms == "opt_come_from_except" and len(come_froms) > 0:
                 come_froms = come_froms[0]
             if not isinstance(come_froms, Token):
+                if len(come_froms):
                     return tokens[first].offset > come_froms[-1].attr
             elif tokens[first].offset > come_froms.attr:
                 return True
@@ -1363,16 +1429,34 @@ class Python37BaseParser(PythonParser):
 
             # Check that the condition portion of the "if"
             # jumps to the "else" part.
-            # Compare with parse30.py of uncompyle6
             if testexpr[0] in ("testtrue", "testfalse"):
                 test = testexpr[0]
+
+                else_suite = ast[3]
+                assert else_suite == "else_suite"
+
                 if len(test) > 1 and test[1].kind.startswith("jmp_"):
                     if last == n:
                         last -= 1
                     jmp = test[1]
                     jmp_target = jmp[0].attr
+
+                    # FIXME: the jump inside "else" check below should be added.
+                    #
+                    # add this until we can find out what's wrong with
+                    # not being able to parse:
+                    #     if a and b or c:
+                    #         x = 1
+                    #     else:
+                    #         x = 2
+
+                    # FIXME: add this
+                    # if jmp_target < else_suite.first_child().off2int():
+                    #     return True
+
                     if tokens[first].off2int() > jmp_target:
                         return True
+
                     return (jmp_target > tokens[last].off2int()) and tokens[
                         last
                     ] != "JUMP_FORWARD"

uncompyle6/semantics/customize37.py

@@ -1,4 +1,4 @@
-#  Copyright (c) 2019 by Rocky Bernstein
+#  Copyright (c) 2019-2020 by Rocky Bernstein
 #
 #  This program is free software: you can redistribute it and/or modify
 #  it under the terms of the GNU General Public License as published by
@@ -22,70 +22,118 @@ def customize_for_version37(self, version):
     # Python 3.7+ changes
     #######################
 
-    PRECEDENCE['attribute37'] =  2
+    PRECEDENCE["attribute37"] = 2
-    PRECEDENCE['if_exp_37a']   = 28
+    PRECEDENCE["call_ex"] = 1
-    PRECEDENCE['if_exp_37b']   = 28
+    PRECEDENCE["call_ex_kw"] = 1
+    PRECEDENCE["call_ex_kw2"] = 1
+    PRECEDENCE["call_ex_kw3"] = 1
+    PRECEDENCE["call_ex_kw4"] = 1
+    PRECEDENCE["call_kw"] = 0
+    PRECEDENCE["call_kw36"] = 1
+    PRECEDENCE["formatted_value1"] = 100
+    PRECEDENCE["if_exp_37a"] = 28
+    PRECEDENCE["if_exp_37b"] = 28
+    PRECEDENCE["unmap_dict"] = 0
 
-    TABLE_DIRECT.update({
+    TABLE_DIRECT.update(
-        'and_not':  ( '%c and not %c',
+        {
-                      (0, 'expr'), (2, 'expr') ),
+            "ann_assign": (
-        'async_forelse_stmt':  (
+                "%|%[2]{attr}: %c\n", 0,
-            '%|async for %c in %c:\n%+%c%-%|else:\n%+%c%-\n\n',
+            ),
-            (7, 'store'), (1, 'expr'), (17, 'for_block'), (25, 'else_suite') ),
+            "ann_assign_init": (
-        'async_for_stmt':  (
+                "%|%[2]{attr}: %c = %c\n", 0, 1,
-            '%|async for %c in %c:\n%+%c%-\n\n',
+            ),
-            (7, 'store'), (1, 'expr'), (17, 'for_block')),
+            "async_for_stmt": (
-        'async_for_stmt37':  (
+                "%|async for %c in %c:\n%+%c%-\n\n",
-            '%|async for %c in %c:\n%+%c%-%-\n\n',
+                (7, "store"),
-            (7, 'store'), (1, 'expr'), (16, 'for_block') ),
+                (1, "expr"),
-        'attribute37':  ( '%c.%[1]{pattr}', 0 ),
+                (17, "for_block"),
-        'compare_chained1a_37': (
+            ),
-            ' %[3]{pattr.replace("-", " ")} %p %p',
+            "async_for_stmt36": (
-            (0, 19), (-4, 19)),
+                "%|async for %c in %c:\n%+%c%-%-\n\n",
-        'compare_chained1_false_37': (
+                (9, "store"),
-            ' %[3]{pattr.replace("-", " ")} %p %p',
+                (1, "expr"),
-            (0, 19), (-4, 19)),
+                (18, "for_block"),
-        'compare_chained2_false_37': (
+            ),
-            ' %[3]{pattr.replace("-", " ")} %p %p',
+            "async_for_stmt37": (
-            (0, 19), (-5, 19)),
+                "%|async for %c in %c:\n%+%c%-%-\n\n",
-        'compare_chained1b_37': (
+                (7, "store"),
-            ' %[3]{pattr.replace("-", " ")} %p %p',
+                (1, "expr"),
-            (0, 19), (-4, 19)),
+                (16, "for_block"),
-        'compare_chained1c_37': (
+            ),
-            ' %[3]{pattr.replace("-", " ")} %p %p',
+            "and_not": ("%c and not %c", (0, "expr"), (2, "expr")),
-            (0, 19), (-2, 19)),
+            "async_with_stmt": ("%|async with %c:\n%+%c%-", (0, "expr"), 7),
-        'compare_chained2a_37': (
+            "async_with_as_stmt": (
-            '%[1]{pattr.replace("-", " ")} %p',
+                "%|async with %c as %c:\n%+%c%-",
-            (0, 19) ),
-        'compare_chained2b_37': (
-            '%[1]{pattr.replace("-", " ")} %p',
-            (0, 19) ),
-        'compare_chained2a_false_37': (
-            '%[1]{pattr.replace("-", " ")} %p',
-            (0, 19 ) ),
-        'compare_chained2c_37': (
-            '%[3]{pattr.replace("-", " ")} %p %p', (0, 19), (6, 19) ),
-        'if_exp_37a': ( '%p if %p else %p', (1, 'expr', 27), (0, 27), (4, 'expr', 27) ),
-        'if_exp_37b': ( '%p if %p else %p', (2, 'expr', 27), (0, 'expr', 27), (5, 'expr', 27) ),
-        'list_if37':  ( " if %p%c", (0, 27), 1 ),
-        'testfalse_not_or': ( "not %c or %c",
                 (0, "expr"),
-                              (2, "expr")  ),
+                (6, "store"),
-        'testfalse_not_and': ( "not (%c)", 0 ),
+                7,
-
+            ),
-        })
+            "async_forelse_stmt": (
-
+                "%|async for %c in %c:\n%+%c%-%|else:\n%+%c%-\n\n",
-    def n_import_from(node):
+                (7, "store"),
-        relative_path_index = 0
+                (1, "expr"),
-        if node[relative_path_index].pattr > 0:
+                (17, "for_block"),
-            node[2].pattr = ("." * node[relative_path_index].pattr) + node[2].pattr
+                (25, "else_suite"),
-        if isinstance(node[1].pattr, tuple):
+            ),
-            imports = node[1].pattr
+            "attribute37": ("%c.%[1]{pattr}", 0),
-            for pattr in imports:
+            "await_expr": ("await %c", 0),
-                node[1].pattr = pattr
+            "await_stmt": ("%|%c\n", 0),
-                self.default(node)
+            "call_ex": ("%c(%p)", (0, "expr"), (1, 100)),
-            return
+            "compare_chained1a_37": (
-        self.default(node)
+                ' %[3]{pattr.replace("-", " ")} %p %p',
-
+                (0, 19),
-    self.n_import_from = n_import_from
+                (-4, 19),
-    self.n_import_from_star = n_import_from
+            ),
+            "compare_chained1_false_37": (
+                ' %[3]{pattr.replace("-", " ")} %p %p',
+                (0, 19),
+                (-4, 19),
+            ),
+            "compare_chained2_false_37": (
+                ' %[3]{pattr.replace("-", " ")} %p %p',
+                (0, 19),
+                (-5, 19),
+            ),
+            "compare_chained1b_37": (
+                ' %[3]{pattr.replace("-", " ")} %p %p',
+                (0, 19),
+                (-4, 19),
+            ),
+            "compare_chained1c_37": (
+                ' %[3]{pattr.replace("-", " ")} %p %p',
+                (0, 19),
+                (-2, 19),
+            ),
+            "compare_chained2a_37": ('%[1]{pattr.replace("-", " ")} %p', (0, 19)),
+            "compare_chained2b_37": ('%[1]{pattr.replace("-", " ")} %p', (0, 19)),
+            "compare_chained2a_false_37": ('%[1]{pattr.replace("-", " ")} %p', (0, 19)),
+            "compare_chained2c_37": (
+                '%[3]{pattr.replace("-", " ")} %p %p',
+                (0, 19),
+                (6, 19),
+            ),
+            "except_return": ("%|except:\n%+%c%-", 3),
+            "if_exp_37a": (
+                "%p if %p else %p",
+                (1, "expr", 27),
+                (0, 27),
+                (4, "expr", 27),
+            ),
+            "if_exp_37b": (
+                "%p if %p else %p",
+                (2, "expr", 27),
+                (0, "expr", 27),
+                (5, "expr", 27),
+            ),
+            "ifstmtl": ("%|if %c:\n%+%c%-", (0, "testexpr"), (1, "_ifstmts_jumpl")),
+            'list_if37':  ( " if %p%c", (0, 27), 1 ),
+            "testfalse_not_or": ("not %c or %c", (0, "expr"), (2, "expr")),
+            "testfalse_not_and": ("not (%c)", 0),
+            "try_except36": ("%|try:\n%+%c%-%c\n\n", 1, -2),
+            "tryfinally36": ("%|try:\n%+%c%-%|finally:\n%+%c%-\n\n", (1, "returns"), 3),
+            "unmap_dict": ("{**%C}", (0, -1, ", **")),
+            "unpack_list": ("*%c", (0, "list")),
+            "yield_from": ("yield from %c", (0, "expr")),
+        }
+    )

uncompyle6/semantics/transform.py

@@ -1,4 +1,4 @@
-#  Copyright (c) 2019 by Rocky Bernstein
+#  Copyright (c) 2019-2020 by Rocky Bernstein
 
 #  This program is free software: you can redistribute it and/or modify
 #  it under the terms of the GNU General Public License as published by
@@ -296,6 +296,28 @@ class TreeTransform(GenericASTTraversal, object):
             list_for_node.transformed_by = ("n_list_for",)
         return list_for_node
 
+    def n_stmts(self, node):
+        if node.first_child() == "SETUP_ANNOTATIONS":
+            prev = node[0][0][0]
+            new_stmts = [node[0]]
+            for i, sstmt in enumerate(node[1:]):
+                ann_assign = sstmt[0][0]
+                if (sstmt[0] == "stmt" and ann_assign == "ann_assign" and prev == "assign"):
+                    annotate_var = ann_assign[-2]
+                    if annotate_var.attr == prev[-1][0].attr:
+                        del new_stmts[-1]
+                        sstmt[0][0] = SyntaxTree(
+                            "ann_assign_init",
+                            [ann_assign[0], prev[0], annotate_var])
+                        sstmt[0][0].transformed_by="n_stmts"
+                        pass
+                    pass
+                new_stmts.append(sstmt)
+                prev = ann_assign
+                pass
+            node.data = new_stmts
+        return node
+
     def traverse(self, node, is_lambda=False):
         node = self.preorder(node)
         return node