Adapt decompyle3's 3.8 try/return grammar rules

.gitignore

@@ -10,6 +10,7 @@
 /.pytest_cache
 /.python-version
 /.tox
+.mypy_cache
 /.venv*
 /README
 /__pkginfo__.pyc

test/.gitignore

@@ -1,3 +1,6 @@
 /.coverage
 /.python-version
 /nohup.out
+/pycdc
+/test_unpy33.py
+/test_unpy37.py

test/stdlib/.gitignore

@@ -1 +1,3 @@
 /.python-version
+/runun33.sh
+/runun7.sh

uncompyle6/parsers/parse38.py

@@ -1,4 +1,4 @@
-#  Copyright (c) 2017-2019 Rocky Bernstein
+#  Copyright (c) 2017-2020 Rocky Bernstein
 #
 #  This program is free software: you can redistribute it and/or modify
 #  it under the terms of the GNU General Public License as published by
@@ -39,6 +39,8 @@ class Python38Parser(Python37Parser):
         stmt               ::= forelselaststmtl38
         stmt               ::= tryfinally38stmt
         stmt               ::= tryfinally38rstmt
+        stmt               ::= tryfinally38rstmt2
+        stmt               ::= tryfinally38rstmt3
         stmt               ::= tryfinally38astmt
         stmt               ::= try_elsestmtl38
         stmt               ::= try_except_ret38
@@ -54,15 +56,12 @@ class Python38Parser(Python37Parser):
 
         # FIXME: this should be restricted to being inside a try block
         stmt               ::= except_ret38
+        stmt               ::= except_ret38a
 
         # FIXME: this should be added only when seeing GET_AITER or YIELD_FROM
-        async_for_stmt38   ::= expr
+        async_for          ::= GET_AITER _come_froms
-                               GET_AITER
+                               SETUP_FINALLY GET_ANEXT LOAD_CONST YIELD_FROM POP_BLOCK
-                               SETUP_FINALLY
+        async_for_stmt38   ::= expr async_for
-                               GET_ANEXT
-                               LOAD_CONST
-                               YIELD_FROM
-                               POP_BLOCK
                                store for_block
                                COME_FROM_FINALLY
                                END_ASYNC_FOR
@@ -81,7 +80,20 @@ class Python38Parser(Python37Parser):
                                END_ASYNC_FOR
                                else_suite
 
-        return             ::= ret_expr ROT_TWO POP_TOP RETURN_VALUE
+        # Seems to be used to discard values before a return in a "for" loop
+        discard_top        ::= ROT_TWO POP_TOP
+        discard_tops       ::= discard_top+
+
+        return             ::= ret_expr
+                               discard_tops RETURN_VALUE
+
+        return             ::= popb_return
+        return             ::= pop_return
+        return             ::= pop_ex_return
+        except_stmt        ::= pop_ex_return
+        pop_return         ::= POP_TOP ret_expr RETURN_VALUE
+        popb_return        ::= ret_expr POP_BLOCK RETURN_VALUE
+        pop_ex_return      ::= ret_expr ROT_FOUR POP_EXCEPT RETURN_VALUE
 
         # 3.8 can push a looping JUMP_BACK into into a JUMP_ from a statement that jumps to it
         lastl_stmt         ::= ifpoplaststmtl
@@ -128,8 +140,14 @@ class Python38Parser(Python37Parser):
                                except_handler38
         try_except38       ::= SETUP_FINALLY POP_BLOCK POP_TOP suite_stmts_opt
                                except_handler38a
-        try_except_ret38   ::= SETUP_FINALLY expr POP_BLOCK
+
-                               RETURN_VALUE except_ret38a
+        # suite_stmts has a return
+        try_except38       ::= SETUP_FINALLY POP_BLOCK suite_stmts
+                               except_handler38b
+
+        try_except_ret38   ::= SETUP_FINALLY returns except_ret38a
+        try_except_ret38a  ::= SETUP_FINALLY returns except_handler38c
+                               END_FINALLY come_from_opt
 
         # Note: there is a suite_stmts_opt which seems
         # to be bookkeeping which is not expressed in source code
@@ -149,19 +167,42 @@ class Python38Parser(Python37Parser):
         tryfinallystmt     ::= SETUP_FINALLY suite_stmts_opt POP_BLOCK
                                BEGIN_FINALLY COME_FROM_FINALLY suite_stmts_opt
                                END_FINALLY
-        tryfinally38rstmt  ::= SETUP_FINALLY POP_BLOCK CALL_FINALLY
+
+
+        lc_setup_finally   ::= LOAD_CONST SETUP_FINALLY
+        call_finally_pt    ::= CALL_FINALLY POP_TOP
+        cf_cf_finally      ::= come_from_opt COME_FROM_FINALLY
+        pop_finally_pt     ::= POP_FINALLY POP_TOP
+        ss_end_finally     ::= suite_stmts END_FINALLY
+        sf_pb_call_returns ::= SETUP_FINALLY POP_BLOCK CALL_FINALLY returns
+
+
+        # FIXME: DRY rules below
+        tryfinally38rstmt  ::= sf_pb_call_returns
+                               cf_cf_finally
+                               ss_end_finally
+        tryfinally38rstmt  ::= sf_pb_call_returns
+                               cf_cf_finally END_FINALLY
+                               suite_stmts
+        tryfinally38rstmt  ::= sf_pb_call_returns
+                               cf_cf_finally POP_FINALLY
+                               ss_end_finally
+        tryfinally38rstmt  ::= sf_bp_call_returns
+                               COME_FROM_FINALLY POP_FINALLY
+                               ss_end_finally
+
+        tryfinally38rstmt2 ::= lc_setup_finally POP_BLOCK call_finally_pt
                                returns
-                               COME_FROM_FINALLY END_FINALLY suite_stmts
+                               cf_cf_finally pop_finally_pt
-        tryfinally38rstmt  ::= SETUP_FINALLY POP_BLOCK CALL_FINALLY
+                               ss_end_finally POP_TOP
-                               returns
+        tryfinally38rstmt3 ::= SETUP_FINALLY expr POP_BLOCK CALL_FINALLY RETURN_VALUE
-                               COME_FROM_FINALLY POP_FINALLY returns
+                               COME_FROM COME_FROM_FINALLY
-                               END_FINALLY
+                               ss_end_finally
-        tryfinally38stmt   ::= SETUP_FINALLY suite_stmts_opt POP_BLOCK
+
-                               BEGIN_FINALLY COME_FROM_FINALLY
-                               POP_FINALLY suite_stmts_opt END_FINALLY
         tryfinally38stmt   ::= SETUP_FINALLY suite_stmts_opt POP_BLOCK
                                BEGIN_FINALLY COME_FROM_FINALLY
                                POP_FINALLY suite_stmts_opt END_FINALLY
+
         tryfinally38astmt  ::= LOAD_CONST SETUP_FINALLY suite_stmts_opt POP_BLOCK
                                BEGIN_FINALLY COME_FROM_FINALLY
                                POP_FINALLY POP_TOP suite_stmts_opt END_FINALLY POP_TOP

uncompyle6/semantics/customize38.py

@@ -29,9 +29,9 @@ def customize_for_version38(self, version):
     #     del TABLE_DIRECT[lhs]
 
     TABLE_DIRECT.update({
-        'async_for_stmt38':  (
+        "async_for_stmt38":  (
-            '%|async for %c in %c:\n%+%c%-%-\n\n',
+            "%|async for %c in %c:\n%+%c%-%-\n\n",
-            (7, 'store'), (0, 'expr'), (8, 'for_block') ),
+            (2, "store"), (0, "expr"), (3, "for_block") ),
 
         'async_forelse_stmt38':  (
             '%|async for %c in %c:\n%+%c%-%|else:\n%+%c%-\n\n',
@@ -101,12 +101,26 @@ def customize_for_version38(self, version):
         'try_except38': (
             '%|try:\n%+%c\n%-%|except:\n%|%-%c\n\n',
                    (-2, 'suite_stmts_opt'), (-1, 'except_handler38a') ),
-        'try_except_ret38': (
+        "try_except_ret38": (
-            '%|try:\n%+%|return %c%-\n%|except:\n%+%|%c%-\n\n',
+            "%|try:\n%+%c%-\n%|except:\n%+%|%c%-\n\n",
-                   (1, 'expr'), (-1, 'except_ret38a') ),
+            (1, "returns"),
+            (2, "except_ret38a"),
+        ),
         'tryfinally38rstmt': (
             '%|try:\n%+%c%-%|finally:\n%+%c%-\n\n',
-                   (3, 'returns'), 6 ),
+            (0, "sf_pb_call_returns"),
+            (-1, ("ss_end_finally", "suite_stmts")),
+        ),
+        "tryfinally38rstmt2": (
+            "%|try:\n%+%c%-%|finally:\n%+%c%-\n\n",
+            (4, "returns"),
+            -2, "ss_end_finally"
+        ),
+        "tryfinally38rstmt3": (
+            "%|try:\n%+%|return %c%-\n%|finally:\n%+%c%-\n\n",
+            (1, "expr"),
+            (-1, "ss_end_finally")
+        ),
         'tryfinally38stmt': (
             '%|try:\n%+%c%-%|finally:\n%+%c%-\n\n',
             (1, "suite_stmts_opt"),

uncompyle6/semantics/pysource.py

@@ -2053,10 +2053,17 @@ class SourceWalker(GenericASTTraversal, object):
             elif typ == "c":
                 index = entry[arg]
                 if isinstance(index, tuple):
+                    if isinstance(index[1], str):
                         assert node[index[0]] == index[1], (
                             "at %s[%d], expected '%s' node; got '%s'"
                             % (node.kind, arg, index[1], node[index[0]].kind)
                         )
+                    else:
+                        assert node[index[0]] in index[1], (
+                            "at %s[%d], expected to be in '%s' node; got '%s'"
+                            % (node.kind, arg, index[1], node[index[0]].kind)
+                        )
+
                     index = index[0]
                 assert isinstance(
                     index, int
@@ -2065,6 +2072,16 @@ class SourceWalker(GenericASTTraversal, object):
                     arg,
                     type(index),
                 )
+
+                try:
+                    node[index]
+                except IndexError:
+                    raise RuntimeError(
+                        """
+                        Expanding '%' in template '%s[%s]':
+                        %s is invalid; has only %d entries
+                        """ % (node,kind, enty, arg, index, len(node))
+                    )
                 self.preorder(node[index])
 
                 arg += 1