3.6-3.8 "async for" handling...

And add an if_stmt rule for 3.8. My want to extend it back to
other versions.

uncompyle6/parsers/parse36.py

@@ -68,6 +68,22 @@ class Python36Parser(Python35Parser):
                                for_block POP_BLOCK
                                COME_FROM_LOOP
 
+        stmt      ::= async_for_stmt36
+
+        async_for_stmt36   ::= SETUP_LOOP expr
+                               GET_AITER
+                               LOAD_CONST YIELD_FROM
+                               SETUP_EXCEPT GET_ANEXT LOAD_CONST
+                               YIELD_FROM
+                               store
+                               POP_BLOCK JUMP_BACK COME_FROM_EXCEPT DUP_TOP
+                               LOAD_GLOBAL COMPARE_OP POP_JUMP_IF_TRUE
+                               END_FINALLY for_block
+                               COME_FROM
+                               POP_TOP POP_TOP POP_TOP POP_EXCEPT
+                               POP_TOP POP_BLOCK
+                               COME_FROM_LOOP
+
         async_forelse_stmt ::= SETUP_LOOP expr
                                GET_AITER
                                LOAD_CONST YIELD_FROM SETUP_EXCEPT GET_ANEXT LOAD_CONST
@@ -363,6 +379,7 @@ class Python36Parser(Python35Parser):
                 pass
             pass
         return False
+
 class Python36ParserSingle(Python36Parser, PythonParserSingle):
     pass
 

uncompyle6/parsers/parse37.py

@@ -31,6 +31,7 @@ class Python37Parser(Python36Parser):
         """
         # Where does the POP_TOP really belong?
         stmt     ::= import37
+        stmt     ::= async_for_stmt37
         import37 ::= import POP_TOP
 
         async_for_stmt     ::= SETUP_LOOP expr
@@ -46,6 +47,19 @@ class Python37Parser(Python36Parser):
                                POP_TOP POP_TOP POP_TOP POP_EXCEPT POP_TOP POP_BLOCK
                                COME_FROM_LOOP
 
+        # Order of LOAD_CONST YIELD_FROM is switched from 3.6 to save a LOAD_CONST
+        async_for_stmt37   ::= SETUP_LOOP expr
+                               GET_AITER
+                               SETUP_EXCEPT GET_ANEXT
+                               LOAD_CONST YIELD_FROM
+                               store
+                               POP_BLOCK JUMP_BACK COME_FROM_EXCEPT DUP_TOP
+                               LOAD_GLOBAL COMPARE_OP POP_JUMP_IF_TRUE
+                               END_FINALLY for_block COME_FROM
+                               POP_TOP POP_TOP POP_TOP POP_EXCEPT
+                               POP_TOP POP_BLOCK
+                               COME_FROM_LOOP
+
         async_forelse_stmt ::= SETUP_LOOP expr
                                GET_AITER
                                SETUP_EXCEPT GET_ANEXT LOAD_CONST
@@ -93,6 +107,18 @@ class Python37Parser(Python36Parser):
                                  JUMP_ABSOLUTE END_FINALLY COME_FROM
                                  for_block POP_BLOCK
                                  else_suite COME_FROM_LOOP
+        stmt      ::= async_for_stmt36
+        async_for_stmt36   ::= SETUP_LOOP expr
+                               GET_AITER
+                               LOAD_CONST YIELD_FROM SETUP_EXCEPT GET_ANEXT LOAD_CONST
+                               YIELD_FROM
+                               store
+                               POP_BLOCK JUMP_BACK COME_FROM_EXCEPT DUP_TOP
+                               LOAD_GLOBAL COMPARE_OP POP_JUMP_IF_TRUE
+                               END_FINALLY continues COME_FROM
+                               POP_TOP POP_TOP POP_TOP POP_EXCEPT
+                               POP_TOP POP_BLOCK
+                               COME_FROM_LOOP
         """)
         super(Python37Parser, self).customize_grammar_rules(tokens, customize)
 

uncompyle6/parsers/parse38.py

@@ -20,11 +20,13 @@ from __future__ import print_function
 from uncompyle6.parser import PythonParserSingle
 from spark_parser import DEFAULT_DEBUG as PARSER_DEFAULT_DEBUG
 from uncompyle6.parsers.parse37 import Python37Parser
+from uncompyle6.scanners.tok import Token
 
 class Python38Parser(Python37Parser):
 
     def p_38misc(self, args):
         """
+        stmt               ::= async_for_stmt38
         stmt               ::= for38
         stmt               ::= forelsestmt38
         stmt               ::= forelselaststmt38
@@ -36,6 +38,20 @@ class Python38Parser(Python37Parser):
         # FIXME this should be restricted to being inside a try block
         stmt               ::= except_ret38
 
+        async_for_stmt38   ::= expr
+                              GET_AITER
+                              SETUP_FINALLY
+                              GET_ANEXT
+                              LOAD_CONST
+                              YIELD_FROM
+                              POP_BLOCK
+                              store for_block
+                              COME_FROM_FINALLY
+                              END_ASYNC_FOR
+
+
+        return             ::= ret_expr ROT_TWO POP_TOP RETURN_VALUE
+
         for38              ::= expr get_iter store for_block JUMP_BACK
         for38              ::= expr for_iter store for_block JUMP_BACK
         for38              ::= expr for_iter store for_block JUMP_BACK POP_BLOCK
@@ -46,12 +62,11 @@ class Python38Parser(Python37Parser):
         whilestmt          ::= testexpr l_stmts_opt COME_FROM JUMP_BACK POP_BLOCK
         whilestmt          ::= testexpr l_stmts_opt JUMP_BACK POP_BLOCK
         whilestmt          ::= testexpr returns               POP_BLOCK
-        while1elsestmt     ::=          l_stmts     JUMP_BACK
+        # while1elsestmt   ::=          l_stmts     JUMP_BACK
         whileelsestmt      ::= testexpr l_stmts     JUMP_BACK POP_BLOCK
         whileTruestmt      ::= l_stmts              JUMP_BACK POP_BLOCK
         while1stmt         ::= l_stmts COME_FROM_LOOP
         while1stmt         ::= l_stmts COME_FROM JUMP_BACK COME_FROM_LOOP
-        while1elsestmt     ::= l_stmts JUMP_BACK
         whileTruestmt      ::= l_stmts JUMP_BACK NOP
         whileTruestmt      ::= l_stmts JUMP_BACK POP_BLOCK NOP
         for_block          ::= l_stmts_opt _come_from_loops JUMP_BACK
@@ -99,10 +114,22 @@ class Python38Parser(Python37Parser):
 
     def customize_grammar_rules(self, tokens, customize):
         self.remove_rules("""
+           stmt               ::= async_for_stmt37
            stmt               ::= for
            stmt               ::= forelsestmt
            stmt               ::= try_except36
 
+           async_for_stmt37   ::= SETUP_LOOP expr
+                                  GET_AITER
+                                  SETUP_EXCEPT GET_ANEXT
+                                  LOAD_CONST YIELD_FROM
+                                  store
+                                  POP_BLOCK JUMP_BACK COME_FROM_EXCEPT DUP_TOP
+                                  LOAD_GLOBAL COMPARE_OP POP_JUMP_IF_TRUE
+                                  END_FINALLY for_block COME_FROM
+                                  POP_TOP POP_TOP POP_TOP POP_EXCEPT
+                                  POP_TOP POP_BLOCK
+                               COME_FROM_LOOP
            for                ::= SETUP_LOOP expr for_iter store for_block POP_BLOCK
            for                ::= SETUP_LOOP expr for_iter store for_block POP_BLOCK NOP
 
@@ -123,6 +150,32 @@ class Python38Parser(Python37Parser):
 
         """)
         super(Python37Parser, self).customize_grammar_rules(tokens, customize)
+        self.check_reduce['ifstmt'] = 'tokens'
+
+    def reduce_is_invalid(self, rule, ast, tokens, first, last):
+        invalid = super(Python38Parser,
+                        self).reduce_is_invalid(rule, ast,
+                                                tokens, first, last)
+        if invalid:
+            return invalid
+        if rule[0] == 'ifstmt':
+            # Make sure jumps don't extend beyond the end of the if statement.
+            l = last
+            if l == len(tokens):
+                l -= 1
+            if isinstance(tokens[l].offset, str):
+                last_offset = int(tokens[l].offset.split('_')[0], 10)
+            else:
+                last_offset = tokens[l].offset
+            for i in range(first, l):
+                t = tokens[i]
+                if t.kind == 'POP_JUMP_IF_FALSE':
+                    if t.attr > last_offset:
+                        return True
+                    pass
+                pass
+            pass
+        return False
 
 class Python38ParserSingle(Python38Parser, PythonParserSingle):
     pass

uncompyle6/semantics/check_ast.py

@@ -11,9 +11,10 @@ before reduction and don't reduce when there is a problem.
 def checker(ast, in_loop, errors):
     if ast is None:
         return
-    in_loop = in_loop or ast.kind in ('while1stmt', 'whileTruestmt',
+    in_loop = (in_loop or (ast.kind in ('while1stmt', 'whileTruestmt',
                                       'whilestmt', 'whileelsestmt', 'while1elsestmt',
-                                      'for_block')
+                                      'for_block'))
+               or ast.kind.startswith('async_for'))
     if ast.kind in ('aug_assign1', 'aug_assign2') and ast[0][0] == 'and':
         text = str(ast)
         error_text = '\n# improper augmented assigment (e.g. +=, *=, ...):\n#\t' + '\n# '.join(text.split("\n")) + '\n'

uncompyle6/semantics/customize3.py

@@ -516,6 +516,10 @@ def customize_for_version3(self, version):
                 'tryfinally_return_stmt':
                       ( '%|try:\n%+%c%-%|finally:\n%+%|return%-\n\n', 1 ),
 
+                'async_for_stmt36':  (
+                    '%|async for %c in %c:\n%+%c%-%-\n\n',
+                    (9, 'store'), (1, 'expr'), (18, 'for_block') ),
+
                 'call_ex' : (
                     '%c(%p)',
                     (0, 'expr'), (1, 100)),
@@ -917,10 +921,10 @@ def customize_for_version3(self, version):
                     'attribute37':  ( '%c.%[1]{pattr}', 0 ),
                     'async_forelse_stmt':  (
                         '%|async for %c in %c:\n%+%c%-%|else:\n%+%c%-\n\n',
-                        7, 1, 17, (25, 'else_suite') ),
+                        (7, 'store'), (1, 'expr'), (17, 'for_block'), (25, 'else_suite') ),
                     'async_for_stmt':  (
                         '%|async for %c in %c:\n%+%c%-%-\n\n',
-                        7, 1, 17),
+                        (7, 'store'), (1, 'expr'), (17, 'for_block')),
                     'compare_chained1a_37': ( ' %[3]{pattr.replace("-", " ")} %p %p',
                                              (0, 19),
                                              (-4, 19)),
@@ -931,6 +935,9 @@ def customize_for_version3(self, version):
 
                     'compare_chained2b_37': ( '%[1]{pattr.replace("-", " ")} %p', (0, 19)),
 
+                    'async_for_stmt37':  (
+                        '%|async for %c in %c:\n%+%c%-%-\n\n',
+                        (7, 'store'), (1, 'expr'), (16, 'for_block') ),
                     })
                 if version >= 3.8:
                     ########################
@@ -943,6 +950,10 @@ def customize_for_version3(self, version):
                     #     del TABLE_DIRECT[lhs]
 
                     TABLE_DIRECT.update({
+                        'async_for_stmt38':  (
+                            '%|async for %c in %c:\n%+%c%-%-\n\n',
+                            (0, 'expr'), (7, 'store'), (8, 'for_block') ),
+
                         'for38':            (
                             '%|for %c in %c:\n%+%c%-\n\n',
                             (2, 'store'),