Merge branch 'master' into python-2.4

Makefile

@@ -48,10 +48,14 @@ check-2.4 check-2.5:
 # Skip for now
 2.6 5.0 5.3 5.6 5.8:
 
-#:PyPy pypy3-2.4.0 Python 3:
+#:PyPy pypy3-2.4.0 Python 3.6.1:
 7.1 pypy-3.2 2.4:
 	$(MAKE) -C test $@
 
+#:PyPy pypy3-2.4.0 Python 3.6.9:
+7.2:
+	$(MAKE) -C test $@
+
 #: Run py.test tests
 pytest:
 	$(MAKE) -C pytest check

test/Makefile

@@ -337,12 +337,18 @@ pypy-2.7 5.0 5.3 6.0:
 pypy-3.2 2.4:
 	$(PYTHON) test_pythonlib.py --bytecode-pypy3.2 --verify
 
-#: PyPy 5.0.x with Python 3.6 ...
+#: PyPy 5.0.x with Python 3.6.1 ...
 check-bytecode-pypy3.6: 7.1
 7.1:
 	$(PYTHON) test_pythonlib.py --bytecode-pypy3.6-run --verify-run
 	$(PYTHON) test_pythonlib.py --bytecode-pypy3.6 --verify
 
+#: PyPy 5.0.x with Python 3.6.9
+check-bytecode-pypy3.6: 7.2
+7.2:
+	$(PYTHON) test_pythonlib.py --bytecode-pypy3.6-run --verify-run
+	$(PYTHON) test_pythonlib.py --bytecode-pypy3.6 --verify
+
 
 
 clean: clean-py-dis clean-dis clean-unverified

test/simple_source/bug30/02_try_except_except.py

@@ -7,3 +7,13 @@ def start_new_thread(function, args, kwargs={}):
         pass
     except:
         args()
+
+# Adapted from 3.0.1 code.py
+# Bug is again JUMP_FORWARD elimination compared
+# to earlier and later Pythons.
+def interact():
+    while 1:
+        try:
+            more = 1
+        except KeyboardInterrupt:
+            more = 0

test/simple_source/bug30/02_while1_if_while1.py

@@ -7,3 +7,8 @@ while 1:
             raise RuntimeError
     else:
         raise RuntimeError
+
+# Adapted from 3.0.1 cgi.py
+def _parseparam(s, end):
+    while end > 0 and s.count(''):
+        end = s.find(';')

test/simple_source/bug30/03_ifelse.py

@@ -0,0 +1,50 @@
+# Adapted from 3.0 base64
+# Problem was handling if/else which
+# needs to be like Python 2.6 (and not like 2.7 or 3.1)
+def main(args, f, func, sys):
+    """Small main program"""
+    if args and args[0] != '-':
+        func(f, sys.stdout.buffer)
+    else:
+        func(sys.stdin.buffer, sys.stdout.buffer)
+
+# From Python 3.0 _markupbase.py.
+#
+# The Problem was in the way "if"s are generated in 3.0 which is sort
+# of like a more optimized Python 2.6, with reduced extraneous jumps,
+# but still 2.6-ish and not 2.7- or 3.1-ish.
+def parse_marked_section(fn, i, rawdata, report=1):
+    if report:
+        j = 1
+        fn(rawdata[i: j])
+    return 10
+
+# From 3.0.1 _abcoll.py
+# Bug was in genexpr_func which doesn't have a JUMP_BACK but
+# in its gen_comp_body, we can use COME_FROM in its place.
+# As above omission of JUMPs is a feature of 3.0 that doesn't
+# seem to be in later versions (or earlier like 2.6).
+def __and__(self, other, Iterable):
+    if not isinstance(other, Iterable):
+        return NotImplemented
+    return self._from_iterable(value for value in other if value in self)
+
+# Adapted from 3.0.1 abc.py
+# Bug was in handling multiple COME_FROMs in return_if_stmt
+def __instancecheck__(subtype, subclass, cls):
+    if subtype:
+        if (cls and subclass):
+            return False
+
+
+# Adapted from 3.0.1 abc.py
+# Bug was rule in "jump_absolute_else" and disallowing
+# "else" to the wrong place.
+
+def _strptime(locale_time, found_zone, time):
+    for tz_values in locale_time:
+        if found_zone:
+            if (time and found_zone):
+                break
+            else:
+                break

test/simple_source/bug30/04_and_del.py

@@ -18,3 +18,13 @@ assert normpath(['.']) == []
 assert normpath(['a', 'b', '..']) == ['a']
 assert normpath(['a', 'b', '', 'c']) == ['a', 'b', 'c']
 assert normpath(['a', 'b', '.', '', 'c', '..']) == ['a', 'b']
+
+# Adapted from 3.0.1 cgi.py
+# Bug was in detecting "or" and "and" due to lack of PUSH/POP_IF instructions.
+def handle(format, html, text):
+    formatter = (format and html) or text
+    return formatter
+
+assert handle(False, False, True)
+assert not handle(True, False, False)
+assert handle(True, True, False)

test/test_pyenvlib.py

@@ -43,6 +43,8 @@ TEST_VERSIONS = (
     "pypy3.5-5.9.0",
     "pypy3.5-6.0.0",
     "pypy3.6-7.1.0",
+    "pypy3.6-7.1.1",
+    "pypy3.6-7.2.0",
     "native",
 ) + tuple(python_versions)
 

uncompyle6/parsers/parse26.py

@@ -254,7 +254,7 @@ class Python26Parser(Python2Parser):
                          POP_TOP jb_pb_come_from
 
         generator_exp ::= LOAD_GENEXPR MAKE_FUNCTION_0 expr GET_ITER CALL_FUNCTION_1 COME_FROM
-        list_if ::= list_if ::= expr jmp_false_then list_iter
+        list_if ::= expr jmp_false_then list_iter
         '''
 
     def p_ret26(self, args):
@@ -467,7 +467,7 @@ if __name__ == '__main__':
     p.check_grammar()
     from uncompyle6 import PYTHON_VERSION, IS_PYPY
     if PYTHON_VERSION == 2.6:
-        lhs, rhs, tokens, right_recursive = p.check_sets()
+        lhs, rhs, tokens, right_recursive, dup_rhs = p.check_sets()
         from uncompyle6.scanner import get_scanner
         s = get_scanner(PYTHON_VERSION, IS_PYPY)
         opcode_set = set(s.opc.opname).union(set(

uncompyle6/parsers/parse27.py

@@ -292,7 +292,7 @@ if __name__ == '__main__':
     p.check_grammar()
     from uncompyle6 import PYTHON_VERSION, IS_PYPY
     if PYTHON_VERSION == 2.7:
-        lhs, rhs, tokens, right_recursive = p.check_sets()
+        lhs, rhs, tokens, right_recursive, dup_rhs = p.check_sets()
         from uncompyle6.scanner import get_scanner
         s = get_scanner(PYTHON_VERSION, IS_PYPY)
         opcode_set = set(s.opc.opname).union(set(

uncompyle6/parsers/parse3.py

@@ -1439,8 +1439,9 @@ class Python3Parser(PythonParser):
                                          except_handler COME_FROM else_suitel
                                          opt_come_from_except
                     """,
-                    nop_func,
+                    nop_func
                 )
+
                 custom_ops_processed.add(opname)
             elif opname_base in ("UNPACK_EX",):
                 before_count, after_count = token.attr
@@ -1543,12 +1544,24 @@ class Python3Parser(PythonParser):
             for i in range(cfl - 1, first, -1):
                 if tokens[i] != "POP_BLOCK":
                     break
-            if tokens[i].kind not in ("JUMP_BACK", "RETURN_VALUE"):
+            if tokens[i].kind not in ("JUMP_BACK", "RETURN_VALUE", "BREAK_LOOP"):
                 if not tokens[i].kind.startswith("COME_FROM"):
                     return True
 
             # Check that the SETUP_LOOP jumps to the offset after the
             # COME_FROM_LOOP
+
+            # Python 3.0 has additional:
+            #     JUMP_FORWARD here
+            #     COME_FROM
+            #     POP_TOP
+            #     COME_FROM
+            #  here:
+            #     (target of SETUP_LOOP)
+            # We won't check this.
+            if self.version == 3.0:
+                return False
+
             if 0 <= last < len(tokens) and tokens[last] in (
                 "COME_FROM_LOOP",
                 "JUMP_BACK",

uncompyle6/parsers/parse30.py

@@ -11,13 +11,21 @@ class Python30Parser(Python31Parser):
     def p_30(self, args):
         """
 
+        pt_bp             ::= POP_TOP POP_BLOCK
+
         assert            ::= assert_expr jmp_true LOAD_ASSERT RAISE_VARARGS_1 COME_FROM POP_TOP
+        assert2           ::= assert_expr jmp_true LOAD_ASSERT expr CALL_FUNCTION_1 RAISE_VARARGS_1
+                              come_froms
+        call_stmt         ::= expr _come_froms POP_TOP
+
         return_if_lambda  ::= RETURN_END_IF_LAMBDA COME_FROM POP_TOP
         compare_chained2  ::= expr COMPARE_OP RETURN_END_IF_LAMBDA
 
         # FIXME: combine with parse3.2
-        whileTruestmt     ::= SETUP_LOOP l_stmts_opt JUMP_BACK
+        whileTruestmt     ::= SETUP_LOOP l_stmts_opt
-                              COME_FROM_LOOP
+                              JUMP_BACK COME_FROM_LOOP
+        whileTruestmt     ::= SETUP_LOOP l_stmts_opt
+                              CONTINUE COME_FROM_LOOP
         whileTruestmt     ::= SETUP_LOOP returns
                               COME_FROM_LOOP
 
@@ -42,10 +50,22 @@ class Python30Parser(Python31Parser):
 
         else_suitel ::= l_stmts COME_FROM_LOOP JUMP_BACK
 
-        ifelsestmtl ::= testexpr c_stmts_opt jb_pop_top else_suitel
+        jump_absolute_else ::= COME_FROM JUMP_ABSOLUTE COME_FROM POP_TOP
+
+        jump_cf_pop   ::= _come_froms _jump  _come_froms POP_TOP
+
+        ifelsestmt  ::= testexpr c_stmts_opt jump_cf_pop else_suite COME_FROM
+        ifelsestmtl ::= testexpr c_stmts_opt jump_cf_pop else_suitel
+        ifelsestmtc ::= testexpr c_stmts_opt jump_absolute_else else_suitec
+        ifelsestmtc ::= testexpr c_stmts_opt jump_cf_pop else_suitec
+
+        iflaststmt  ::= testexpr c_stmts_opt JUMP_ABSOLUTE COME_FROM
         iflaststmtl ::= testexpr c_stmts_opt jb_pop_top
+        iflaststmtl ::= testexpr c_stmts_opt come_froms JUMP_BACK COME_FROM POP_TOP
+
         iflaststmt  ::= testexpr c_stmts_opt JUMP_ABSOLUTE COME_FROM POP_TOP
 
+
         withasstmt    ::= expr setupwithas store suite_stmts_opt
                           POP_BLOCK LOAD_CONST COME_FROM_FINALLY
                           LOAD_FAST DELETE_FAST WITH_CLEANUP END_FINALLY
@@ -53,8 +73,8 @@ class Python30Parser(Python31Parser):
         setup_finally ::= STORE_FAST SETUP_FINALLY LOAD_FAST DELETE_FAST
 
         # Need to keep LOAD_FAST as index 1
-        set_comp_func_header ::=  BUILD_SET_0 DUP_TOP STORE_FAST
+        set_comp_header  ::= BUILD_SET_0 DUP_TOP STORE_FAST
-        set_comp_func ::= set_comp_func_header
+        set_comp_func ::= set_comp_header
                           LOAD_FAST FOR_ITER store comp_iter
                           JUMP_BACK POP_TOP JUMP_BACK RETURN_VALUE RETURN_LAST
 
@@ -62,8 +82,10 @@ class Python30Parser(Python31Parser):
         list_comp        ::= list_comp_header
                              LOAD_FAST FOR_ITER store comp_iter
                              JUMP_BACK
+        list_comp        ::= list_comp_header
+                             LOAD_FAST FOR_ITER store comp_iter
+                             JUMP_BACK _come_froms POP_TOP JUMP_BACK
 
-        set_comp_header  ::= BUILD_SET_0 DUP_TOP STORE_FAST
         set_comp         ::= set_comp_header
                              LOAD_FAST FOR_ITER store comp_iter
                              JUMP_BACK
@@ -72,6 +94,24 @@ class Python30Parser(Python31Parser):
         dict_comp        ::= dict_comp_header
                              LOAD_FAST FOR_ITER store dict_comp_iter
                              JUMP_BACK
+        dict_comp        ::= dict_comp_header
+                             LOAD_FAST FOR_ITER store dict_comp_iter
+                             JUMP_BACK _come_froms POP_TOP JUMP_BACK
+
+        stmt         ::= try_except30
+        try_except30 ::= SETUP_EXCEPT suite_stmts_opt
+                        _come_froms pt_bp
+                         except_handler opt_come_from_except
+
+        # From Python 2.6
+
+
+        list_iter  ::= list_if JUMP_BACK
+        list_iter  ::= list_if JUMP_BACK _come_froms POP_TOP
+        lc_body    ::= LOAD_NAME expr LIST_APPEND
+	lc_body    ::= LOAD_FAST expr LIST_APPEND
+        list_if    ::= expr jmp_false_then list_iter
+        #############
 
         dict_comp_iter   ::= expr expr ROT_TWO expr STORE_SUBSCR
 
@@ -87,19 +127,52 @@ class Python30Parser(Python31Parser):
         except_suite          ::= c_stmts POP_EXCEPT jump_except POP_TOP
         except_suite_finalize ::= SETUP_FINALLY c_stmts_opt except_var_finalize END_FINALLY
                                   _jump COME_FROM POP_TOP
-        jump_except           ::= JUMP_FORWARD COME_FROM POP_TOP
+
-        jump_except           ::= JUMP_ABSOLUTE COME_FROM POP_TOP
+        except_handler        ::= jmp_abs COME_FROM_EXCEPT except_stmts END_FINALLY
+
+        _ifstmts_jump         ::= c_stmts_opt JUMP_FORWARD COME_FROM POP_TOP
+        _ifstmts_jump         ::= c_stmts_opt come_froms POP_TOP JUMP_FORWARD _come_froms
+
+        jump_except           ::= _jump COME_FROM POP_TOP
+
         or                    ::= expr jmp_false expr jmp_true expr
         or                    ::= expr jmp_true expr
 
+        import_from ::= LOAD_CONST LOAD_CONST IMPORT_NAME importlist _come_froms POP_TOP
+
         ################################################################################
         # In many ways 3.0 is like 2.6. One similarity is there is no JUMP_IF_TRUE and
         # JUMP_IF_FALSE
         # The below rules in fact are the same or similar.
 
-        jmp_true       ::= JUMP_IF_TRUE POP_TOP
+        jmp_true         ::= JUMP_IF_TRUE POP_TOP
-        jmp_false      ::= JUMP_IF_FALSE _come_froms POP_TOP
+        jmp_true_then    ::= JUMP_IF_TRUE _come_froms POP_TOP
+        jmp_false        ::= JUMP_IF_FALSE _come_froms POP_TOP
+        jmp_false_then   ::= JUMP_IF_FALSE POP_TOP
 
+        # We don't have hacky THEN detection, so we do it
+        # in the grammar below which is also somewhat hacky.
+
+        stmt             ::= ifstmt30
+        stmt             ::= ifnotstmt30
+        ifstmt30         ::= testfalse_then _ifstmts_jump30
+        ifnotstmt30      ::= testtrue_then  _ifstmts_jump30
+
+        testfalse_then   ::= expr jmp_false_then
+        testtrue_then    ::= expr jmp_true_then
+        call_stmt        ::= expr COME_FROM
+        _ifstmts_jump30  ::= c_stmts POP_TOP
+
+        gen_comp_body    ::= expr YIELD_VALUE COME_FROM POP_TOP
+
+        except_handler   ::= jmp_abs COME_FROM_EXCEPT except_stmts
+                             COME_FROM POP_TOP END_FINALLY
+
+        or               ::= expr jmp_true_then expr come_from_opt
+        ret_or           ::= expr jmp_true_then expr come_from_opt
+        ret_and          ::= expr jump_false expr come_from_opt
+
+        ################################################################################
         for_block      ::= l_stmts_opt _come_froms POP_TOP JUMP_BACK
 
         except_handler ::= JUMP_FORWARD COME_FROM_EXCEPT except_stmts
@@ -107,12 +180,17 @@ class Python30Parser(Python31Parser):
         except_handler ::= jmp_abs COME_FROM_EXCEPT except_stmts
                            POP_TOP END_FINALLY
 
-        return_if_stmt ::= ret_expr RETURN_END_IF COME_FROM POP_TOP
+        return_if_stmt ::= ret_expr RETURN_END_IF come_froms POP_TOP
-        and            ::= expr jmp_false expr come_from_opt
+        return_if_stmt ::= ret_expr RETURN_VALUE come_froms POP_TOP
+
+        and            ::= expr jmp_false_then expr come_from_opt
+
         whilestmt      ::= SETUP_LOOP testexpr l_stmts_opt come_from_opt
-                           JUMP_BACK COME_FROM POP_TOP POP_BLOCK COME_FROM_LOOP
+                           JUMP_BACK _come_froms POP_TOP POP_BLOCK COME_FROM_LOOP
         whilestmt      ::= SETUP_LOOP testexpr returns
                            POP_TOP POP_BLOCK COME_FROM_LOOP
+        whilestmt      ::= SETUP_LOOP testexpr l_stmts_opt come_from_opt
+                           come_froms POP_TOP POP_BLOCK COME_FROM_LOOP
 
 
         # compare_chained is like x <= y <= z
@@ -123,34 +201,149 @@ class Python30Parser(Python31Parser):
         compare_chained2 ::= expr COMPARE_OP RETURN_END_IF
         """
 
-    def customize_grammar_rules(self, tokens, customize):
+
-        super(Python30Parser, self).customize_grammar_rules(tokens, customize)
+    def remove_rules_30(self):
         self.remove_rules("""
         iflaststmtl        ::= testexpr c_stmts_opt JUMP_BACK COME_FROM_LOOP
         ifelsestmtl        ::= testexpr c_stmts_opt JUMP_BACK else_suitel
         iflaststmt         ::= testexpr c_stmts_opt JUMP_ABSOLUTE
         _ifstmts_jump      ::= c_stmts_opt JUMP_FORWARD _come_froms
+
         jump_forward_else  ::= JUMP_FORWARD ELSE
         jump_absolute_else ::= JUMP_ABSOLUTE ELSE
         whilestmt          ::= SETUP_LOOP testexpr l_stmts_opt COME_FROM JUMP_BACK POP_BLOCK
                                COME_FROM_LOOP
         whilestmt          ::= SETUP_LOOP testexpr returns
                                POP_BLOCK COME_FROM_LOOP
+
         assert             ::= assert_expr jmp_true LOAD_ASSERT RAISE_VARARGS_1
+
         return_if_lambda   ::= RETURN_END_IF_LAMBDA
         except_suite       ::= c_stmts POP_EXCEPT jump_except
         whileelsestmt      ::= SETUP_LOOP testexpr l_stmts JUMP_BACK POP_BLOCK
                                else_suitel COME_FROM_LOOP
 
-        # No JUMP_IF_FALSE_OR_POP
+        # No JUMP_IF_FALSE_OR_POP, JUMP_IF_TRUE_OR_POP,
+        # POP_JUMP_IF_FALSE, or POP_JUMP_IF_TRUE
+
+        jmp_false        ::= POP_JUMP_IF_FALSE
+        jmp_true         ::= JUMP_IF_TRUE_OR_POP POP_TOP
+        jmp_true         ::= POP_JUMP_IF_TRUE
+
         compare_chained1 ::= expr DUP_TOP ROT_THREE COMPARE_OP JUMP_IF_FALSE_OR_POP
                              compare_chained1 COME_FROM
         compare_chained1 ::= expr DUP_TOP ROT_THREE COMPARE_OP JUMP_IF_FALSE_OR_POP
                              compare_chained2 COME_FROM
+        ret_or           ::= expr JUMP_IF_TRUE_OR_POP  ret_expr_or_cond COME_FROM
+        ret_and          ::= expr JUMP_IF_FALSE_OR_POP ret_expr_or_cond COME_FROM
+        ret_cond         ::= expr POP_JUMP_IF_FALSE expr RETURN_END_IF
+                             COME_FROM ret_expr_or_cond
+        ret_expr_or_cond ::= ret_cond
+        or               ::= expr JUMP_IF_TRUE_OR_POP expr COME_FROM
+        and              ::= expr JUMP_IF_TRUE_OR_POP expr COME_FROM
+        and              ::= expr JUMP_IF_FALSE_OR_POP expr COME_FROM
         """)
 
+    def customize_grammar_rules(self, tokens, customize):
+        super(Python30Parser, self).customize_grammar_rules(tokens, customize)
+        self.remove_rules_30()
+
+        self.check_reduce["iflaststmtl"] = "AST"
+        self.check_reduce['ifstmt'] = "AST"
+        self.check_reduce["ifelsestmtc"] = "AST"
+        self.check_reduce["ifelsestmt"] = "AST"
+        # self.check_reduce["and"] = "stmt"
         return
+
+    def reduce_is_invalid(self, rule, ast, tokens, first, last):
+        invalid = super(Python30Parser,
+                        self).reduce_is_invalid(rule, ast,
+                                                tokens, first, last)
+        if invalid:
+            return invalid
+        lhs = rule[0]
+        if (
+            lhs in ("iflaststmtl", "ifstmt",
+                        "ifelsestmt", "ifelsestmtc") and ast[0] == "testexpr"
+        ):
+            testexpr = ast[0]
+            if testexpr[0] == "testfalse":
+                testfalse = testexpr[0]
+                if lhs == "ifelsestmtc" and ast[2] == "jump_absolute_else":
+                    jump_absolute_else = ast[2]
+                    come_from = jump_absolute_else[2]
+                    return come_from == "COME_FROM" and come_from.attr < tokens[first].offset
+                    pass
+                elif lhs in ("ifelsestmt", "ifelsestmtc") and ast[2] == "jump_cf_pop":
+                    jump_cf_pop = ast[2]
+                    come_froms = jump_cf_pop[0]
+                    for come_from in come_froms:
+                        if come_from.attr < tokens[first].offset:
+                            return True
+                    come_froms = jump_cf_pop[2]
+                    if come_froms == "COME_FROM":
+                        if come_froms.attr < tokens[first].offset:
+                            return True
+                        pass
+                    elif come_froms == "_come_froms":
+                        for come_from in come_froms:
+                            if come_from.attr < tokens[first].offset:
+                                return True
+
+                    return False
+                elif testfalse[1] == "jmp_false":
+                    jmp_false = testfalse[1]
+                    if last == len(tokens):
+                        last -= 1
+                    while (isinstance(tokens[first].offset, str) and first < last):
+                        first += 1
+                    if first == last:
+                        return True
+                    while (first < last and isinstance(tokens[last].offset, str)):
+                        last -= 1
+                    if rule[0] == "iflaststmtl":
+                        return not (jmp_false[0].attr <= tokens[last].offset)
+                    else:
+                        jmp_false_target = jmp_false[0].attr
+                        if tokens[first].offset > jmp_false_target:
+                            return True
+                        return (
+                            (jmp_false_target > tokens[last].offset) and tokens[last] != "JUMP_FORWARD")
+                    pass
+                pass
+            pass
+        # elif lhs == "and":
+        #     return tokens[last+1] == "JUMP_FORWARD"
+
     pass
 
 class Python30ParserSingle(Python30Parser, PythonParserSingle):
     pass
+
+if __name__ == '__main__':
+    # Check grammar
+    p = Python30Parser()
+    p.remove_rules_30()
+    p.check_grammar()
+    from uncompyle6 import PYTHON_VERSION, IS_PYPY
+    if PYTHON_VERSION == 3.0:
+        lhs, rhs, tokens, right_recursive, dup_rhs = p.check_sets()
+        from uncompyle6.scanner import get_scanner
+        s = get_scanner(PYTHON_VERSION, IS_PYPY)
+        opcode_set = set(s.opc.opname).union(set(
+            """JUMP_BACK CONTINUE RETURN_END_IF COME_FROM
+               LOAD_GENEXPR LOAD_ASSERT LOAD_SETCOMP LOAD_DICTCOMP LOAD_CLASSNAME
+               LAMBDA_MARKER RETURN_LAST
+            """.split()))
+        ## FIXME: try this
+        remain_tokens = set(tokens) - opcode_set
+        import re
+        remain_tokens = set([re.sub(r'_\d+$', '',  t) for t in remain_tokens])
+        remain_tokens = set([re.sub('_CONT$', '', t) for t in remain_tokens])
+        remain_tokens = set(remain_tokens) - opcode_set
+        print(remain_tokens)
+        import sys
+        if len(sys.argv) > 1:
+            from spark_parser.spark import rule2str
+            for rule in sorted(p.rule2name.items()):
+                print(rule2str(rule[0]))

uncompyle6/parsers/parse34.py

@@ -72,7 +72,7 @@ if __name__ == '__main__':
     p.check_grammar()
     from uncompyle6 import PYTHON_VERSION, IS_PYPY
     if PYTHON_VERSION == 3.4:
-        lhs, rhs, tokens, right_recursive = p.check_sets()
+        lhs, rhs, tokens, right_recursive, dup_rhs = p.check_sets()
         from uncompyle6.scanner import get_scanner
         s = get_scanner(PYTHON_VERSION, IS_PYPY)
         opcode_set = set(s.opc.opname).union(set(

uncompyle6/parsers/parse35.py

@@ -262,7 +262,7 @@ if __name__ == '__main__':
     p.check_grammar()
     from uncompyle6 import PYTHON_VERSION, IS_PYPY
     if PYTHON_VERSION == 3.5:
-        lhs, rhs, tokens, right_recursive = p.check_sets()
+        lhs, rhs, tokens, right_recursive, dup_rhs = p.check_sets()
         from uncompyle6.scanner import get_scanner
         s = get_scanner(PYTHON_VERSION, IS_PYPY)
         opcode_set = set(s.opc.opname).union(set(

uncompyle6/parsers/parse36.py

@@ -440,7 +440,7 @@ if __name__ == '__main__':
     p.check_grammar()
     from uncompyle6 import PYTHON_VERSION, IS_PYPY
     if PYTHON_VERSION == 3.6:
-        lhs, rhs, tokens, right_recursive = p.check_sets()
+        lhs, rhs, tokens, right_recursive, dup_rhs = p.check_sets()
         from uncompyle6.scanner import get_scanner
         s = get_scanner(PYTHON_VERSION, IS_PYPY)
         opcode_set = set(s.opc.opname).union(set(

uncompyle6/parsers/parse37.py

@@ -140,31 +140,6 @@ class Python37Parser(Python36Parser):
         """
 
     def customize_grammar_rules(self, tokens, customize):
-        self.remove_rules("""
-          async_forelse_stmt ::= SETUP_LOOP expr
-                                 GET_AITER
-                                 LOAD_CONST YIELD_FROM SETUP_EXCEPT GET_ANEXT LOAD_CONST
-                                 YIELD_FROM
-                                 store
-                                 POP_BLOCK JUMP_FORWARD COME_FROM_EXCEPT DUP_TOP
-                                 LOAD_GLOBAL COMPARE_OP POP_JUMP_IF_FALSE
-                                 POP_TOP POP_TOP POP_TOP POP_EXCEPT POP_BLOCK
-                                 JUMP_ABSOLUTE END_FINALLY COME_FROM
-                                 for_block POP_BLOCK
-                                 else_suite COME_FROM_LOOP
-        stmt      ::= async_for_stmt36
-        async_for_stmt36   ::= SETUP_LOOP expr
-                               GET_AITER
-                               LOAD_CONST YIELD_FROM SETUP_EXCEPT GET_ANEXT LOAD_CONST
-                               YIELD_FROM
-                               store
-                               POP_BLOCK JUMP_BACK COME_FROM_EXCEPT DUP_TOP
-                               LOAD_GLOBAL COMPARE_OP POP_JUMP_IF_TRUE
-                               END_FINALLY continues COME_FROM
-                               POP_TOP POP_TOP POP_TOP POP_EXCEPT
-                               POP_TOP POP_BLOCK
-                               COME_FROM_LOOP
-        """)
         super(Python37Parser, self).customize_grammar_rules(tokens, customize)
 
 class Python37ParserSingle(Python37Parser, PythonParserSingle):
@@ -172,22 +147,33 @@ class Python37ParserSingle(Python37Parser, PythonParserSingle):
 
 if __name__ == '__main__':
     # Check grammar
+    # FIXME: DRY this with other parseXX.py routines
     p = Python37Parser()
     p.check_grammar()
     from uncompyle6 import PYTHON_VERSION, IS_PYPY
+
     if PYTHON_VERSION == 3.7:
-        lhs, rhs, tokens, right_recursive = p.check_sets()
+        lhs, rhs, tokens, right_recursive, dup_rhs = p.check_sets()
         from uncompyle6.scanner import get_scanner
+
         s = get_scanner(PYTHON_VERSION, IS_PYPY)
-        opcode_set = set(s.opc.opname).union(set(
+        opcode_set = set(s.opc.opname).union(
-            """JUMP_BACK CONTINUE RETURN_END_IF COME_FROM
+            set(
+                """JUMP_BACK CONTINUE RETURN_END_IF COME_FROM
                LOAD_GENEXPR LOAD_ASSERT LOAD_SETCOMP LOAD_DICTCOMP LOAD_CLASSNAME
                LAMBDA_MARKER RETURN_LAST
-            """.split()))
+            """.split()
+            )
+        )
         remain_tokens = set(tokens) - opcode_set
         import re
-        remain_tokens = set([re.sub(r'_\d+$', '', t) for t in remain_tokens])
+
-        remain_tokens = set([re.sub('_CONT$', '', t) for t in remain_tokens])
+        remain_tokens = set([re.sub(r"_\d+$", "", t) for t in remain_tokens])
+        remain_tokens = set([re.sub("_CONT$", "", t) for t in remain_tokens])
         remain_tokens = set(remain_tokens) - opcode_set
         print(remain_tokens)
-        # print(sorted(p.rule2name.items()))
+        import sys
+        if len(sys.argv) > 1:
+            from spark_parser.spark import rule2str
+            for rule in sorted(p.rule2name.items()):
+                print(rule2str(rule[0]))

uncompyle6/parsers/parse38.py

@@ -159,7 +159,7 @@ class Python38Parser(Python37Parser):
         super(Python38Parser, self).__init__(debug_parser)
         self.customized = {}
 
-    def customize_grammar_rules(self, tokens, customize):
+    def remove_rules_38(self):
         self.remove_rules("""
            stmt               ::= async_for_stmt37
            stmt               ::= for
@@ -226,7 +226,10 @@ class Python38Parser(Python37Parser):
 
 
         """)
+
+    def customize_grammar_rules(self, tokens, customize):
         super(Python37Parser, self).customize_grammar_rules(tokens, customize)
+        self.remove_rules_38()
         self.check_reduce['ifstmt'] = 'tokens'
         self.check_reduce['whileTruestmt38'] = 'tokens'
 
@@ -234,6 +237,7 @@ class Python38Parser(Python37Parser):
         invalid = super(Python38Parser,
                         self).reduce_is_invalid(rule, ast,
                                                 tokens, first, last)
+        self.remove_rules_38()
         if invalid:
             return invalid
         if rule[0] == 'ifstmt':
@@ -264,24 +268,34 @@ class Python38Parser(Python37Parser):
 class Python38ParserSingle(Python38Parser, PythonParserSingle):
     pass
 
-if __name__ == '__main__':
+if __name__ == "__main__":
     # Check grammar
+    # FIXME: DRY this with other parseXX.py routines
     p = Python38Parser()
+    p.remove_rules_38()
     p.check_grammar()
     from uncompyle6 import PYTHON_VERSION, IS_PYPY
+
     if PYTHON_VERSION == 3.8:
-        lhs, rhs, tokens, right_recursive = p.check_sets()
+        lhs, rhs, tokens, right_recursive, dup_rhs = p.check_sets()
         from uncompyle6.scanner import get_scanner
+
         s = get_scanner(PYTHON_VERSION, IS_PYPY)
-        opcode_set = set(s.opc.opname).union(set(
+        opcode_set = set(s.opc.opname).union(
-            """JUMP_BACK CONTINUE RETURN_END_IF COME_FROM
+            set(
-               LOAD_GENEXPR LOAD_ASSERT LOAD_SETCOMP LOAD_DICTCOMP LOAD_CLASSNAME
+                """JUMP_BACK CONTINUE RETURN_END_IF COME_FROM
-               LAMBDA_MARKER RETURN_LAST
+                LOAD_GENEXPR LOAD_ASSERT LOAD_SETCOMP LOAD_DICTCOMP LOAD_CLASSNAME
+                LAMBDA_MARKER RETURN_LAST
             """.split()))
         remain_tokens = set(tokens) - opcode_set
         import re
-        remain_tokens = set([re.sub(r'_\d+$', '', t) for t in remain_tokens])
+
-        remain_tokens = set([re.sub('_CONT$', '', t) for t in remain_tokens])
+        remain_tokens = set([re.sub(r"_\d+$", "", t) for t in remain_tokens])
+        remain_tokens = set([re.sub("_CONT$", "", t) for t in remain_tokens])
         remain_tokens = set(remain_tokens) - opcode_set
         print(remain_tokens)
-        # print(sorted(p.rule2name.items()))
+        import sys
+        if len(sys.argv) > 1:
+            from spark_parser.spark import rule2str
+            for rule in sorted(p.rule2name.items()):
+                print(rule2str(rule[0]))

uncompyle6/parsers/treenode.py

@@ -54,3 +54,11 @@ class SyntaxTree(spark_AST):
             rv += "\n" + child
             i += 1
         return rv
+
+    def first_child(self):
+        if len(self) > 0:
+            child = self[0]
+            if not isinstance(child, SyntaxTree):
+                return child
+            return self[0].first_child()
+        return self

uncompyle6/scanners/scanner3.py

@@ -265,21 +265,29 @@ class Scanner3(Scanner):
             # RAISE_VARARGS then we have a "raise" statement
             # else we have an "assert" statement.
             if self.version == 3.0:
-                # There is a an implied JUMP_IF_TRUE that we are not testing for (yet?) here
+                # Like 2.6, 3.0 doesn't have POP_JUMP_IF... so we have
+                # to go through more machinations
                 assert_can_follow = inst.opname == "POP_TOP" and i + 1 < n
+                if assert_can_follow:
+                    prev_inst = self.insts[i - 1]
+                    assert_can_follow = (
+                        prev_inst.opname in ("JUMP_IF_TRUE", "JUMP_IF_FALSE")
+                        and i + 1 < n )
+                    jump_if_inst = prev_inst
             else:
                 assert_can_follow = (
                     inst.opname in ("POP_JUMP_IF_TRUE", "POP_JUMP_IF_FALSE")
                     and i + 1 < n
                 )
+                jump_if_inst = inst
             if assert_can_follow:
                 next_inst = self.insts[i + 1]
                 if (
                     next_inst.opname == "LOAD_GLOBAL"
                     and next_inst.argval == "AssertionError"
-                    and inst.argval
+                    and jump_if_inst.argval
                 ):
-                    raise_idx = self.offset2inst_index[self.prev_op[inst.argval]]
+                    raise_idx = self.offset2inst_index[self.prev_op[jump_if_inst.argval]]
                     raise_inst = self.insts[raise_idx]
                     if raise_inst.opname.startswith("RAISE_VARARGS"):
                         self.load_asserts.add(next_inst.offset)
@@ -482,6 +490,12 @@ class Scanner3(Scanner):
                         and self.insts[i + 1].opname == "JUMP_FORWARD"
                     )
 
+                    if (self.version == 3.0 and self.insts[i + 1].opname == "JUMP_FORWARD"
+                        and not is_continue):
+                        target_prev = self.offset2inst_index[self.prev_op[target]]
+                        is_continue = (
+                            self.insts[target_prev].opname == "SETUP_LOOP")
+
                     if is_continue or (
                         inst.offset in self.stmts
                         and (

uncompyle6/semantics/customize26_27.py

@@ -40,6 +40,7 @@ def customize_for_version26_27(self, version):
             'testtrue_then': ( 'not %p', (0, 22) ),
         })
 
+    # FIXME: this should be a transformation
     def n_call(node):
         mapping = self._get_mapping(node)
         key = node

uncompyle6/semantics/customize3.py

@@ -181,6 +181,19 @@ def customize_for_version3(self, version):
         # the iteration variable. These rules we can ignore
         # since we pick up the iteration variable some other way and
         # we definitely don't include in the source  _[dd].
+        TABLE_DIRECT.update({
+            "ifstmt30":	( "%|if %c:\n%+%c%-",
+                          (0, "testfalse_then"),
+                          (1, "_ifstmts_jump30") ),
+            "ifnotstmt30": ( "%|if not %c:\n%+%c%-",
+                             (0, "testtrue_then"),
+                             (1, "_ifstmts_jump30") ),
+            "try_except30": ( "%|try:\n%+%c%-%c\n\n",
+                              (1, "suite_stmts_opt"),
+                              (4, "except_handler") ),
+
+            })
+
         def n_comp_iter(node):
             if node[0] == "expr":
                 n = node[0][0]
@@ -197,11 +210,14 @@ def customize_for_version3(self, version):
         # FIXME: perhaps this can be folded into the 3.4+ case?
         def n_yield_from(node):
             assert node[0] == "expr"
-            assert node[0][0] == "get_iter"
+            if node[0][0] == "get_iter":
-            # Skip over yield_from.expr.get_iter which adds an
+                # Skip over yield_from.expr.get_iter which adds an
-            # extra iter(). Maybe we can do in tranformation phase instead?
+                # extra iter(). Maybe we can do in tranformation phase instead?
-            template = ("yield from %c", (0, "expr"))
+                template = ("yield from %c", (0, "expr"))
-            self.template_engine(template, node[0][0])
+                self.template_engine(template, node[0][0])
+            else:
+                template = ("yield from %c", (0, "attribute"))
+                self.template_engine(template, node[0][0][0])
             self.prune()
 
         self.n_yield_from = n_yield_from

uncompyle6/semantics/fragments.py

@@ -792,7 +792,7 @@ class FragmentsWalker(pysource.SourceWalker, object):
         self.name = code_name
 
         # Issue created with later Python code generation is that there
-        # is a lamda set up with a dummy argument name that is then called
+        # is a lambda set up with a dummy argument name that is then called
         # So we can't just translate that as is but need to replace the
         # dummy name. Below we are picking out the variable name as seen
         # in the code. And trying to generate code for the other parts

uncompyle6/semantics/transform.py

@@ -115,11 +115,14 @@ class TreeTransform(GenericASTTraversal, object):
 
                     call = expr[0]
                     LOAD_ASSERT = call[0]
-                    expr = call[1][0]
+                    if isinstance(call[1], SyntaxTree):
-                    node = SyntaxTree(
+                        expr = call[1][0]
-                        kind,
+                        node = SyntaxTree(
-                        [assert_expr, jump_cond, LOAD_ASSERT, expr, RAISE_VARARGS_1]
+                            kind,
-                    )
+                            [assert_expr, jump_cond, LOAD_ASSERT, expr, RAISE_VARARGS_1]
+                        )
+                        pass
+                    pass
                 else:
                     # ifstmt
                     #   0. testexpr (2)