Port over some recent decompyle3 3.8 fixes

test/simple_source/bug36/10_fstring.py

@@ -14,6 +14,12 @@ assert (
 assert "def0" == f"{abc}0"
 assert "defdef" == f"{abc}{abc!s}"
 
+# From 3.8 test/test_string.py
+# We had the precedence of yield vs. lambda incorrect.
+def fn(x):
+    yield f"x:{yield (lambda i: x * i)}"
+
+
 # From 3.6 functools.py
 # Bug was handling format operator strings.
 

test/simple_source/bug38/00_bug_dict_comp.py

@@ -0,0 +1,12 @@
+# Issue 104
+# Python 3.8 reverses the order or keys and values in
+# dictionary comprehensions from the order in all previous Pythons.
+# Also we were looking in the wrong place for the collection of the
+# dictionary comprehension
+# RUNNABLE!
+
+"""This program is self-checking!"""
+x = [(0, [1]), (2, [3])]
+for i in range(0, 1):
+    y = {key: val[i - 1] for (key, val) in x}
+assert y == {0: 1, 2: 3}

uncompyle6/parsers/parse38.py

@@ -17,7 +17,7 @@ spark grammar differences over Python 3.7 for Python 3.8
 """
 from __future__ import print_function
 
-from uncompyle6.parser import PythonParserSingle
+from uncompyle6.parser import PythonParserSingle, nop_func
 from spark_parser import DEFAULT_DEBUG as PARSER_DEFAULT_DEBUG
 from uncompyle6.parsers.parse37 import Python37Parser
 
@@ -367,6 +367,195 @@ class Python38Parser(Python37Parser):
         self.check_reduce["whilestmt38"] = "tokens"
         self.check_reduce["try_elsestmtl38"] = "AST"
 
+        # For a rough break out on the first word. This may
+        # include instructions that don't need customization,
+        # but we'll do a finer check after the rough breakout.
+        customize_instruction_basenames = frozenset(
+            (
+                "BEFORE",
+                "BUILD",
+                "CALL",
+                "DICT",
+                "GET",
+                "FORMAT",
+                "LIST",
+                "LOAD",
+                "MAKE",
+                "SETUP",
+                "UNPACK",
+            )
+        )
+
+        # Opcode names in the custom_ops_processed set have rules that get added
+        # unconditionally and the rules are constant. So they need to be done
+        # only once and if we see the opcode a second we don't have to consider
+        # adding more rules.
+        #
+        custom_ops_processed = frozenset()
+
+        # A set of instruction operation names that exist in the token stream.
+        # We use this customize the grammar that we create.
+        # 2.6-compatible set comprehensions
+        self.seen_ops = frozenset([t.kind for t in tokens])
+        self.seen_op_basenames = frozenset(
+            [opname[: opname.rfind("_")] for opname in self.seen_ops]
+        )
+
+        custom_ops_processed = set(["DICT_MERGE"])
+
+        # Loop over instructions adding custom grammar rules based on
+        # a specific instruction seen.
+
+        if "PyPy" in customize:
+            self.addRule(
+                """
+              stmt ::= assign3_pypy
+              stmt ::= assign2_pypy
+              assign3_pypy       ::= expr expr expr store store store
+              assign2_pypy       ::= expr expr store store
+              """,
+                nop_func,
+            )
+
+        n = len(tokens)
+        # Determine if we have an iteration CALL_FUNCTION_1.
+        has_get_iter_call_function1 = False
+        for i, token in enumerate(tokens):
+            if (
+                token == "GET_ITER"
+                and i < n - 2
+                and tokens[i + 1] == "CALL_FUNCTION_1"
+            ):
+                has_get_iter_call_function1 = True
+
+        for i, token in enumerate(tokens):
+            opname = token.kind
+
+            # Do a quick breakout before testing potentially
+            # each of the dozen or so instruction in if elif.
+            if (
+                opname[: opname.find("_")] not in customize_instruction_basenames
+                or opname in custom_ops_processed
+            ):
+                continue
+
+            opname_base = opname[: opname.rfind("_")]
+
+            # Do a quick breakout before testing potentially
+            # each of the dozen or so instruction in if elif.
+            if (
+                opname[: opname.find("_")] not in customize_instruction_basenames
+                or opname in custom_ops_processed
+            ):
+                continue
+            if opname_base in (
+                "BUILD_LIST",
+                "BUILD_SET",
+                "BUILD_SET_UNPACK",
+                "BUILD_TUPLE",
+                "BUILD_TUPLE_UNPACK",
+            ):
+                v = token.attr
+
+                is_LOAD_CLOSURE = False
+                if opname_base == "BUILD_TUPLE":
+                    # If is part of a "load_closure", then it is not part of a
+                    # "list".
+                    is_LOAD_CLOSURE = True
+                    for j in range(v):
+                        if tokens[i - j - 1].kind != "LOAD_CLOSURE":
+                            is_LOAD_CLOSURE = False
+                            break
+                    if is_LOAD_CLOSURE:
+                        rule = "load_closure ::= %s%s" % (("LOAD_CLOSURE " * v), opname)
+                        self.add_unique_rule(rule, opname, token.attr, customize)
+
+                elif opname_base == "BUILD_LIST":
+                    v = token.attr
+                    if v == 0:
+                        rule_str = """
+                           list        ::= BUILD_LIST_0
+                           list_unpack ::= BUILD_LIST_0 expr LIST_EXTEND
+                           list        ::= list_unpack
+                        """
+                        self.add_unique_doc_rules(rule_str, customize)
+
+                elif opname == "BUILD_TUPLE_UNPACK_WITH_CALL":
+                    # FIXME: should this be parameterized by EX value?
+                    self.addRule(
+                        """expr        ::= call_ex_kw3
+                           call_ex_kw3 ::= expr
+                                           build_tuple_unpack_with_call
+                                           expr
+                                           CALL_FUNCTION_EX_KW
+                        """,
+                        nop_func,
+                    )
+
+                if not is_LOAD_CLOSURE or v == 0:
+                    # We do this complicated test to speed up parsing of
+                    # pathelogically long literals, especially those over 1024.
+                    build_count = token.attr
+                    thousands = build_count // 1024
+                    thirty32s = (build_count // 32) % 32
+                    if thirty32s > 0:
+                        rule = "expr32 ::=%s" % (" expr" * 32)
+                        self.add_unique_rule(rule, opname_base, build_count, customize)
+                        pass
+                    if thousands > 0:
+                        self.add_unique_rule(
+                            "expr1024 ::=%s" % (" expr32" * 32),
+                            opname_base,
+                            build_count,
+                            customize,
+                        )
+                        pass
+                    collection = opname_base[opname_base.find("_") + 1 :].lower()
+                    rule = (
+                        ("%s ::= " % collection)
+                        + "expr1024 " * thousands
+                        + "expr32 " * thirty32s
+                        + "expr " * (build_count % 32)
+                        + opname
+                    )
+                    self.add_unique_rules(["expr ::= %s" % collection, rule], customize)
+                    continue
+                continue
+            elif opname == "LOAD_CLOSURE":
+                self.addRule("""load_closure ::= LOAD_CLOSURE+""", nop_func)
+
+            elif opname == "LOOKUP_METHOD":
+                # A PyPy speciality - DRY with parse3
+                self.addRule(
+                    """
+                             expr      ::= attribute
+                             attribute ::= expr LOOKUP_METHOD
+                             """,
+                    nop_func,
+                )
+                custom_ops_processed.add(opname)
+
+            elif opname == "MAKE_FUNCTION_8":
+                if "LOAD_DICTCOMP" in self.seen_ops:
+                    # Is there something general going on here?
+                    rule = """
+                       dict_comp ::= load_closure LOAD_DICTCOMP LOAD_STR
+                                     MAKE_FUNCTION_8 expr
+                                     GET_ITER CALL_FUNCTION_1
+                       """
+                    self.addRule(rule, nop_func)
+                elif "LOAD_SETCOMP" in self.seen_ops:
+                    rule = """
+                       set_comp ::= load_closure LOAD_SETCOMP LOAD_STR
+                                    MAKE_FUNCTION_CLOSURE expr
+                                    GET_ITER CALL_FUNCTION_1
+                       """
+                    self.addRule(rule, nop_func)
+
+
+
+
+
     def reduce_is_invalid(self, rule, ast, tokens, first, last):
         invalid = super(Python38Parser,
                         self).reduce_is_invalid(rule, ast,

uncompyle6/semantics/consts.py

@@ -50,14 +50,15 @@ NO_PARENTHESIS_EVER = 100
 # fmt: off
 PRECEDENCE = {
     "named_expr":             40,  # :=
-    "yield":                  38,  # Needs to be below named_expr
-    "yield_from":             38,
-    "tuple_list_starred":     38,  # *x, *y, *z - about at the level of yield?
     "dict_unpack":            38,  # **kwargs
     "list_unpack":            38,  # *args
+    "yield_from":             38,
+    "tuple_list_starred":     38,  # *x, *y, *z - about at the level of yield?
 
     "_lambda_body":           30,
-    "lambda_body":            30,  # lambda ... : lambda_body
+    "lambda_body":            32,  # lambda ... : lambda_body
+
+    "yield":                  30,  # Needs to be below named_expr and lambda_body
 
     "if_exp":                 28,  # IfExp ( a if x else b)
     "if_exp_lambda":          28,  # IfExp involving a lambda expression

uncompyle6/semantics/customize38.py

@@ -66,6 +66,9 @@ def customize_for_version38(self, version):
                 (1, "c_suite_stmts_opt"),
                 (-2, "c_suite_stmts_opt"),
             ),
+            # Python 3.8 reverses the order of keys and items
+            # from all prior versions of Python.
+            "dict_comp_body": ("%c: %c", (0, "expr"), (1, "expr"),),
             "except_cond1a": ("%|except %c:\n", (1, "expr"),),
             "except_cond_as": (
                 "%|except %c as %c:\n",

uncompyle6/semantics/gencomp.py

@@ -302,6 +302,9 @@ class ComprehensionMixin:
             store = set_iter_async[1]
             assert store == "store"
             n = set_iter_async[2]
+        elif node == "list_comp" and tree[0] == "expr":
+            tree = tree[0][0]
+            n = tree[iter_index]
         else:
             n = tree[iter_index]