From 85ba8352ba0e8a147e954b6b499daac355d09eda Mon Sep 17 00:00:00 2001 From: rocky Date: Sun, 26 Jun 2022 04:26:15 -0400 Subject: [PATCH] Port over some recent decompyle3 3.8 fixes --- test/bytecode_3.8_run/00_bug_dict_comp.pyc | Bin 0 -> 356 bytes test/simple_source/bug36/10_fstring.py | 6 + test/simple_source/bug38/00_bug_dict_comp.py | 12 ++ uncompyle6/parsers/parse38.py | 191 ++++++++++++++++++- uncompyle6/semantics/consts.py | 9 +- uncompyle6/semantics/customize38.py | 3 + uncompyle6/semantics/gencomp.py | 3 + 7 files changed, 219 insertions(+), 5 deletions(-) create mode 100644 test/bytecode_3.8_run/00_bug_dict_comp.pyc create mode 100644 test/simple_source/bug38/00_bug_dict_comp.py diff --git a/test/bytecode_3.8_run/00_bug_dict_comp.pyc b/test/bytecode_3.8_run/00_bug_dict_comp.pyc new file mode 100644 index 0000000000000000000000000000000000000000..8fd6638c79e6085c095b1242594adb5ab8dcebca GIT binary patch literal 356 zcmYjN!AiqG5S`ggqE=c&MD)_D2&>hg*}fx-wMg%V2g9tj^b zvbqsdm>v5$@G+eTji literal 0 HcmV?d00001 diff --git a/test/simple_source/bug36/10_fstring.py b/test/simple_source/bug36/10_fstring.py index ef38dabf..876f15e9 100644 --- a/test/simple_source/bug36/10_fstring.py +++ b/test/simple_source/bug36/10_fstring.py @@ -14,6 +14,12 @@ assert ( assert "def0" == f"{abc}0" assert "defdef" == f"{abc}{abc!s}" +# From 3.8 test/test_string.py +# We had the precedence of yield vs. lambda incorrect. +def fn(x): + yield f"x:{yield (lambda i: x * i)}" + + # From 3.6 functools.py # Bug was handling format operator strings. diff --git a/test/simple_source/bug38/00_bug_dict_comp.py b/test/simple_source/bug38/00_bug_dict_comp.py new file mode 100644 index 00000000..c9b69db2 --- /dev/null +++ b/test/simple_source/bug38/00_bug_dict_comp.py @@ -0,0 +1,12 @@ +# Issue 104 +# Python 3.8 reverses the order or keys and values in +# dictionary comprehensions from the order in all previous Pythons. +# Also we were looking in the wrong place for the collection of the +# dictionary comprehension +# RUNNABLE! + +"""This program is self-checking!""" +x = [(0, [1]), (2, [3])] +for i in range(0, 1): + y = {key: val[i - 1] for (key, val) in x} +assert y == {0: 1, 2: 3} diff --git a/uncompyle6/parsers/parse38.py b/uncompyle6/parsers/parse38.py index 2bead40b..3eb1b70a 100644 --- a/uncompyle6/parsers/parse38.py +++ b/uncompyle6/parsers/parse38.py @@ -17,7 +17,7 @@ spark grammar differences over Python 3.7 for Python 3.8 """ from __future__ import print_function -from uncompyle6.parser import PythonParserSingle +from uncompyle6.parser import PythonParserSingle, nop_func from spark_parser import DEFAULT_DEBUG as PARSER_DEFAULT_DEBUG from uncompyle6.parsers.parse37 import Python37Parser @@ -367,6 +367,195 @@ class Python38Parser(Python37Parser): self.check_reduce["whilestmt38"] = "tokens" self.check_reduce["try_elsestmtl38"] = "AST" + # For a rough break out on the first word. This may + # include instructions that don't need customization, + # but we'll do a finer check after the rough breakout. + customize_instruction_basenames = frozenset( + ( + "BEFORE", + "BUILD", + "CALL", + "DICT", + "GET", + "FORMAT", + "LIST", + "LOAD", + "MAKE", + "SETUP", + "UNPACK", + ) + ) + + # Opcode names in the custom_ops_processed set have rules that get added + # unconditionally and the rules are constant. So they need to be done + # only once and if we see the opcode a second we don't have to consider + # adding more rules. + # + custom_ops_processed = frozenset() + + # A set of instruction operation names that exist in the token stream. + # We use this customize the grammar that we create. + # 2.6-compatible set comprehensions + self.seen_ops = frozenset([t.kind for t in tokens]) + self.seen_op_basenames = frozenset( + [opname[: opname.rfind("_")] for opname in self.seen_ops] + ) + + custom_ops_processed = set(["DICT_MERGE"]) + + # Loop over instructions adding custom grammar rules based on + # a specific instruction seen. + + if "PyPy" in customize: + self.addRule( + """ + stmt ::= assign3_pypy + stmt ::= assign2_pypy + assign3_pypy ::= expr expr expr store store store + assign2_pypy ::= expr expr store store + """, + nop_func, + ) + + n = len(tokens) + # Determine if we have an iteration CALL_FUNCTION_1. + has_get_iter_call_function1 = False + for i, token in enumerate(tokens): + if ( + token == "GET_ITER" + and i < n - 2 + and tokens[i + 1] == "CALL_FUNCTION_1" + ): + has_get_iter_call_function1 = True + + for i, token in enumerate(tokens): + opname = token.kind + + # Do a quick breakout before testing potentially + # each of the dozen or so instruction in if elif. + if ( + opname[: opname.find("_")] not in customize_instruction_basenames + or opname in custom_ops_processed + ): + continue + + opname_base = opname[: opname.rfind("_")] + + # Do a quick breakout before testing potentially + # each of the dozen or so instruction in if elif. + if ( + opname[: opname.find("_")] not in customize_instruction_basenames + or opname in custom_ops_processed + ): + continue + if opname_base in ( + "BUILD_LIST", + "BUILD_SET", + "BUILD_SET_UNPACK", + "BUILD_TUPLE", + "BUILD_TUPLE_UNPACK", + ): + v = token.attr + + is_LOAD_CLOSURE = False + if opname_base == "BUILD_TUPLE": + # If is part of a "load_closure", then it is not part of a + # "list". + is_LOAD_CLOSURE = True + for j in range(v): + if tokens[i - j - 1].kind != "LOAD_CLOSURE": + is_LOAD_CLOSURE = False + break + if is_LOAD_CLOSURE: + rule = "load_closure ::= %s%s" % (("LOAD_CLOSURE " * v), opname) + self.add_unique_rule(rule, opname, token.attr, customize) + + elif opname_base == "BUILD_LIST": + v = token.attr + if v == 0: + rule_str = """ + list ::= BUILD_LIST_0 + list_unpack ::= BUILD_LIST_0 expr LIST_EXTEND + list ::= list_unpack + """ + self.add_unique_doc_rules(rule_str, customize) + + elif opname == "BUILD_TUPLE_UNPACK_WITH_CALL": + # FIXME: should this be parameterized by EX value? + self.addRule( + """expr ::= call_ex_kw3 + call_ex_kw3 ::= expr + build_tuple_unpack_with_call + expr + CALL_FUNCTION_EX_KW + """, + nop_func, + ) + + if not is_LOAD_CLOSURE or v == 0: + # We do this complicated test to speed up parsing of + # pathelogically long literals, especially those over 1024. + build_count = token.attr + thousands = build_count // 1024 + thirty32s = (build_count // 32) % 32 + if thirty32s > 0: + rule = "expr32 ::=%s" % (" expr" * 32) + self.add_unique_rule(rule, opname_base, build_count, customize) + pass + if thousands > 0: + self.add_unique_rule( + "expr1024 ::=%s" % (" expr32" * 32), + opname_base, + build_count, + customize, + ) + pass + collection = opname_base[opname_base.find("_") + 1 :].lower() + rule = ( + ("%s ::= " % collection) + + "expr1024 " * thousands + + "expr32 " * thirty32s + + "expr " * (build_count % 32) + + opname + ) + self.add_unique_rules(["expr ::= %s" % collection, rule], customize) + continue + continue + elif opname == "LOAD_CLOSURE": + self.addRule("""load_closure ::= LOAD_CLOSURE+""", nop_func) + + elif opname == "LOOKUP_METHOD": + # A PyPy speciality - DRY with parse3 + self.addRule( + """ + expr ::= attribute + attribute ::= expr LOOKUP_METHOD + """, + nop_func, + ) + custom_ops_processed.add(opname) + + elif opname == "MAKE_FUNCTION_8": + if "LOAD_DICTCOMP" in self.seen_ops: + # Is there something general going on here? + rule = """ + dict_comp ::= load_closure LOAD_DICTCOMP LOAD_STR + MAKE_FUNCTION_8 expr + GET_ITER CALL_FUNCTION_1 + """ + self.addRule(rule, nop_func) + elif "LOAD_SETCOMP" in self.seen_ops: + rule = """ + set_comp ::= load_closure LOAD_SETCOMP LOAD_STR + MAKE_FUNCTION_CLOSURE expr + GET_ITER CALL_FUNCTION_1 + """ + self.addRule(rule, nop_func) + + + + + def reduce_is_invalid(self, rule, ast, tokens, first, last): invalid = super(Python38Parser, self).reduce_is_invalid(rule, ast, diff --git a/uncompyle6/semantics/consts.py b/uncompyle6/semantics/consts.py index e62d5ea5..06c9d91c 100644 --- a/uncompyle6/semantics/consts.py +++ b/uncompyle6/semantics/consts.py @@ -50,14 +50,15 @@ NO_PARENTHESIS_EVER = 100 # fmt: off PRECEDENCE = { "named_expr": 40, # := - "yield": 38, # Needs to be below named_expr - "yield_from": 38, - "tuple_list_starred": 38, # *x, *y, *z - about at the level of yield? "dict_unpack": 38, # **kwargs "list_unpack": 38, # *args + "yield_from": 38, + "tuple_list_starred": 38, # *x, *y, *z - about at the level of yield? "_lambda_body": 30, - "lambda_body": 30, # lambda ... : lambda_body + "lambda_body": 32, # lambda ... : lambda_body + + "yield": 30, # Needs to be below named_expr and lambda_body "if_exp": 28, # IfExp ( a if x else b) "if_exp_lambda": 28, # IfExp involving a lambda expression diff --git a/uncompyle6/semantics/customize38.py b/uncompyle6/semantics/customize38.py index 6c1a37a4..0e6e5e70 100644 --- a/uncompyle6/semantics/customize38.py +++ b/uncompyle6/semantics/customize38.py @@ -66,6 +66,9 @@ def customize_for_version38(self, version): (1, "c_suite_stmts_opt"), (-2, "c_suite_stmts_opt"), ), + # Python 3.8 reverses the order of keys and items + # from all prior versions of Python. + "dict_comp_body": ("%c: %c", (0, "expr"), (1, "expr"),), "except_cond1a": ("%|except %c:\n", (1, "expr"),), "except_cond_as": ( "%|except %c as %c:\n", diff --git a/uncompyle6/semantics/gencomp.py b/uncompyle6/semantics/gencomp.py index c9a05895..52f228ca 100644 --- a/uncompyle6/semantics/gencomp.py +++ b/uncompyle6/semantics/gencomp.py @@ -302,6 +302,9 @@ class ComprehensionMixin: store = set_iter_async[1] assert store == "store" n = set_iter_async[2] + elif node == "list_comp" and tree[0] == "expr": + tree = tree[0][0] + n = tree[iter_index] else: n = tree[iter_index]