From c9f33edde4917f68f3255fc74ee0a57685024da7 Mon Sep 17 00:00:00 2001 From: rocky Date: Wed, 4 May 2022 18:40:50 -0400 Subject: [PATCH] Handle 3.6 "async for" better --- uncompyle6/parsers/parse36.py | 66 ++++++++++++- uncompyle6/semantics/customize3.py | 130 ------------------------- uncompyle6/semantics/customize36.py | 6 ++ uncompyle6/semantics/gencomp.py | 145 +++++++++++++++++++++++++++- 4 files changed, 212 insertions(+), 135 deletions(-) diff --git a/uncompyle6/parsers/parse36.py b/uncompyle6/parsers/parse36.py index 19122b8a..0ed4c73c 100644 --- a/uncompyle6/parsers/parse36.py +++ b/uncompyle6/parsers/parse36.py @@ -222,7 +222,18 @@ class Python36Parser(Python35Parser): else_suite COME_FROM_LOOP """) - self.check_reduce['call_kw'] = 'AST' + self.check_reduce["call_kw"] = "AST" + + # Opcode names in the custom_ops_processed set have rules that get added + # unconditionally and the rules are constant. So they need to be done + # only once and if we see the opcode a second we don't have to consider + # adding more rules. + # + # Note: BUILD_TUPLE_UNPACK_WITH_CALL gets considered by + # default because it starts with BUILD. So we'll set to ignore it from + # the start. + custom_ops_processed = set() + for i, token in enumerate(tokens): opname = token.kind @@ -307,6 +318,59 @@ class Python36Parser(Python35Parser): self.addRule(rule, nop_func) rule = ('starred ::= %s %s' % ('expr ' * v, opname)) self.addRule(rule, nop_func) + elif opname == "GET_AITER": + self.addRule( + """ + expr ::= generator_exp_async + generator_exp_async ::= load_genexpr LOAD_STR MAKE_FUNCTION_0 expr + GET_AITER CALL_FUNCTION_1 + + stmt ::= genexpr_func_async + + func_async_prefix ::= _come_froms + LOAD_CONST YIELD_FROM + SETUP_EXCEPT GET_ANEXT LOAD_CONST YIELD_FROM + func_async_middle ::= POP_BLOCK JUMP_FORWARD COME_FROM_EXCEPT + DUP_TOP LOAD_GLOBAL COMPARE_OP POP_JUMP_IF_TRUE + END_FINALLY COME_FROM + genexpr_func_async ::= LOAD_FAST func_async_prefix + store func_async_middle comp_iter + JUMP_BACK COME_FROM + POP_TOP POP_TOP POP_TOP POP_EXCEPT POP_TOP + + expr ::= list_comp_async + list_comp_async ::= LOAD_LISTCOMP LOAD_STR MAKE_FUNCTION_0 + expr GET_AITER CALL_FUNCTION_1 + GET_AWAITABLE LOAD_CONST + YIELD_FROM + + expr ::= list_comp_async + list_afor2 ::= func_async_prefix + store func_async_middle list_iter + JUMP_BACK + POP_TOP POP_TOP POP_TOP POP_EXCEPT POP_TOP + list_comp_async ::= BUILD_LIST_0 LOAD_FAST list_afor2 + get_aiter ::= expr GET_AITER + list_afor ::= get_aiter list_afor2 + list_iter ::= list_afor + """, + nop_func, + ) + elif opname == "GET_ANEXT": + self.addRule( + """ + func_async_prefix ::= _come_froms SETUP_FINALLY GET_ANEXT LOAD_CONST YIELD_FROM POP_BLOCK + func_async_middle ::= JUMP_FORWARD COME_FROM_EXCEPT + DUP_TOP LOAD_GLOBAL COMPARE_OP POP_JUMP_IF_TRUE + list_afor2 ::= func_async_prefix + store list_iter + JUMP_BACK COME_FROM_FINALLY + END_ASYNC_FOR + """, + nop_func, + ) + custom_ops_processed.add(opname) + elif opname == 'SETUP_ANNOTATIONS': # 3.6 Variable Annotations PEP 526 # This seems to come before STORE_ANNOTATION, and doesn't diff --git a/uncompyle6/semantics/customize3.py b/uncompyle6/semantics/customize3.py index bb8abe27..2f388893 100644 --- a/uncompyle6/semantics/customize3.py +++ b/uncompyle6/semantics/customize3.py @@ -80,136 +80,6 @@ def customize_for_version3(self, version): self.default(node) self.n_tryfinallystmt = tryfinallystmt - def listcomp_closure3(node): - """List comprehensions in Python 3 when handled as a closure. - See if we can combine code. - """ - - # FIXME: DRY with comprehension_walk_newer - p = self.prec - self.prec = 27 - - code_obj = node[1].attr - assert iscode(code_obj), node[1] - code = Code(code_obj, self.scanner, self.currentclass, self.debug_opts["asm"]) - - ast = self.build_ast(code._tokens, code._customize, code) - self.customize(code._customize) - - # skip over: sstmt, stmt, return, return_expr - # and other singleton derivations - while len(ast) == 1 or ( - ast in ("sstmt", "return") and ast[-1] in ("RETURN_LAST", "RETURN_VALUE") - ): - self.prec = 100 - ast = ast[0] - - n = ast[1] - - # Pick out important parts of the comprehension: - # * the variables we iterate over: "stores" - # * the results we accumulate: "n" - - # collections is the name of the expression(s) we are iterating over - collections = [node[-3]] - list_ifs = [] - - if self.version[:2] == (3, 0) and n != "list_iter": - # FIXME 3.0 is a snowflake here. We need - # special code for this. Not sure if this is totally - # correct. - stores = [ast[3]] - assert ast[4] == "comp_iter" - n = ast[4] - # Find the list comprehension body. It is the inner-most - # node that is not comp_.. . - while n == "comp_iter": - if n[0] == "comp_for": - n = n[0] - stores.append(n[2]) - n = n[3] - elif n[0] in ("comp_if", "comp_if_not"): - n = n[0] - # FIXME: just a guess - if n[0].kind == "expr": - list_ifs.append(n) - else: - list_ifs.append([1]) - n = n[2] - pass - else: - break - pass - - # Skip over n[0] which is something like: _[1] - self.preorder(n[1]) - - else: - assert n == "list_iter" - stores = [] - # Find the list comprehension body. It is the inner-most - # node that is not list_.. . - while n == "list_iter": - - # recurse one step - n = n[0] - - # FIXME: adjust for set comprehension - if n == "list_for": - stores.append(n[2]) - n = n[3] - if n[0] == "list_for": - # Dog-paddle down largely singleton reductions - # to find the collection (expr) - c = n[0][0] - if c == "expr": - c = c[0] - # FIXME: grammar is wonky here? Is this really an attribute? - if c == "attribute": - c = c[0] - collections.append(c) - pass - elif n in ("list_if", "list_if_not", "list_if_or_not"): - if n[0].kind == "expr": - list_ifs.append(n) - else: - list_ifs.append([1]) - n = n[-2] if n[-1] == "come_from_opt" else n[-1] - pass - elif n == "list_if37": - list_ifs.append(n) - n = n[-1] - pass - elif n == "list_afor": - collections.append(n[0][0]) - n = n[1] - stores.append(n[1][0]) - n = n[2] if n[2].kind == "list_iter" else n[3] - pass - - assert n == "lc_body", ast - - self.preorder(n[0]) - - # FIXME: add indentation around "for"'s and "in"'s - n_colls = len(collections) - for i, store in enumerate(stores): - if i >= n_colls: - break - if collections[i] == "LOAD_DEREF" and co_flags_is_async(code_obj.co_flags): - self.write(" async") - pass - self.write(" for ") - self.preorder(store) - self.write(" in ") - self.preorder(collections[i]) - if i < len(list_ifs): - self.preorder(list_ifs[i]) - pass - pass - self.prec = p - self.listcomp_closure3 = listcomp_closure3 - def n_classdef3(node): """Handle "classdef" nonterminal for 3.0 >= version 3.0 < 3.6 """ diff --git a/uncompyle6/semantics/customize36.py b/uncompyle6/semantics/customize36.py index eb5cceb1..71b49a3d 100644 --- a/uncompyle6/semantics/customize36.py +++ b/uncompyle6/semantics/customize36.py @@ -73,6 +73,12 @@ def customize_for_version36(self, version): "ifstmtl": ("%|if %c:\n%+%c%-", (0, "testexpr"), (1, "_ifstmts_jumpl")), + + "list_afor": ( + " async for %[1]{%c} in %c%[1]{%c}", + (1, "store"), (0, "get_aiter"), (3, "list_iter"), + ), + "try_except36": ("%|try:\n%+%c%-%c\n\n", 1, -2), "tryfinally36": ("%|try:\n%+%c%-%|finally:\n%+%c%-\n\n", (1, "returns"), 3), "tryfinally_return_stmt": ("%|try:\n%+%c%-%|finally:\n%+%|return%-\n\n", 1), diff --git a/uncompyle6/semantics/gencomp.py b/uncompyle6/semantics/gencomp.py index 6341d6da..2ad0e558 100644 --- a/uncompyle6/semantics/gencomp.py +++ b/uncompyle6/semantics/gencomp.py @@ -19,7 +19,7 @@ Generators and comprehension functions from typing import Optional -from xdis import iscode +from xdis import co_flags_is_async, iscode from uncompyle6.parser import get_python_parser from uncompyle6.scanner import Code @@ -27,7 +27,6 @@ from uncompyle6.semantics.consts import PRECEDENCE from uncompyle6.semantics.helper import is_lambda_mode from uncompyle6.scanners.tok import Token - class ComprehensionMixin: """ These functions hand nonterminal common actions that occur @@ -39,7 +38,8 @@ class ComprehensionMixin: are not seen. """ def closure_walk(self, node, collection_index): - """Dictionary and comprehensions using closure the way they are done in Python3. + """ + Dictionary and comprehensions using closure the way they are done in Python3. """ p = self.prec self.prec = 27 @@ -59,6 +59,10 @@ class ComprehensionMixin: list_if = None assert n == "comp_iter" + # Pick out important parts of the comprehension: + # * the variables we iterate over: "stores" + # * the results we accumulate: "n" + # Find inner-most node. while n == "comp_iter": n = n[0] # recurse one step @@ -128,7 +132,7 @@ class ComprehensionMixin: assert iscode(cn.attr) - code = Code(cn.attr, self.scanner, self.currentclass) + code = Code(cn.attr, self.scanner, self.currentclass, self.debug_opts["asm"]) # FIXME: is there a way we can avoid this? # The problem is that in filter in top-level list comprehensions we can @@ -435,3 +439,136 @@ class ComprehensionMixin: self.prec = 100 tree = tree[1] if tree[0] in ("dom_start", "dom_start_opt") else tree[0] return tree + + def listcomp_closure3(self, node): + """ + List comprehensions in Python 3 when handled as a closure. + See if we can combine code. + """ + + # FIXME: DRY with comprehension_walk_newer + p = self.prec + self.prec = 27 + + code_obj = node[1].attr + assert iscode(code_obj), node[1] + code = Code(code_obj, self.scanner, self.currentclass, self.debug_opts["asm"]) + + tree = self.build_ast(code._tokens, code._customize, code) + self.customize(code._customize) + + # skip over: sstmt, stmt, return, return_expr + # and other singleton derivations + while len(tree) == 1 or ( + tree in ("sstmt", "return") and tree[-1] in ("RETURN_LAST", "RETURN_VALUE") + ): + self.prec = 100 + tree = tree[0] + + n = tree[1] + + # Pick out important parts of the comprehension: + # * the variables we iterate over: "stores" + # * the results we accumulate: "n" + + # collections is the name of the expression(s) we are iterating over + collections = [node[-3]] + list_ifs = [] + + if self.version[:2] == (3, 0) and n != "list_iter": + # FIXME 3.0 is a snowflake here. We need + # special code for this. Not sure if this is totally + # correct. + stores = [tree[3]] + assert tree[4] == "comp_iter" + n = tree[4] + # Find the list comprehension body. It is the inner-most + # node that is not comp_.. . + while n == "comp_iter": + if n[0] == "comp_for": + n = n[0] + stores.append(n[2]) + n = n[3] + elif n[0] in ("comp_if", "comp_if_not"): + n = n[0] + # FIXME: just a guess + if n[0].kind == "expr": + list_ifs.append(n) + else: + list_ifs.append([1]) + n = n[2] + pass + else: + break + pass + + # Skip over n[0] which is something like: _[1] + self.preorder(n[1]) + + else: + assert n == "list_iter" + stores = [] + # Find the list comprehension body. It is the inner-most + # node that is not list_.. . + while n == "list_iter": + + # recurse one step + n = n[0] + + # FIXME: adjust for set comprehension + if n == "list_for": + stores.append(n[2]) + n = n[3] + if n[0] == "list_for": + # Dog-paddle down largely singleton reductions + # to find the collection (expr) + c = n[0][0] + if c == "expr": + c = c[0] + # FIXME: grammar is wonky here? Is this really an attribute? + if c == "attribute": + c = c[0] + collections.append(c) + pass + elif n in ("list_if", "list_if_not", "list_if_or_not"): + if n[0].kind == "expr": + list_ifs.append(n) + else: + list_ifs.append([1]) + n = n[-2] if n[-1] == "come_from_opt" else n[-1] + pass + elif n == "list_if37": + list_ifs.append(n) + n = n[-1] + pass + elif n == "list_afor": + collections.append(n[0][0]) + n = n[1] + stores.append(n[1][0]) + n = n[2] if n[2].kind == "list_iter" else n[3] + pass + + assert n == "lc_body", tree + + self.preorder(n[0]) + + # FIXME: add indentation around "for"'s and "in"'s + n_colls = len(collections) + for i, store in enumerate(stores): + if i >= n_colls: + break + token = collections[i] + if not isinstance(token, Token): + token = token.first_child() + if token == "LOAD_DEREF" and co_flags_is_async(code_obj.co_flags): + self.write(" async") + pass + self.write(" for ") + self.preorder(store) + self.write(" in ") + self.preorder(collections[i]) + if i < len(list_ifs): + self.preorder(list_ifs[i]) + pass + pass + self.prec = p