From 08009f9fc7939d1cfbb41efbde00c607717337fb Mon Sep 17 00:00:00 2001 From: rocky Date: Sun, 25 Feb 2024 08:19:18 -0500 Subject: [PATCH] improve list comprehensions --- test/stdlib/3.3-exclude.sh | 2 +- test/stdlib/3.8-exclude.sh | 9 ++++---- uncompyle6/parsers/parse2.py | 12 +++++----- uncompyle6/parsers/parse3.py | 12 ++++++---- uncompyle6/parsers/parse37base.py | 26 ++++++++++----------- uncompyle6/semantics/gencomp.py | 38 ++++++++++++++++++++++++++----- uncompyle6/semantics/n_actions.py | 2 +- 7 files changed, 64 insertions(+), 37 deletions(-) diff --git a/test/stdlib/3.3-exclude.sh b/test/stdlib/3.3-exclude.sh index b2d0afe8..a45c81ca 100644 --- a/test/stdlib/3.3-exclude.sh +++ b/test/stdlib/3.3-exclude.sh @@ -10,7 +10,7 @@ SKIP_TESTS=( # tgt.append(elem) [test_itertools.py]=1 - [test_buffer.py]=1 # FIXME: Works on c90ff51 + [test_buffer.py]=pytest # FIXME: Works on c90ff51 [test_cmath.py]=pytest [test_atexit.py]=1 # The atexit test starting at 3.3 looks for specific comments in error lines diff --git a/test/stdlib/3.8-exclude.sh b/test/stdlib/3.8-exclude.sh index d4393ae9..61f8700c 100644 --- a/test/stdlib/3.8-exclude.sh +++ b/test/stdlib/3.8-exclude.sh @@ -28,12 +28,12 @@ SKIP_TESTS=( # These and the above may be due to new code generation or tests # between 3.8.3 and 3.8.5 ? - [test_decorators.py]=1 # + [test_decorators.py]=1 # parse error - [test_dtrace.py]=1 # - [test_exceptions.py]=1 # + [test_dtrace.py]=1 # parse error + [test_exceptions.py]=1 # parse error [test_ftplib.py]=1 # - [test_gc.py]=1 # + [test_gc.py]=1 # FIXME: return return strip_python_stderr(stderr) [test_gzip.py]=1 # [test_hashlib.py]=1 # [test_iter.py]=1 # @@ -51,7 +51,6 @@ SKIP_TESTS=( [test_audioop.py]=1 # test failure [test_audit.py]=1 # parse error - [test_base64.py]=1 # parse error [test_baseexception.py]=1 # [test_bigaddrspace.py]=1 # parse error [test_bigmem.py]=1 # parse error diff --git a/uncompyle6/parsers/parse2.py b/uncompyle6/parsers/parse2.py index 86cf8bd4..6b871968 100644 --- a/uncompyle6/parsers/parse2.py +++ b/uncompyle6/parsers/parse2.py @@ -1,4 +1,4 @@ -# Copyright (c) 2015-2021 Rocky Bernstein +# Copyright (c) 2015-2021, 2024 Rocky Bernstein # Copyright (c) 2000-2002 by hartmut Goebel # # Copyright (c) 1999 John Aycock @@ -27,11 +27,12 @@ that a later phase can turn into a sequence of ASCII text. from __future__ import print_function -from uncompyle6.parsers.reducecheck import except_handler_else, ifelsestmt, tryelsestmt -from uncompyle6.parser import PythonParser, PythonParserSingle, nop_func -from uncompyle6.parsers.treenode import SyntaxTree from spark_parser import DEFAULT_DEBUG as PARSER_DEFAULT_DEBUG +from uncompyle6.parser import PythonParser, PythonParserSingle, nop_func +from uncompyle6.parsers.reducecheck import except_handler_else, ifelsestmt, tryelsestmt +from uncompyle6.parsers.treenode import SyntaxTree + class Python2Parser(PythonParser): def __init__(self, debug_parser=PARSER_DEFAULT_DEBUG): @@ -405,7 +406,6 @@ class Python2Parser(PythonParser): "CALL_FUNCTION_VAR_KW", "CALL_FUNCTION_KW", ): - args_pos, args_kw = self.get_pos_kw(token) # number of apply equiv arguments: @@ -526,7 +526,7 @@ class Python2Parser(PythonParser): custom_seen_ops.add(opname) continue elif opname == "LOAD_LISTCOMP": - self.addRule("expr ::= listcomp", nop_func) + self.addRule("expr ::= list_comp", nop_func) custom_seen_ops.add(opname) continue elif opname == "LOAD_SETCOMP": diff --git a/uncompyle6/parsers/parse3.py b/uncompyle6/parsers/parse3.py index 969d9109..681b8d76 100644 --- a/uncompyle6/parsers/parse3.py +++ b/uncompyle6/parsers/parse3.py @@ -1085,7 +1085,9 @@ class Python3Parser(PythonParser): ) custom_ops_processed.add(opname) elif opname == "LOAD_LISTCOMP": - self.add_unique_rule("expr ::= listcomp", opname, token.attr, customize) + self.add_unique_rule( + "expr ::= list_comp", opname, token.attr, customize + ) custom_ops_processed.add(opname) elif opname == "LOAD_SETCOMP": # Should this be generalized and put under MAKE_FUNCTION? @@ -1154,7 +1156,7 @@ class Python3Parser(PythonParser): # and have GET_ITER CALL_FUNCTION_1 # Todo: For Pypy we need to modify this slightly rule_pat = ( - "listcomp ::= %sload_closure LOAD_LISTCOMP %%s%s expr " + "list_comp ::= %sload_closure LOAD_LISTCOMP %%s%s expr " "GET_ITER CALL_FUNCTION_1" % ("pos_arg " * pos_args_count, opname) ) @@ -1348,14 +1350,14 @@ class Python3Parser(PythonParser): # 'exprs' in the rule above into a # tuple. rule_pat = ( - "listcomp ::= load_closure LOAD_LISTCOMP %%s%s " + "list_comp ::= load_closure LOAD_LISTCOMP %%s%s " "expr GET_ITER CALL_FUNCTION_1" % (opname,) ) self.add_make_function_rule( rule_pat, opname, token.attr, customize ) rule_pat = ( - "listcomp ::= %sLOAD_LISTCOMP %%s%s expr " + "list_comp ::= %sLOAD_LISTCOMP %%s%s expr " "GET_ITER CALL_FUNCTION_1" % ("expr " * pos_args_count, opname) ) @@ -1399,7 +1401,7 @@ class Python3Parser(PythonParser): # and have GET_ITER CALL_FUNCTION_1 # Todo: For Pypy we need to modify this slightly rule_pat = ( - "listcomp ::= %sLOAD_LISTCOMP %%s%s expr " + "list_comp ::= %sLOAD_LISTCOMP %%s%s expr " "GET_ITER CALL_FUNCTION_1" % ("expr " * pos_args_count, opname) ) diff --git a/uncompyle6/parsers/parse37base.py b/uncompyle6/parsers/parse37base.py index f5ef7065..31f92a62 100644 --- a/uncompyle6/parsers/parse37base.py +++ b/uncompyle6/parsers/parse37base.py @@ -2,11 +2,10 @@ """ Python 3.7 base code. We keep non-custom-generated grammar rules out of this file. """ -from uncompyle6.parser import ParserError, PythonParser, nop_func -from uncompyle6.parsers.treenode import SyntaxTree from spark_parser import DEFAULT_DEBUG as PARSER_DEFAULT_DEBUG from spark_parser.spark import rule2str +from uncompyle6.parser import ParserError, PythonParser, nop_func from uncompyle6.parsers.reducecheck import ( and_invalid, ifelsestmt, @@ -16,9 +15,10 @@ from uncompyle6.parsers.reducecheck import ( or_check, testtrue, tryelsestmtl3, - while1stmt, while1elsestmt, + while1stmt, ) +from uncompyle6.parsers.treenode import SyntaxTree class Python37BaseParser(PythonParser): @@ -54,7 +54,7 @@ class Python37BaseParser(PythonParser): expr call CALL_FUNCTION_3 - """ + """ # FIXME: I bet this can be simplified # look for next MAKE_FUNCTION for i in range(i + 1, len(tokens)): @@ -104,7 +104,6 @@ class Python37BaseParser(PythonParser): # organization for this. For example, arrange organize by opcode base? def customize_grammar_rules(self, tokens, customize): - is_pypy = False # For a rough break out on the first word. This may @@ -348,7 +347,6 @@ class Python37BaseParser(PythonParser): self.addRule(rule, nop_func) elif opname_base in ("BUILD_MAP", "BUILD_MAP_UNPACK"): - if opname == "BUILD_MAP_UNPACK": self.addRule( """ @@ -525,7 +523,6 @@ class Python37BaseParser(PythonParser): "CALL_FUNCTION_VAR_KW", ) ) or opname.startswith("CALL_FUNCTION_KW"): - if opname == "CALL_FUNCTION" and token.attr == 1: rule = """ expr ::= dict_comp @@ -720,7 +717,9 @@ class Python37BaseParser(PythonParser): ) custom_ops_processed.add(opname) elif opname == "LOAD_LISTCOMP": - self.add_unique_rule("expr ::= listcomp", opname, token.attr, customize) + self.add_unique_rule( + "expr ::= list_comp", opname, token.attr, customize + ) custom_ops_processed.add(opname) elif opname == "LOAD_NAME": if ( @@ -799,7 +798,7 @@ class Python37BaseParser(PythonParser): # and have GET_ITER CALL_FUNCTION_1 # Todo: For Pypy we need to modify this slightly rule_pat = ( - "listcomp ::= %sload_closure LOAD_LISTCOMP %%s%s expr " + "list_comp ::= %sload_closure LOAD_LISTCOMP %%s%s expr " "GET_ITER CALL_FUNCTION_1" % ("pos_arg " * args_pos, opname) ) @@ -897,14 +896,14 @@ class Python37BaseParser(PythonParser): # 'exprs' in the rule above into a # tuple. rule_pat = ( - "listcomp ::= load_closure LOAD_LISTCOMP %%s%s " + "list_comp ::= load_closure LOAD_LISTCOMP %%s%s " "expr GET_ITER CALL_FUNCTION_1" % (opname,) ) self.add_make_function_rule( rule_pat, opname, token.attr, customize ) rule_pat = ( - "listcomp ::= %sLOAD_LISTCOMP %%s%s expr " + "list_comp ::= %sLOAD_LISTCOMP %%s%s expr " "GET_ITER CALL_FUNCTION_1" % ("expr " * args_pos, opname) ) self.add_make_function_rule( @@ -938,7 +937,7 @@ class Python37BaseParser(PythonParser): # and have GET_ITER CALL_FUNCTION_1 # Todo: For Pypy we need to modify this slightly rule_pat = ( - "listcomp ::= %sLOAD_LISTCOMP %%s%s expr " + "list_comp ::= %sLOAD_LISTCOMP %%s%s expr " "GET_ITER CALL_FUNCTION_1" % ("expr " * args_pos, opname) ) self.add_make_function_rule( @@ -1259,7 +1258,8 @@ class Python37BaseParser(PythonParser): if fn: return fn(self, lhs, n, rule, ast, tokens, first, last) except Exception: - import sys, traceback + import sys + import traceback print( f"Exception in {fn.__name__} {sys.exc_info()[1]}\n" diff --git a/uncompyle6/semantics/gencomp.py b/uncompyle6/semantics/gencomp.py index 936b19e2..f15b632d 100644 --- a/uncompyle6/semantics/gencomp.py +++ b/uncompyle6/semantics/gencomp.py @@ -23,9 +23,9 @@ from xdis import co_flags_is_async, iscode from uncompyle6.parser import get_python_parser from uncompyle6.scanner import Code +from uncompyle6.scanners.tok import Token from uncompyle6.semantics.consts import PRECEDENCE from uncompyle6.semantics.helper import is_lambda_mode -from uncompyle6.scanners.tok import Token class ComprehensionMixin: @@ -174,7 +174,10 @@ class ComprehensionMixin: tree = tree[1] pass - if tree in ("genexpr_func", "genexpr_func_async",): + if tree in ( + "genexpr_func", + "genexpr_func_async", + ): for i in range(3, 5): if tree[i] == "comp_iter": iter_index = i @@ -332,8 +335,19 @@ class ComprehensionMixin: assert store == "store" n = set_iter_async[2] elif node == "list_comp" and tree[0] == "expr": - tree = tree[0][0] - n = tree[iter_index] + list_iter = None + for list_iter_try in tree: + if list_iter_try == "list_iter": + list_iter = list_iter_try + break + if not list_iter_try: + tree = tree[0][0] + n = tree[iter_index] + else: + n = list_iter + pass + pass + pass else: n = tree[iter_index] @@ -407,6 +421,9 @@ class ComprehensionMixin: n = n[0] if n in ("list_for", "comp_for"): + if n == "list_for" and not comp_for and n[0] == "expr": + comp_for = n[0] + n_index = 3 if ( (n[2] == "store") @@ -496,11 +513,21 @@ class ComprehensionMixin: if comp_for: self.preorder(comp_for) else: + try: + node[in_node_index] + except: + from trepan.api import debug + + debug() self.preorder(node[in_node_index]) # Here is where we handle nested list iterations. if tree == "list_comp" and self.version != (3, 0): - list_iter = tree[1] + list_iter = None + for list_iter_try in tree: + if list_iter_try == "list_iter": + list_iter = list_iter_try + break assert list_iter == "list_iter" if list_iter[0] == "list_for": self.preorder(list_iter[0][3]) @@ -639,7 +666,6 @@ class ComprehensionMixin: # Find the list comprehension body. It is the inner-most # node that is not list_.. . while n == "list_iter": - # recurse one step n = n[0] diff --git a/uncompyle6/semantics/n_actions.py b/uncompyle6/semantics/n_actions.py index 83619f77..ab4c2713 100644 --- a/uncompyle6/semantics/n_actions.py +++ b/uncompyle6/semantics/n_actions.py @@ -1036,7 +1036,7 @@ class NonterminalActions: self.prec = p self.prune() # stop recursing - def n_listcomp(self, node): + def n_list_comp(self, node): self.write("[") if node[0].kind == "load_closure": assert self.version >= (3, 0)