jinx/python-uncompyle6
1
Fork 0
mirror of https://github.com/rocky/python-uncompyle6.git synced 2025-08-03 00:45:53 +08:00
Code Issues Packages Projects Releases Wiki Activity

improve list comprehensions

Browse Source
This commit is contained in:
rocky
2024-02-25 08:19:18 -05:00
parent d3ed646a8e
commit 08009f9fc7
7 changed files with 64 additions and 37 deletions
Download Patch File Download Diff File Expand all files Collapse all files

2
test/stdlib/3.3-exclude.sh
View File

@@ -10,7 +10,7 @@ SKIP_TESTS=(
# tgt.append(elem) # tgt.append(elem)
[test_itertools.py]=1 [test_itertools.py]=1
[test_buffer.py]=1 # FIXME: Works on c90ff51 [test_buffer.py]=pytest # FIXME: Works on c90ff51
[test_cmath.py]=pytest [test_cmath.py]=pytest
[test_atexit.py]=1 # The atexit test starting at 3.3 looks for specific comments in error lines [test_atexit.py]=1 # The atexit test starting at 3.3 looks for specific comments in error lines

9
test/stdlib/3.8-exclude.sh
View File

@@ -28,12 +28,12 @@ SKIP_TESTS=(
# These and the above may be due to new code generation or tests # These and the above may be due to new code generation or tests
# between 3.8.3 and 3.8.5 ? # between 3.8.3 and 3.8.5 ?
[test_decorators.py]=1 # [test_decorators.py]=1 # parse error
[test_dtrace.py]=1 # [test_dtrace.py]=1 # parse error
[test_exceptions.py]=1 # [test_exceptions.py]=1 # parse error
[test_ftplib.py]=1 # [test_ftplib.py]=1 #
[test_gc.py]=1 # [test_gc.py]=1 # FIXME: return return strip_python_stderr(stderr)
[test_gzip.py]=1 # [test_gzip.py]=1 #
[test_hashlib.py]=1 # [test_hashlib.py]=1 #
[test_iter.py]=1 # [test_iter.py]=1 #
@@ -51,7 +51,6 @@ SKIP_TESTS=(
[test_audioop.py]=1 # test failure [test_audioop.py]=1 # test failure
[test_audit.py]=1 # parse error [test_audit.py]=1 # parse error
[test_base64.py]=1 # parse error
[test_baseexception.py]=1 # [test_baseexception.py]=1 #
[test_bigaddrspace.py]=1 # parse error [test_bigaddrspace.py]=1 # parse error
[test_bigmem.py]=1 # parse error [test_bigmem.py]=1 # parse error

12
uncompyle6/parsers/parse2.py
View File

@@ -1,4 +1,4 @@
# Copyright (c) 2015-2021 Rocky Bernstein # Copyright (c) 2015-2021, 2024 Rocky Bernstein
# Copyright (c) 2000-2002 by hartmut Goebel <h.goebel@crazy-compilers.com> # Copyright (c) 2000-2002 by hartmut Goebel <h.goebel@crazy-compilers.com>
# #
# Copyright (c) 1999 John Aycock # Copyright (c) 1999 John Aycock
@@ -27,11 +27,12 @@ that a later phase can turn into a sequence of ASCII text.
from __future__ import print_function from __future__ import print_function
from uncompyle6.parsers.reducecheck import except_handler_else, ifelsestmt, tryelsestmt
from uncompyle6.parser import PythonParser, PythonParserSingle, nop_func
from uncompyle6.parsers.treenode import SyntaxTree
from spark_parser import DEFAULT_DEBUG as PARSER_DEFAULT_DEBUG from spark_parser import DEFAULT_DEBUG as PARSER_DEFAULT_DEBUG
from uncompyle6.parser import PythonParser, PythonParserSingle, nop_func
from uncompyle6.parsers.reducecheck import except_handler_else, ifelsestmt, tryelsestmt
from uncompyle6.parsers.treenode import SyntaxTree
class Python2Parser(PythonParser): class Python2Parser(PythonParser):
def __init__(self, debug_parser=PARSER_DEFAULT_DEBUG): def __init__(self, debug_parser=PARSER_DEFAULT_DEBUG):
@@ -405,7 +406,6 @@ class Python2Parser(PythonParser):
"CALL_FUNCTION_VAR_KW", "CALL_FUNCTION_VAR_KW",
"CALL_FUNCTION_KW", "CALL_FUNCTION_KW",
): ):
args_pos, args_kw = self.get_pos_kw(token) args_pos, args_kw = self.get_pos_kw(token)
# number of apply equiv arguments: # number of apply equiv arguments:
@@ -526,7 +526,7 @@ class Python2Parser(PythonParser):
custom_seen_ops.add(opname) custom_seen_ops.add(opname)
continue continue
elif opname == "LOAD_LISTCOMP": elif opname == "LOAD_LISTCOMP":
self.addRule("expr ::= listcomp", nop_func) self.addRule("expr ::= list_comp", nop_func)
custom_seen_ops.add(opname) custom_seen_ops.add(opname)
continue continue
elif opname == "LOAD_SETCOMP": elif opname == "LOAD_SETCOMP":

12
uncompyle6/parsers/parse3.py
View File

@@ -1085,7 +1085,9 @@ class Python3Parser(PythonParser):
) )
custom_ops_processed.add(opname) custom_ops_processed.add(opname)
elif opname == "LOAD_LISTCOMP": elif opname == "LOAD_LISTCOMP":
self.add_unique_rule("expr ::= listcomp", opname, token.attr, customize) self.add_unique_rule(
"expr ::= list_comp", opname, token.attr, customize
)
custom_ops_processed.add(opname) custom_ops_processed.add(opname)
elif opname == "LOAD_SETCOMP": elif opname == "LOAD_SETCOMP":
# Should this be generalized and put under MAKE_FUNCTION? # Should this be generalized and put under MAKE_FUNCTION?
@@ -1154,7 +1156,7 @@ class Python3Parser(PythonParser):
# and have GET_ITER CALL_FUNCTION_1 # and have GET_ITER CALL_FUNCTION_1
# Todo: For Pypy we need to modify this slightly # Todo: For Pypy we need to modify this slightly
rule_pat = ( rule_pat = (
"listcomp ::= %sload_closure LOAD_LISTCOMP %%s%s expr " "list_comp ::= %sload_closure LOAD_LISTCOMP %%s%s expr "
"GET_ITER CALL_FUNCTION_1" "GET_ITER CALL_FUNCTION_1"
% ("pos_arg " * pos_args_count, opname) % ("pos_arg " * pos_args_count, opname)
) )
@@ -1348,14 +1350,14 @@ class Python3Parser(PythonParser):
# 'exprs' in the rule above into a # 'exprs' in the rule above into a
# tuple. # tuple.
rule_pat = ( rule_pat = (
"listcomp ::= load_closure LOAD_LISTCOMP %%s%s " "list_comp ::= load_closure LOAD_LISTCOMP %%s%s "
"expr GET_ITER CALL_FUNCTION_1" % (opname,) "expr GET_ITER CALL_FUNCTION_1" % (opname,)
) )
self.add_make_function_rule( self.add_make_function_rule(
rule_pat, opname, token.attr, customize rule_pat, opname, token.attr, customize
) )
rule_pat = ( rule_pat = (
"listcomp ::= %sLOAD_LISTCOMP %%s%s expr " "list_comp ::= %sLOAD_LISTCOMP %%s%s expr "
"GET_ITER CALL_FUNCTION_1" "GET_ITER CALL_FUNCTION_1"
% ("expr " * pos_args_count, opname) % ("expr " * pos_args_count, opname)
) )
@@ -1399,7 +1401,7 @@ class Python3Parser(PythonParser):
# and have GET_ITER CALL_FUNCTION_1 # and have GET_ITER CALL_FUNCTION_1
# Todo: For Pypy we need to modify this slightly # Todo: For Pypy we need to modify this slightly
rule_pat = ( rule_pat = (
"listcomp ::= %sLOAD_LISTCOMP %%s%s expr " "list_comp ::= %sLOAD_LISTCOMP %%s%s expr "
"GET_ITER CALL_FUNCTION_1" "GET_ITER CALL_FUNCTION_1"
% ("expr " * pos_args_count, opname) % ("expr " * pos_args_count, opname)
) )

26
uncompyle6/parsers/parse37base.py
View File

@@ -2,11 +2,10 @@
""" """
Python 3.7 base code. We keep non-custom-generated grammar rules out of this file. Python 3.7 base code. We keep non-custom-generated grammar rules out of this file.
""" """
from uncompyle6.parser import ParserError, PythonParser, nop_func
from uncompyle6.parsers.treenode import SyntaxTree
from spark_parser import DEFAULT_DEBUG as PARSER_DEFAULT_DEBUG from spark_parser import DEFAULT_DEBUG as PARSER_DEFAULT_DEBUG
from spark_parser.spark import rule2str from spark_parser.spark import rule2str
from uncompyle6.parser import ParserError, PythonParser, nop_func
from uncompyle6.parsers.reducecheck import ( from uncompyle6.parsers.reducecheck import (
and_invalid, and_invalid,
ifelsestmt, ifelsestmt,
@@ -16,9 +15,10 @@ from uncompyle6.parsers.reducecheck import (
or_check, or_check,
testtrue, testtrue,
tryelsestmtl3, tryelsestmtl3,
while1stmt,
while1elsestmt, while1elsestmt,
while1stmt,
) )
from uncompyle6.parsers.treenode import SyntaxTree
class Python37BaseParser(PythonParser): class Python37BaseParser(PythonParser):
@@ -54,7 +54,7 @@ class Python37BaseParser(PythonParser):
expr expr
call call
CALL_FUNCTION_3 CALL_FUNCTION_3
""" """
# FIXME: I bet this can be simplified # FIXME: I bet this can be simplified
# look for next MAKE_FUNCTION # look for next MAKE_FUNCTION
for i in range(i + 1, len(tokens)): for i in range(i + 1, len(tokens)):
@@ -104,7 +104,6 @@ class Python37BaseParser(PythonParser):
# organization for this. For example, arrange organize by opcode base? # organization for this. For example, arrange organize by opcode base?
def customize_grammar_rules(self, tokens, customize): def customize_grammar_rules(self, tokens, customize):
is_pypy = False is_pypy = False
# For a rough break out on the first word. This may # For a rough break out on the first word. This may
@@ -348,7 +347,6 @@ class Python37BaseParser(PythonParser):
self.addRule(rule, nop_func) self.addRule(rule, nop_func)
elif opname_base in ("BUILD_MAP", "BUILD_MAP_UNPACK"): elif opname_base in ("BUILD_MAP", "BUILD_MAP_UNPACK"):
if opname == "BUILD_MAP_UNPACK": if opname == "BUILD_MAP_UNPACK":
self.addRule( self.addRule(
""" """
@@ -525,7 +523,6 @@ class Python37BaseParser(PythonParser):
"CALL_FUNCTION_VAR_KW", "CALL_FUNCTION_VAR_KW",
) )
) or opname.startswith("CALL_FUNCTION_KW"): ) or opname.startswith("CALL_FUNCTION_KW"):
if opname == "CALL_FUNCTION" and token.attr == 1: if opname == "CALL_FUNCTION" and token.attr == 1:
rule = """ rule = """
expr ::= dict_comp expr ::= dict_comp
@@ -720,7 +717,9 @@ class Python37BaseParser(PythonParser):
) )
custom_ops_processed.add(opname) custom_ops_processed.add(opname)
elif opname == "LOAD_LISTCOMP": elif opname == "LOAD_LISTCOMP":
self.add_unique_rule("expr ::= listcomp", opname, token.attr, customize) self.add_unique_rule(
"expr ::= list_comp", opname, token.attr, customize
)
custom_ops_processed.add(opname) custom_ops_processed.add(opname)
elif opname == "LOAD_NAME": elif opname == "LOAD_NAME":
if ( if (
@@ -799,7 +798,7 @@ class Python37BaseParser(PythonParser):
# and have GET_ITER CALL_FUNCTION_1 # and have GET_ITER CALL_FUNCTION_1
# Todo: For Pypy we need to modify this slightly # Todo: For Pypy we need to modify this slightly
rule_pat = ( rule_pat = (
"listcomp ::= %sload_closure LOAD_LISTCOMP %%s%s expr " "list_comp ::= %sload_closure LOAD_LISTCOMP %%s%s expr "
"GET_ITER CALL_FUNCTION_1" "GET_ITER CALL_FUNCTION_1"
% ("pos_arg " * args_pos, opname) % ("pos_arg " * args_pos, opname)
) )
@@ -897,14 +896,14 @@ class Python37BaseParser(PythonParser):
# 'exprs' in the rule above into a # 'exprs' in the rule above into a
# tuple. # tuple.
rule_pat = ( rule_pat = (
"listcomp ::= load_closure LOAD_LISTCOMP %%s%s " "list_comp ::= load_closure LOAD_LISTCOMP %%s%s "
"expr GET_ITER CALL_FUNCTION_1" % (opname,) "expr GET_ITER CALL_FUNCTION_1" % (opname,)
) )
self.add_make_function_rule( self.add_make_function_rule(
rule_pat, opname, token.attr, customize rule_pat, opname, token.attr, customize
) )
rule_pat = ( rule_pat = (
"listcomp ::= %sLOAD_LISTCOMP %%s%s expr " "list_comp ::= %sLOAD_LISTCOMP %%s%s expr "
"GET_ITER CALL_FUNCTION_1" % ("expr " * args_pos, opname) "GET_ITER CALL_FUNCTION_1" % ("expr " * args_pos, opname)
) )
self.add_make_function_rule( self.add_make_function_rule(
@@ -938,7 +937,7 @@ class Python37BaseParser(PythonParser):
# and have GET_ITER CALL_FUNCTION_1 # and have GET_ITER CALL_FUNCTION_1
# Todo: For Pypy we need to modify this slightly # Todo: For Pypy we need to modify this slightly
rule_pat = ( rule_pat = (
"listcomp ::= %sLOAD_LISTCOMP %%s%s expr " "list_comp ::= %sLOAD_LISTCOMP %%s%s expr "
"GET_ITER CALL_FUNCTION_1" % ("expr " * args_pos, opname) "GET_ITER CALL_FUNCTION_1" % ("expr " * args_pos, opname)
) )
self.add_make_function_rule( self.add_make_function_rule(
@@ -1259,7 +1258,8 @@ class Python37BaseParser(PythonParser):
if fn: if fn:
return fn(self, lhs, n, rule, ast, tokens, first, last) return fn(self, lhs, n, rule, ast, tokens, first, last)
except Exception: except Exception:
import sys, traceback import sys
import traceback
print( print(
f"Exception in {fn.__name__} {sys.exc_info()[1]}\n" f"Exception in {fn.__name__} {sys.exc_info()[1]}\n"

38
uncompyle6/semantics/gencomp.py
View File

@@ -23,9 +23,9 @@ from xdis import co_flags_is_async, iscode
from uncompyle6.parser import get_python_parser from uncompyle6.parser import get_python_parser
from uncompyle6.scanner import Code from uncompyle6.scanner import Code
from uncompyle6.scanners.tok import Token
from uncompyle6.semantics.consts import PRECEDENCE from uncompyle6.semantics.consts import PRECEDENCE
from uncompyle6.semantics.helper import is_lambda_mode from uncompyle6.semantics.helper import is_lambda_mode
from uncompyle6.scanners.tok import Token
class ComprehensionMixin: class ComprehensionMixin:
@@ -174,7 +174,10 @@ class ComprehensionMixin:
tree = tree[1] tree = tree[1]
pass pass
if tree in ("genexpr_func", "genexpr_func_async",): if tree in (
"genexpr_func",
"genexpr_func_async",
):
for i in range(3, 5): for i in range(3, 5):
if tree[i] == "comp_iter": if tree[i] == "comp_iter":
iter_index = i iter_index = i
@@ -332,8 +335,19 @@ class ComprehensionMixin:
assert store == "store" assert store == "store"
n = set_iter_async[2] n = set_iter_async[2]
elif node == "list_comp" and tree[0] == "expr": elif node == "list_comp" and tree[0] == "expr":
tree = tree[0][0] list_iter = None
n = tree[iter_index] for list_iter_try in tree:
if list_iter_try == "list_iter":
list_iter = list_iter_try
break
if not list_iter_try:
tree = tree[0][0]
n = tree[iter_index]
else:
n = list_iter
pass
pass
pass
else: else:
n = tree[iter_index] n = tree[iter_index]
@@ -407,6 +421,9 @@ class ComprehensionMixin:
n = n[0] n = n[0]
if n in ("list_for", "comp_for"): if n in ("list_for", "comp_for"):
if n == "list_for" and not comp_for and n[0] == "expr":
comp_for = n[0]
n_index = 3 n_index = 3
if ( if (
(n[2] == "store") (n[2] == "store")
@@ -496,11 +513,21 @@ class ComprehensionMixin:
if comp_for: if comp_for:
self.preorder(comp_for) self.preorder(comp_for)
else: else:
try:
node[in_node_index]
except:
from trepan.api import debug
debug()
self.preorder(node[in_node_index]) self.preorder(node[in_node_index])
# Here is where we handle nested list iterations. # Here is where we handle nested list iterations.
if tree == "list_comp" and self.version != (3, 0): if tree == "list_comp" and self.version != (3, 0):
list_iter = tree[1] list_iter = None
for list_iter_try in tree:
if list_iter_try == "list_iter":
list_iter = list_iter_try
break
assert list_iter == "list_iter" assert list_iter == "list_iter"
if list_iter[0] == "list_for": if list_iter[0] == "list_for":
self.preorder(list_iter[0][3]) self.preorder(list_iter[0][3])
@@ -639,7 +666,6 @@ class ComprehensionMixin:
# Find the list comprehension body. It is the inner-most # Find the list comprehension body. It is the inner-most
# node that is not list_.. . # node that is not list_.. .
while n == "list_iter": while n == "list_iter":
# recurse one step # recurse one step
n = n[0] n = n[0]

2
uncompyle6/semantics/n_actions.py
View File

@@ -1036,7 +1036,7 @@ class NonterminalActions:
self.prec = p self.prec = p
self.prune() # stop recursing self.prune() # stop recursing
def n_listcomp(self, node): def n_list_comp(self, node):
self.write("[") self.write("[")
if node[0].kind == "load_closure": if node[0].kind == "load_closure":
assert self.version >= (3, 0) assert self.version >= (3, 0)