jinx/python-uncompyle6
1
Fork 0
mirror of https://github.com/rocky/python-uncompyle6.git synced 2025-08-04 09:22:40 +08:00
Code Issues Packages Projects Releases Wiki Activity

WIP

Browse Source
This commit is contained in:
rocky
2022-03-04 05:07:31 -05:00
parent 05f743ed14
commit 2efe2b5b47
17 changed files with 316 additions and 165 deletions
Download Patch File Download Diff File Expand all files Collapse all files

2
test/simple_source/bug36/02_genexpr.py
View File

@@ -1,5 +1,5 @@
# Python 3.6, uses rule:
# genexpr ::= load_closure load_genexpr LOAD_CONST
# MAKE_FUNCTION_8 expr GET_ITER CALL_FUNCTION_1
# MAKE_FUNCTION_CLOSURE expr GET_ITER CALL_FUNCTION_1
def __sub__(self, other): # SList()-other
return self.__class__(i for i in self if i not in other)

4
test/simple_source/bug36/05_36lambda.py
View File

@@ -4,8 +4,8 @@ def __init__(self, msg = None, digestmod = None):
self.digest_cons = lambda d='': digestmod.new(d)
# From Python 3.6 functools.py
# Bug was handling lambda for MAKE_FUNCTION_8 (closure)
# vs to MAKE_FUNCTION_9 (pos_args + closure)
# Bug was handling lambda for MAKE_FUNCTION_CLOSURE (closure)
# vs to MAKE_FUNCTION_CLOSURE_POS (pos_args + closure)
def bug():
def register(cls, func=None):
return lambda f: register(cls, f)

3
uncompyle6/bin/uncompile.py
View File

@@ -133,7 +133,8 @@ def main_bin():
elif opt in ('--tree+', '-T'):
if 'showast' not in options:
options['showast'] = {}
options['showast']['Full'] = True
options['showast']['after'] = True
options['showast']['before'] = True
options['do_verify'] = None
elif opt in ('--grammar', '-g'):
options['showgrammar'] = True

44
uncompyle6/main.py
View File

@@ -1,4 +1,4 @@
# Copyright (C) 2018-2021 Rocky Bernstein <rocky@gnu.org>
# Copyright (C) 2018-2022 Rocky Bernstein <rocky@gnu.org>
#
# This program is free software: you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
@@ -19,6 +19,7 @@ from xdis import iscode
from xdis.version_info import IS_PYPY, PYTHON_VERSION_TRIPLE, version_tuple_to_str
from uncompyle6.disas import check_object_path
from uncompyle6.semantics import pysource
from uncompyle6.semantics.pysource import PARSER_DEFAULT_DEBUG
from uncompyle6.parser import ParserError
from uncompyle6.version import __version__
@@ -42,9 +43,9 @@ def _get_outstream(outfile):
return open(outfile, 'wb')
def decompile(
bytecode_version,
co,
out=None,
bytecode_version = PYTHON_VERSION_TRIPLE,
out=sys.stdout,
showasm=None,
showast={},
timestamp=None,
@@ -98,7 +99,7 @@ def decompile(
write("# -*- coding: %s -*-" % source_encoding)
write(
"# uncompyle6 version %s\n"
"# %sPython bytecode %s%s\n# Decompiled from: %sPython %s"
"# %sPython bytecode version base %s%s\n# Decompiled from: %sPython %s"
% (
__version__,
co_pypy_str,
@@ -107,10 +108,6 @@ def decompile(
"\n# ".join(sys_version_lines),
)
)
if bytecode_version >= 3.0:
write(
"# Warning: this version of Python has problems handling the Python 3 byte type in constants properly.\n"
)
if co.co_filename:
write("# Embedded file name: %s" % co.co_filename)
if timestamp:
@@ -120,7 +117,17 @@ def decompile(
real_out.write("# Size of source mod 2**32: %d bytes\n" %
source_size)
debug_opts = {"asm": showasm, "ast": showast, "grammar": showgrammar}
# maybe a second -a will do before as well
if showasm:
asm = "after"
else:
asm = None
grammar = dict(PARSER_DEFAULT_DEBUG)
if showgrammar:
grammar["reduce"] = True
debug_opts = {"asm": asm, "tree": showast, "grammar": grammar}
try:
if mapstream:
@@ -128,10 +135,12 @@ def decompile(
mapstream = _get_outstream(mapstream)
deparsed = deparse_code_with_map(
bytecode_version,
co,
out,
bytecode_version,
debug_opts,
showasm,
showast,
showgrammar,
code_objects=code_objects,
is_pypy=is_pypy,
compile_mode=compile_mode,
@@ -182,7 +191,7 @@ def decompile_file(
filename,
outstream=None,
showasm=None,
showast=False,
showast={},
showgrammar=False,
source_encoding=None,
mapstream=None,
@@ -201,11 +210,11 @@ def decompile_file(
if isinstance(co, list):
deparsed = []
for con in co:
for bytecode in co:
deparsed.append(
decompile(
bytecode,
version,
con,
outstream,
showasm,
showast,
@@ -215,14 +224,14 @@ def decompile_file(
code_objects=code_objects,
is_pypy=is_pypy,
magic_int=magic_int,
mapstream=mapstream,
),
mapstream=mapstream,
)
else:
deparsed = [
decompile(
version,
co,
version,
outstream,
showasm,
showast,
@@ -235,6 +244,7 @@ def decompile_file(
magic_int=magic_int,
mapstream=mapstream,
do_fragments=do_fragments,
compile_mode="exec",
)
]
co = None
@@ -249,7 +259,7 @@ def main(
source_files,
outfile=None,
showasm=None,
showast=False,
showast={},
do_verify=False,
showgrammar=False,
source_encoding=None,

6
uncompyle6/parsers/parse36.py
View File

@@ -219,19 +219,19 @@ class Python36Parser(Python35Parser):
formatted_value2 ::= expr expr FORMAT_VALUE_ATTR
"""
self.add_unique_doc_rules(rules_str, customize)
elif opname == 'MAKE_FUNCTION_8':
elif opname == 'MAKE_FUNCTION_CLOSURE':
if 'LOAD_DICTCOMP' in self.seen_ops:
# Is there something general going on here?
rule = """
dict_comp ::= load_closure LOAD_DICTCOMP LOAD_STR
MAKE_FUNCTION_8 expr
MAKE_FUNCTION_CLOSURE expr
GET_ITER CALL_FUNCTION_1
"""
self.addRule(rule, nop_func)
elif 'LOAD_SETCOMP' in self.seen_ops:
rule = """
set_comp ::= load_closure LOAD_SETCOMP LOAD_STR
MAKE_FUNCTION_8 expr
MAKE_FUNCTION_CLOSURE expr
GET_ITER CALL_FUNCTION_1
"""
self.addRule(rule, nop_func)

6
uncompyle6/parsers/parse37.py
View File

@@ -1204,19 +1204,19 @@ class Python37Parser(Python37BaseParser):
formatted_value2 ::= expr expr FORMAT_VALUE_ATTR
"""
self.add_unique_doc_rules(rules_str, customize)
elif opname == "MAKE_FUNCTION_8":
elif opname == "MAKE_FUNCTION_CLOSURE":
if "LOAD_DICTCOMP" in self.seen_ops:
# Is there something general going on here?
rule = """
dict_comp ::= load_closure LOAD_DICTCOMP LOAD_STR
MAKE_FUNCTION_8 expr
MAKE_FUNCTION_CLOSURE expr
GET_ITER CALL_FUNCTION_1
"""
self.addRule(rule, nop_func)
elif "LOAD_SETCOMP" in self.seen_ops:
rule = """
set_comp ::= load_closure LOAD_SETCOMP LOAD_STR
MAKE_FUNCTION_8 expr
MAKE_FUNCTION_CLOSURE expr
GET_ITER CALL_FUNCTION_1
"""
self.addRule(rule, nop_func)

21
uncompyle6/parsers/parse37base.py
View File

@@ -584,6 +584,21 @@ class Python37BaseParser(PythonParser):
"""
self.add_unique_doc_rules(rules_str, customize)
elif opname == "GET_ANEXT":
self.addRule(
"""
func_async_prefix ::= _come_froms SETUP_FINALLY GET_ANEXT LOAD_CONST YIELD_FROM POP_BLOCK
func_async_middle ::= JUMP_FORWARD COME_FROM_EXCEPT
DUP_TOP LOAD_GLOBAL COMPARE_OP POP_JUMP_IF_TRUE
list_afor2 ::= func_async_prefix
store list_iter
JUMP_BACK COME_FROM_FINALLY
END_ASYNC_FOR
""",
nop_func,
)
custom_ops_processed.add(opname)
elif opname == "FORMAT_VALUE_ATTR":
rules_str = """
expr ::= formatted_value2
@@ -932,19 +947,19 @@ class Python37BaseParser(PythonParser):
)
self.add_unique_rule(rule, opname, token.attr, customize)
elif opname == "MAKE_FUNCTION_8":
elif opname == "MAKE_FUNCTION_CLOSURE":
if "LOAD_DICTCOMP" in self.seen_ops:
# Is there something general going on here?
rule = """
dict_comp ::= load_closure LOAD_DICTCOMP LOAD_STR
MAKE_FUNCTION_8 expr
MAKE_FUNCTION_CLOSURE expr
GET_ITER CALL_FUNCTION_1
"""
self.addRule(rule, nop_func)
elif "LOAD_SETCOMP" in self.seen_ops:
rule = """
set_comp ::= load_closure LOAD_SETCOMP LOAD_STR
MAKE_FUNCTION_8 expr
MAKE_FUNCTION_CLOSURE expr
GET_ITER CALL_FUNCTION_1
"""
self.addRule(rule, nop_func)

5
uncompyle6/parsers/parse38.py
View File

@@ -74,6 +74,11 @@ class Python38Parser(Python37Parser):
COME_FROM_FINALLY
END_ASYNC_FOR
genexpr_func_async ::= LOAD_FAST func_async_prefix
store comp_iter
JUMP_BACK COME_FROM_FINALLY
END_ASYNC_FOR
# FIXME: come froms after the else_suite or END_ASYNC_FOR distinguish which of
# for / forelse is used. Add come froms and check of add up control-flow detection phase.
async_forelse_stmt38 ::= expr

8
uncompyle6/scanners/scanner3.py
View File

@@ -397,7 +397,13 @@ class Scanner3(Scanner):
if self.version >= (3, 6):
# 3.6+ doesn't have MAKE_CLOSURE, so opname == 'MAKE_FUNCTION'
flags = argval
opname = "MAKE_FUNCTION_%d" % (flags)
# FIXME: generalize this
if flags == 8:
opname = "MAKE_FUNCTION_CLOSURE"
elif flags == 9:
opname = "MAKE_FUNCTION_CLOSURE_POS"
else:
opname = "MAKE_FUNCTION_%d" % (flags)
attr = []
for flag in self.MAKE_FUNCTION_FLAGS:
bit = flags & 1

6
uncompyle6/semantics/aligner.py
View File

@@ -1,4 +1,4 @@
# Copyright (c) 2018 by Rocky Bernstein
# Copyright (c) 2018, 2022 by Rocky Bernstein
#
# This program is free software: you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
@@ -141,8 +141,8 @@ def code_deparse_align(co, out=sys.stderr, version=None, is_pypy=None,
debug_parser=debug_parser, compile_mode=compile_mode,
is_pypy = is_pypy)
isTopLevel = co.co_name == '<module>'
deparsed.ast = deparsed.build_ast(tokens, customize, co, isTopLevel=isTopLevel)
is_top_level_module = co.co_name == '<module>'
deparsed.ast = deparsed.build_ast(tokens, customize, co, is_top_level_module=is_top_level_module)
assert deparsed.ast == 'stmts', 'Should have parsed grammar start'

35
uncompyle6/semantics/consts.py
View File

@@ -1,4 +1,4 @@
# Copyright (c) 2017-2021 by Rocky Bernstein
# Copyright (c) 2017-2022 by Rocky Bernstein
#
# This program is free software: you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
@@ -44,6 +44,9 @@ maxint = sys.maxint
# say to 100, to make sure we avoid additional prenthesis in
# call((.. op ..)).
NO_PARENTHESIS_EVER = 100
# fmt: off
PRECEDENCE = {
"named_expr": 40, # :=
"yield": 38, # Needs to be below named_expr
@@ -168,11 +171,14 @@ TABLE_R = {
"DELETE_ATTR": ("%|del %c.%[-1]{pattr}\n", 0),
}
TABLE_R0 = {
# "BUILD_LIST": ( "[%C]", (0,-1,", ") ),
# "BUILD_TUPLE": ( "(%C)", (0,-1,", ") ),
# "CALL_FUNCTION": ( "%c(%P)", 0, (1,-1,", ") ),
}
# I'll leave this in for historical interest.
# TABLE_R0 it was like TABLE_R but the key was the *child* of the last child,
# or a grandchild of the node that this is considered.
# TABLE_R0 = {
# "BUILD_LIST": ( "[%C]", (0,-1,", ") ),
# "BUILD_TUPLE": ( "(%C)", (0,-1,", ") ),
# "CALL_FUNCTION": ( "%c(%P)", 0, (1,-1,", ") ),
# }
TABLE_DIRECT = {
"BINARY_ADD": ("+",),
@@ -236,8 +242,19 @@ TABLE_DIRECT = {
(0, "expr", PRECEDENCE["subscript"]),
(1, "expr"),
),
"subscript": ("%p[%c]", (0, "expr", PRECEDENCE["subscript"]), (1, "expr")),
"subscript2": ("%p[%c]", (0, "expr", PRECEDENCE["subscript"]), (1, "expr")),
"subscript": (
"%p[%p]",
(0, "expr", PRECEDENCE["subscript"]),
(1, "expr", NO_PARENTHESIS_EVER)
),
"subscript2": (
"%p[%p]",
(0, "expr", PRECEDENCE["subscript"]),
(1, "expr", NO_PARENTHESIS_EVER)
),
"store_subscript": ("%p[%c]", (0, "expr", PRECEDENCE["subscript"]), (1, "expr")),
"STORE_FAST": ("%{pattr}",),
"STORE_NAME": ("%{pattr}",),
@@ -427,7 +444,6 @@ TABLE_DIRECT = {
MAP_DIRECT = (TABLE_DIRECT,)
MAP_R0 = (TABLE_R0, -1, 0)
MAP_R = (TABLE_R, -1)
MAP = {
@@ -435,7 +451,6 @@ MAP = {
"call": MAP_R,
"delete": MAP_R,
"store": MAP_R,
"exprlist": MAP_R0,
}
ASSIGN_TUPLE_PARAM = lambda param_name: SyntaxTree(

8
uncompyle6/semantics/customize3.py
View File

@@ -154,6 +154,7 @@ def customize_for_version3(self, version):
# recurse one step
n = n[0]
# FIXME: adjust for set comprehension
if n == "list_for":
stores.append(n[2])
n = n[3]
@@ -168,13 +169,12 @@ def customize_for_version3(self, version):
c = c[0]
collections.append(c)
pass
elif n in ("list_if", "list_if_not"):
# FIXME: just a guess
elif n in ("list_if", "list_if_not", "list_if_or_not"):
if n[0].kind == "expr":
list_ifs.append(n)
else:
list_ifs.append([1])
n = n[2]
n = n[-2] if n[-1] == "come_from_opt" else n[-1]
pass
elif n == "list_if37":
list_ifs.append(n)
@@ -184,7 +184,7 @@ def customize_for_version3(self, version):
collections.append(n[0][0])
n = n[1]
stores.append(n[1][0])
n = n[3]
n = n[2] if n[2].kind == "list_iter" else n[3]
pass
assert n == "lc_body", ast

2
uncompyle6/semantics/customize36.py
View File

@@ -338,7 +338,7 @@ def customize_for_version36(self, version):
kwargs = kwargs[0]
call_function_ex = node[-1]
assert call_function_ex == "CALL_FUNCTION_EX_KW" or (
self.version >= 3.6 and call_function_ex == "CALL_FUNCTION_EX"
self.version >= (3, 6) and call_function_ex == "CALL_FUNCTION_EX"
)
# FIXME: decide if the below test be on kwargs == 'dict'
if (

43
uncompyle6/semantics/fragments.py
View File

@@ -1,4 +1,4 @@
# Copyright (c) 2015-2019, 2021 by Rocky Bernstein
# Copyright (c) 2015-2019, 2021-2022 by Rocky Bernstein
#
# This program is free software: you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
@@ -64,6 +64,7 @@ The node position 0 will be associated with "import".
# FIXME: DRY code with pysource
import re
from StringIO import StringIO
from uncompyle6.semantics import pysource
from uncompyle6 import parser
@@ -75,7 +76,7 @@ from uncompyle6.show import maybe_show_asm, maybe_show_tree
from uncompyle6.parsers.treenode import SyntaxTree
from uncompyle6.semantics.pysource import ParserError, StringIO
from uncompyle6.semantics.pysource import ParserError
from xdis import iscode
from xdis.version_info import IS_PYPY, PYTHON_VERSION_TRIPLE
@@ -628,32 +629,6 @@ class FragmentsWalker(pysource.SourceWalker, object):
self.indent_less()
self.prune() # stop recursing
def n_list_comp(self, node):
"""List comprehensions"""
p = self.prec
self.prec = 27
n = node[-1]
assert n == "list_iter"
# find innermost node
while n == "list_iter":
n = n[0] # recurse one step
if n == "list_for":
n = n[3]
elif n == "list_if":
n = n[2]
elif n == "list_if_not":
n = n[2]
assert n == "lc_body"
if node[0].kind.startswith("BUILD_LIST"):
start = len(self.f.getvalue())
self.set_pos_info(node[0], start, start + 1)
self.write("[ ")
self.preorder(n[0]) # lc_body
self.preorder(node[-1]) # for/if parts
self.write(" ]")
self.prec = p
self.prune() # stop recursing
def comprehension_walk(self, node, iter_index, code_index=-5):
p = self.prec
self.prec = 27
@@ -946,7 +921,7 @@ class FragmentsWalker(pysource.SourceWalker, object):
self.set_pos_info(node[0], start - 1, start)
self.comprehension_walk3(node, 1, 0)
elif node[0].kind == "load_closure":
self.setcomprehension_walk3(node, collection_index=4)
self.closure_walk(node, collection_index=4)
else:
self.comprehension_walk(node, iter_index=4)
self.write("}")
@@ -1011,7 +986,7 @@ class FragmentsWalker(pysource.SourceWalker, object):
):
self.set_pos_info(node[1], node[0][0].start, node[0][0].finish)
def setcomprehension_walk3(self, node, collection_index):
def closure_walk(self, node, collection_index):
"""Set comprehensions the way they are done in Python3.
They're more other comprehensions, e.g. set comprehensions
See if we can combine code.
@@ -1185,7 +1160,7 @@ class FragmentsWalker(pysource.SourceWalker, object):
code,
is_lambda=False,
noneInNames=False,
isTopLevel=False,
is_top_level_module=False,
):
# FIXME: DRY with pysource.py
@@ -1227,7 +1202,7 @@ class FragmentsWalker(pysource.SourceWalker, object):
# Python 3.4's classes can add a "return None" which is
# invalid syntax.
if tokens[-2].kind == "LOAD_CONST":
if isTopLevel or tokens[-2].pattr is None:
if is_top_level_module or tokens[-2].pattr is None:
del tokens[-2:]
else:
tokens.append(Token("RETURN_LAST"))
@@ -2102,8 +2077,8 @@ def code_deparse(
is_pypy=is_pypy,
)
isTopLevel = co.co_name == "<module>"
deparsed.ast = deparsed.build_ast(tokens, customize, co, isTopLevel=isTopLevel)
is_top_level_module = co.co_name == "<module>"
deparsed.ast = deparsed.build_ast(tokens, customize, co, is_top_level_module=is_top_level_module)
assert deparsed.ast == "stmts", "Should have parsed grammar start"

284
uncompyle6/semantics/pysource.py
View File

@@ -43,7 +43,7 @@ Python.
# describe rules and not have to create methods at all.
#
# So another other way to specify a semantic rule for a nonterminal is via
# one of the tables MAP_R0, MAP_R, or MAP_DIRECT where the key is the
# either tables MAP_R, or MAP_DIRECT where the key is the
# nonterminal name.
#
# These dictionaries use a printf-like syntax to direct substitution
@@ -63,15 +63,14 @@ Python.
# parse tree for N.
#
#
# N&K N N
# / | ... \ / | ... \ / | ... \
# O O O O O K O O O
# |
# K
# TABLE_DIRECT TABLE_R TABLE_R0
# N&K N
# / | ... \ / | ... \
# O O O O O K
#
#
# TABLE_DIRECT TABLE_R
#
# The default table is TABLE_DIRECT mapping By far, most rules used work this way.
# TABLE_R0 is rarely used.
#
# The key K is then extracted from the subtree and used to find one
# of the tables, T listed above. The result after applying T[K] is
@@ -139,7 +138,7 @@ from xdis.version_info import PYTHON_VERSION_TRIPLE
from uncompyle6.parser import get_python_parser
from uncompyle6.parsers.treenode import SyntaxTree
from spark_parser import GenericASTTraversal, DEFAULT_DEBUG as PARSER_DEFAULT_DEBUG
from spark_parser import GenericASTTraversal
from uncompyle6.scanner import Code, get_scanner
import uncompyle6.parser as python_parser
from uncompyle6.semantics.check_ast import checker
@@ -185,6 +184,25 @@ from StringIO import StringIO
DEFAULT_DEBUG_OPTS = {"asm": False, "tree": False, "grammar": False}
def unicode(x): return x
from StringIO import StringIO
PARSER_DEFAULT_DEBUG = {
"rules": False,
"transition": False,
"reduce": False,
"errorstack": "full",
"context": True,
"dups": False,
}
TREE_DEFAULT_DEBUG = {"before": False, "after": False}
DEFAULT_DEBUG_OPTS = {
"asm": False,
"tree": TREE_DEFAULT_DEBUG,
"grammar": dict(PARSER_DEFAULT_DEBUG),
}
class SourceWalkerError(Exception):
def __init__(self, errmsg):
@@ -202,7 +220,7 @@ class SourceWalker(GenericASTTraversal, object):
version,
out,
scanner,
showast=False,
showast=TREE_DEFAULT_DEBUG,
debug_parser=PARSER_DEFAULT_DEBUG,
compile_mode="exec",
is_pypy=IS_PYPY,
@@ -225,9 +243,9 @@ class SourceWalker(GenericASTTraversal, object):
mode that was used to create the Syntax Tree and specifies a
gramar variant within a Python version to use.
`is_pypy' should be True if the Syntax Tree was generated for PyPy.
`is_pypy` should be True if the Syntax Tree was generated for PyPy.
`linestarts' is a dictionary of line number to bytecode offset. This
`linestarts` is a dictionary of line number to bytecode offset. This
can sometimes assist in determinte which kind of source-code construct
to use when there is ambiguity.
@@ -244,9 +262,10 @@ class SourceWalker(GenericASTTraversal, object):
is_pypy=is_pypy,
)
self.treeTransform = TreeTransform(
version=version, show_ast=showast, is_pypy=is_pypy
)
# Initialize p_lambda on demand
self.p_lambda = None
self.treeTransform = TreeTransform(version=self.version, show_ast=showast)
self.debug_parser = dict(debug_parser)
self.showast = showast
self.params = params
@@ -286,25 +305,28 @@ class SourceWalker(GenericASTTraversal, object):
# An example is:
# __module__ = __name__
self.hide_internal = True
self.compile_mode = "exec"
self.compile_mode = compile_mode
self.name = None
self.version = version
self.is_pypy = is_pypy
customize_for_version(self, is_pypy, version)
return
def maybe_show_tree(self, ast):
if self.showast and self.treeTransform.showast:
def maybe_show_tree(self, ast, phase):
if self.showast.get("before", False):
self.println(
"""
---- end before transform
"""
)
if self.showast.get("after", False):
self.println(
"""
---- begin after transform
"""
+ " "
+ " "
)
if isinstance(self.showast, dict) and self.showast.get:
if self.showast.get(phase, False):
maybe_show_tree(self, ast)
def str_with_template(self, ast):
@@ -586,8 +608,10 @@ class SourceWalker(GenericASTTraversal, object):
self.prec = 6
# print("XXX", n.kind, p, "<", self.prec)
# print(self.f.getvalue())
if p < self.prec:
# print(f"PREC {p}, {node[0].kind}")
self.write("(")
self.preorder(node[0])
self.write(")")
@@ -1111,8 +1135,8 @@ class SourceWalker(GenericASTTraversal, object):
ast = ast[0]
n = ast[iter_index]
assert n == "comp_iter", n
assert n == "comp_iter", n.kind
# Find the comprehension body. It is the inner-most
# node that is not list_.. .
while n == "comp_iter": # list_iter
@@ -1154,10 +1178,24 @@ class SourceWalker(GenericASTTraversal, object):
code_index = -6
if self.version > (3, 6):
# Python 3.7+ adds optional "come_froms" at node[0]
iter_index = 4
if node[0].kind in ("load_closure", "load_genexpr") and self.version >= (3, 8):
is_lambda = self.is_lambda
if node[0].kind == "load_genexpr":
self.is_lambda = False
self.closure_walk(node, collection_index=4)
self.is_lambda = is_lambda
else:
code_index = -6
if self.version < (3, 8):
iter_index = 4
else:
iter_index = 3
self.comprehension_walk(node, iter_index=iter_index, code_index=code_index)
pass
pass
else:
code_index = -5
self.comprehension_walk(node, iter_index=iter_index, code_index=code_index)
self.comprehension_walk(node, iter_index=iter_index, code_index=code_index)
self.write(")")
self.prune()
@@ -1168,7 +1206,7 @@ class SourceWalker(GenericASTTraversal, object):
if node[0] in ["LOAD_SETCOMP", "LOAD_DICTCOMP"]:
self.comprehension_walk_newer(node, 1, 0)
elif node[0].kind == "load_closure" and self.version >= (3, 0):
self.setcomprehension_walk3(node, collection_index=4)
self.closure_walk(node, collection_index=4)
else:
self.comprehension_walk(node, iter_index=4)
self.write("}")
@@ -1180,15 +1218,19 @@ class SourceWalker(GenericASTTraversal, object):
"""Non-closure-based comprehensions the way they are done in Python3
and some Python 2.7. Note: there are also other set comprehensions.
"""
# FIXME: DRY with listcomp_closure3
p = self.prec
self.prec = 27
code_obj = node[code_index].attr
assert iscode(code_obj), node[code_index]
self.debug_opts["asm"]
code = Code(code_obj, self.scanner, self.currentclass, self.debug_opts["asm"])
ast = self.build_ast(code._tokens, code._customize, code)
ast = self.build_ast(
code._tokens, code._customize, code, is_lambda=self.is_lambda
)
self.customize(code._customize)
# skip over: sstmt, stmt, return, return_expr
@@ -1336,7 +1378,6 @@ class SourceWalker(GenericASTTraversal, object):
else:
self.preorder(store)
# FIXME this is all merely approximate
self.write(" in ")
self.preorder(node[in_node_index])
@@ -1356,6 +1397,7 @@ class SourceWalker(GenericASTTraversal, object):
self.write(" if ")
if have_not:
self.write("not ")
pass
self.prec = 27
self.preorder(if_node)
pass
@@ -1375,32 +1417,86 @@ class SourceWalker(GenericASTTraversal, object):
self.write("]")
self.prune()
def setcomprehension_walk3(self, node, collection_index):
"""Set comprehensions the way they are done in Python3.
They're more other comprehensions, e.g. set comprehensions
See if we can combine code.
def get_comprehension_function(self, node, code_index):
"""
Build the body of a comprehension function and then
find the comprehension node buried in the tree which may
be surrounded with start-like symbols or dominiators,.
"""
self.prec = 27
code_node = node[code_index]
if code_node == "load_genexpr":
code_node = code_node[0]
code_obj = code_node.attr
assert iscode(code_obj), code_node
code = Code(code_obj, self.scanner, self.currentclass, self.debug_opts["asm"])
# FIXME: is there a way we can avoid this?
# The problem is that in filterint top-level list comprehensions we can
# encounter comprehensions of other kinds, and lambdas
if self.compile_mode in ("listcomp",): # add other comprehensions to this list
p_save = self.p
self.p = get_python_parser(
self.version, compile_mode="exec", is_pypy=self.is_pypy,
)
tree = self.build_ast(
code._tokens, code._customize, code, is_lambda=self.is_lambda
)
self.p = p_save
else:
tree = self.build_ast(
code._tokens, code._customize, code, is_lambda=self.is_lambda
)
self.customize(code._customize)
# skip over: sstmt, stmt, return, return_expr
# and other singleton derivations
if tree == "lambda_start":
if tree[0] in ("dom_start", "dom_start_opt"):
tree = tree[1]
while len(tree) == 1 or (
tree in ("stmt", "sstmt", "return", "return_expr", "return_expr_lambda")
):
self.prec = 100
tree = tree[0]
return tree
def closure_walk(self, node, collection_index):
"""Dictionary and comprehensions using closure the way they are done in Python3.
"""
p = self.prec
self.prec = 27
code = Code(node[1].attr, self.scanner, self.currentclass)
ast = self.build_ast(code._tokens, code._customize, code)
self.customize(code._customize)
if node[0] == "load_genexpr":
code_index = 0
else:
code_index = 1
tree = self.get_comprehension_function(node, code_index=code_index)
# Remove single reductions as in ("stmts", "sstmt"):
while len(ast) == 1:
ast = ast[0]
while len(tree) == 1:
tree = tree[0]
store = ast[3]
store = tree[3]
collection = node[collection_index]
n = ast[4]
if tree == "genexpr_func_async":
iter_index = 3
else:
iter_index = 4
n = tree[iter_index]
list_if = None
assert n == "comp_iter"
# Find inner-most node.
while n == "comp_iter":
n = n[0] # recurse one step
# FIXME: adjust for set comprehension
if n == "list_for":
store = n[2]
@@ -1419,7 +1515,7 @@ class SourceWalker(GenericASTTraversal, object):
pass
pass
assert n == "comp_body", ast
assert n == "comp_body", tree
self.preorder(n[0])
self.write(" for ")
@@ -1813,6 +1909,7 @@ class SourceWalker(GenericASTTraversal, object):
self.kv_map(node[-1], sep, line_number, indent)
pass
pass
if sep.startswith(",\n"):
self.write(sep[1:])
if node[0] != "dict_entry":
@@ -1874,6 +1971,7 @@ class SourceWalker(GenericASTTraversal, object):
self.write("(")
endchar = ")"
else:
# from trepan.api import debug; debug()
raise TypeError(
"Internal Error: n_build_list expects list, tuple, set, or unpack"
)
@@ -2044,23 +2142,22 @@ class SourceWalker(GenericASTTraversal, object):
index = entry[arg]
if isinstance(index, tuple):
if isinstance(index[1], str):
# if node[index[0]] != index[1]:
# from trepan.api import debug; debug()
assert node[index[0]] == index[1], (
"at %s[%d], expected '%s' node; got '%s'"
% (node.kind, arg, index[1], node[index[0]].kind)
% (node.kind, arg, index[1], node[index[0]].kind,)
)
else:
assert node[index[0]] in index[1], (
"at %s[%d], expected to be in '%s' node; got '%s'"
% (node.kind, arg, index[1], node[index[0]].kind)
% (node.kind, arg, index[1], node[index[0]].kind,)
)
index = index[0]
assert isinstance(
index, int
), "at %s[%d], %s should be int or tuple" % (
node.kind,
arg,
type(index),
assert isinstance(index, int), (
"at %s[%d], %s should be int or tuple"
% (node.kind, arg, type(index),)
)
try:
@@ -2082,10 +2179,17 @@ class SourceWalker(GenericASTTraversal, object):
assert isinstance(tup, tuple)
if len(tup) == 3:
(index, nonterm_name, self.prec) = tup
assert node[index] == nonterm_name, (
"at %s[%d], expected '%s' node; got '%s'"
% (node.kind, arg, nonterm_name, node[index].kind)
)
if isinstance(tup[1], str):
assert node[index] == nonterm_name, (
"at %s[%d], expected '%s' node; got '%s'"
% (node.kind, arg, nonterm_name, node[index].kind,)
)
else:
assert node[tup[0]] in tup[1], (
"at %s[%d], expected to be in '%s' node; got '%s'"
% (node.kind, arg, index[1], node[index[0]].kind,)
)
else:
assert len(tup) == 2
(index, self.prec) = entry[arg]
@@ -2416,10 +2520,10 @@ class SourceWalker(GenericASTTraversal, object):
# print stmt[-1]
# Add "global" declaration statements at the top
globals, nonlocals = find_globals_and_nonlocals(
ast, set(), set(), code, self.version
)
# Add "global" declaration statements at the top
# of the function
for g in sorted(globals):
self.println(indent, "global ", g)
@@ -2458,11 +2562,8 @@ class SourceWalker(GenericASTTraversal, object):
self.println(self.indent, "pass")
else:
self.customize(customize)
if is_lambda:
self.write(self.traverse(ast, is_lambda=is_lambda))
else:
self.text = self.traverse(ast, is_lambda=is_lambda)
self.println(self.text)
self.text = self.traverse(ast, is_lambda=is_lambda)
self.println(self.text)
self.name = old_name
self.return_none = rn
@@ -2473,7 +2574,7 @@ class SourceWalker(GenericASTTraversal, object):
code,
is_lambda=False,
noneInNames=False,
isTopLevel=False,
is_top_level_module=False,
):
# FIXME: DRY with fragments.py
@@ -2500,10 +2601,10 @@ class SourceWalker(GenericASTTraversal, object):
raise ParserError(e, tokens, self.p.debug["reduce"])
except AssertionError, e:
raise ParserError(e, tokens, self.p.debug["reduce"])
transform_ast = self.treeTransform.transform(ast, code)
self.maybe_show_tree(ast)
transform_tree = self.treeTransform.transform(ast, code)
self.maybe_show_tree(ast, phase="after")
del ast # Save memory
return transform_ast
return transform_tree
# The bytecode for the end of the main routine has a
# "return None". However you can't issue a "return" statement in
@@ -2515,7 +2616,7 @@ class SourceWalker(GenericASTTraversal, object):
# Python 3.4's classes can add a "return None" which is
# invalid syntax.
if tokens[-2].kind == "LOAD_CONST":
if isTopLevel or tokens[-2].pattr is None:
if is_top_level_module or tokens[-2].pattr is None:
del tokens[-2:]
else:
tokens.append(Token("RETURN_LAST"))
@@ -2540,12 +2641,12 @@ class SourceWalker(GenericASTTraversal, object):
checker(ast, False, self.ast_errors)
self.customize(customize)
transform_ast = self.treeTransform.transform(ast, code)
transform_tree = self.treeTransform.transform(ast, code)
self.maybe_show_tree(ast)
self.maybe_show_tree(ast, phase="before")
del ast # Save memory
return transform_ast
return transform_tree
@classmethod
def _get_mapping(cls, node):
@@ -2573,16 +2674,13 @@ def code_deparse(
version = PYTHON_VERSION_TRIPLE
# store final output stream for case of error
scanner = get_scanner(version, is_pypy=is_pypy)
scanner = get_scanner(version, is_pypy=is_pypy, show_asm=debug_opts["asm"])
tokens, customize = scanner.ingest(
co, code_objects=code_objects, show_asm=debug_opts["asm"]
)
debug_parser = dict(PARSER_DEFAULT_DEBUG)
if debug_opts.get("grammar", None):
debug_parser["reduce"] = debug_opts["grammar"]
debug_parser["errorstack"] = "full"
debug_parser = debug_opts.get("grammar", dict(PARSER_DEFAULT_DEBUG))
# Build Syntax Tree from disassembly.
linestarts = dict(scanner.opc.findlinestarts(co))
@@ -2590,27 +2688,49 @@ def code_deparse(
version,
out,
scanner,
showast=debug_opts.get("ast", None),
showast=debug_opts.get("tree", TREE_DEFAULT_DEBUG),
debug_parser=debug_parser,
compile_mode=compile_mode,
is_pypy=is_pypy,
linestarts=linestarts,
)
isTopLevel = co.co_name == "<module>"
is_top_level_module = co.co_name == "<module>"
if compile_mode == "eval":
deparsed.hide_internal = False
deparsed.ast = deparsed.build_ast(tokens, customize, co, isTopLevel=isTopLevel)
deparsed.compile_mode = compile_mode
deparsed.ast = deparsed.build_ast(
tokens,
customize,
co,
is_lambda=(compile_mode == "lambda"),
is_top_level_module=is_top_level_module,
)
#### XXX workaround for profiling
if deparsed.ast is None:
return None
if compile_mode != "eval":
assert deparsed.ast == "stmts", "Should have parsed grammar start"
# FIXME use a lookup table here.
if compile_mode == "lambda":
expected_start = "lambda_start"
elif compile_mode == "eval":
expected_start = "expr_start"
elif compile_mode == "expr":
expected_start = "expr_start"
elif compile_mode == "exec":
expected_start = "stmts"
elif compile_mode == "single":
expected_start = "single_start"
else:
assert deparsed.ast == "eval_expr", "Should have parsed grammar start"
expected_start = None
if expected_start:
assert (
deparsed.ast == expected_start
), (
"Should have parsed grammar start to '%s'; got: %s" %
(expected_start, deparsed.ast.kind)
)
# save memory
del tokens
@@ -2652,7 +2772,11 @@ def code_deparse(
# What we've been waiting for: Generate source from Syntax Tree!
deparsed.gen_source(
deparsed.ast, name=co.co_name, customize=customize, debug_opts=debug_opts
deparsed.ast,
name=co.co_name,
customize=customize,
is_lambda=compile_mode == "lambda",
debug_opts=debug_opts,
)
for g in sorted(deparsed.mod_globs):
@@ -2660,7 +2784,7 @@ def code_deparse(
if deparsed.ast_errors:
deparsed.write("# NOTE: have internal decompilation grammar errors.\n")
deparsed.write("# Use -t option to show full context.")
deparsed.write("# Use -T option to show full context.")
for err in deparsed.ast_errors:
deparsed.write(err)
raise SourceWalkerError("Deparsing hit an internal grammar-rule bug")

2
uncompyle6/show.py
View File

@@ -52,7 +52,7 @@ def maybe_show_tree(walker, ast):
stream = sys.stdout
if (
isinstance(walker.showast, dict)
and walker.showast.get("Full", False)
and walker.showast.get("after", False)
and hasattr(walker, "str_with_template")
):
walker.str_with_template(ast)

2
uncompyle6/version.py
View File

@@ -14,4 +14,4 @@
# This file is suitable for sourcing inside POSIX shell as
# well as importing into Python
# fmt: off
__version__="3.8.1.dev0" # noqa
__version__="3.9.0a1" # noqa