jinx/python-uncompyle6
1
Fork 0
mirror of https://github.com/rocky/python-uncompyle6.git synced 2025-08-03 08:49:51 +08:00
Code Issues Packages Projects Releases Wiki Activity

Merge branch 'master' into python-2.4

Browse Source
This commit is contained in:
rocky
2018-01-31 16:45:53 -05:00
parent 4ce769399f a753e2c08f
commit 75755c8cfc
7 changed files with 114 additions and 56 deletions
Download Patch File Download Diff File Expand all files Collapse all files

25
pytest/test_deparse.py
View File

@@ -1,3 +1,4 @@
import pytest
from uncompyle6.semantics.fragments import deparse_code as deparse
from uncompyle6 import PYTHON_VERSION, PYTHON3
@@ -32,19 +33,20 @@ def get_parsed_for_fn(fn):
code = fn.func_code
return deparse(PYTHON_VERSION, code)
def check_expect(expect, parsed):
def check_expect(expect, parsed, fn_name):
debug = False
i = 2
max_expect = len(expect)
for name, offset in sorted(parsed.offsets.keys()):
assert i+1 <= max_expect, "ran out if items in testing node"
assert i+1 <= max_expect, (
"%s: ran out if items in testing node" % fn_name)
nodeInfo = parsed.offsets[name, offset]
node = nodeInfo.node
extractInfo = parsed.extract_node_info(node)
assert expect[i] == extractInfo.selectedLine, \
('line %s expect:\n%s\ngot:\n%s' %
(i, expect[i], extractInfo.selectedLine))
('%s: line %s expect:\n%s\ngot:\n%s' %
(fn_name, i, expect[i], extractInfo.selectedLine))
assert expect[i+1] == extractInfo.markerLine, \
('line %s expect:\n%s\ngot:\n%s' %
(i+1, expect[i+1], extractInfo.markerLine))
@@ -72,6 +74,7 @@ def check_expect(expect, parsed):
pass
@pytest.mark.skip(reason='needs reworking')
def test_stuff():
parsed = get_parsed_for_fn(map_stmts)
expect = """
@@ -83,10 +86,10 @@ return (x, y)
-------------
0
x = []
--
-
Contained in...
x = []
------
--
3
x = []
-
@@ -95,10 +98,10 @@ x = []
------
6
y = {}
--
-
Contained in...
y = {}
------
--
9
y = {}
-
@@ -130,7 +133,7 @@ Contained in...
x = [] ...
------ ...
""".split("\n")
check_expect(expect, parsed)
check_expect(expect, parsed, 'map_stmts')
########################################################
# return
@@ -167,7 +170,7 @@ Contained in...
return (x, y)
-------------
""".split("\n")
check_expect(expect, parsed)
check_expect(expect, parsed, 'return_stmt')
########################################################
# # try
@@ -315,4 +318,4 @@ for i in range(2): ...
""".split("\n")
parsed = get_parsed_for_fn(for_range_stmt)
if not PYTHON3:
check_expect(expect, parsed)
check_expect(expect, parsed, 'range_stmt')

4
test/stdlib/runtests.sh
View File

@@ -79,7 +79,7 @@ case $PYVERSION in
[test_ioctl.py]=1 # Test takes too long to run
[test_itertools.py]=1 # Syntax error - look at!
[test_memoryio.py]=1 # FIX
[test_multiprocessing.py]=1 # FIX
[test_multiprocessing.py]=1 # On uncompyle2, taks 24 secs
[test_pep352.py]=1 # ?
[test_select.py]=1 # Runs okay but takes 11 seconds
[test_socket.py]=1 # Runs ok but takes 22 seconds
@@ -105,7 +105,7 @@ cd $srcdir
fulldir=$(pwd)
# DECOMPILER=uncompyle2
DECOMPILER="$fulldir/../../bin/uncompyle6"
DECOMPILER=${DECOMPILER:-"$fulldir/../../bin/uncompyle6"}
TESTDIR=/tmp/test${PYVERSION}
if [[ -e $TESTDIR ]] ; then
rm -fr $TESTDIR

9
uncompyle6/bin/uncompile.py
View File

@@ -33,6 +33,7 @@ Options:
-d print timestamps
-p <integer> use <integer> number of processes
-r recurse directories looking for .pyc and .pyo files
--fragments use fragments deparser
--verify compare generated source with input byte-code
--verify-run compile generated source, run it and check exit code
--weak-verify compile generated source
@@ -84,9 +85,11 @@ def main_bin():
try:
opts, files = getopt.getopt(sys.argv[1:], 'hagtdrVo:c:p:',
'help asm grammar linemaps recurse timestamp tree '
'verify verify-run version showgrammar'.split(' '))
except getopt.GetoptError, e:
'fragments verify verify-run version weak-verify '
'showgrammar'.split(' '))
except getopt.GetoptError(e):
sys.stderr.write('%s: %s\n' % (os.path.basename(sys.argv[0]), e))
print('%s: %s' % (os.path.basename(sys.argv[0]), e), file=sys.stderr)
sys.exit(-1)
options = {}
@@ -101,6 +104,8 @@ def main_bin():
options['do_verify'] = 'strong'
elif opt == '--weak-verify':
options['do_verify'] = 'weak'
elif opt == '--fragments':
options['do_fragments'] = True
elif opt == '--verify-run':
options['do_verify'] = 'verify-run'
elif opt == '--linemaps':

40
uncompyle6/main.py
View File

@@ -6,9 +6,10 @@ from uncompyle6.disas import check_object_path
from uncompyle6.semantics import pysource
from uncompyle6.parser import ParserError
from uncompyle6.version import VERSION
from uncompyle6.linenumbers import line_number_mapping
# from uncompyle6.linenumbers import line_number_mapping
from uncompyle6.semantics.pysource import deparse_code
from uncompyle6.semantics.fragments import deparse_code as deparse_code_fragments
from uncompyle6.semantics.linemap import deparse_code_with_map
from xdis.load import load_module
@@ -28,7 +29,7 @@ def decompile(
bytecode_version, co, out=None, showasm=None, showast=False,
timestamp=None, showgrammar=False, code_objects={},
source_size=None, is_pypy=False, magic_int=None,
mapstream=None):
mapstream=None, do_fragments=False):
"""
ingests and deparses a given code block 'co'
@@ -89,9 +90,13 @@ def decompile(
sorted(deparsed.source_linemap.keys())]
mapstream.write("\n\n# %s\n" % linemap)
else:
deparsed = deparse_code(bytecode_version, co, out, showasm, showast,
showgrammar, code_objects=code_objects,
is_pypy=is_pypy)
if do_fragments:
deparse_fn = deparse_code_fragments
else:
deparse_fn = deparse_code
deparsed = deparse_fn(bytecode_version, co, out, showasm, showast,
showgrammar, code_objects=code_objects,
is_pypy=is_pypy)
pass
return deparsed
except pysource.SourceWalkerError, e:
@@ -99,7 +104,7 @@ def decompile(
raise pysource.SourceWalkerError(str(e))
def decompile_file(filename, outstream=None, showasm=None, showast=False,
showgrammar=False, mapstream=None):
showgrammar=False, mapstream=None, do_fragments=False):
"""
decompile Python byte-code file (.pyc). Return objects to
all of the deparsed objects found in `filename`.
@@ -123,7 +128,7 @@ def decompile_file(filename, outstream=None, showasm=None, showast=False,
timestamp, showgrammar,
code_objects=code_objects, source_size=source_size,
is_pypy=is_pypy, magic_int=magic_int,
mapstream=mapstream)]
mapstream=mapstream, do_fragments=do_fragments)]
co = None
return deparsed
@@ -132,7 +137,7 @@ def decompile_file(filename, outstream=None, showasm=None, showast=False,
def main(in_base, out_base, files, codes, outfile=None,
showasm=None, showast=False, do_verify=False,
showgrammar=False, raise_on_error=False,
do_linemaps=False):
do_linemaps=False, do_fragments=False):
"""
in_base base directory for input files
out_base base directory for output files (ignored when
@@ -196,7 +201,24 @@ def main(in_base, out_base, files, codes, outfile=None,
# Try to uncompile the input file
try:
decompile_file(infile, outstream, showasm, showast, showgrammar, linemap_stream)
deparsed = decompile_file(infile, outstream, showasm, showast, showgrammar,
linemap_stream, do_fragments)
if do_fragments:
for d in deparsed:
last_mod = None
offsets = d.offsets
for e in sorted(offsets.keys()):
if e[0] != last_mod:
line = '=' * len(e[0])
outstream.write("%s\n%s\n%s\n" % (line, e[0], line))
last_mod = e[0]
info = offsets[e]
extractInfo = d.extract_node_info(info)
outstream.write("%s" % info.node.format().strip() + "\n")
outstream.write(extractInfo.selectedLine + "\n")
outstream.write(extractInfo.markerLine + "\n\n")
pass
pass
tot_files += 1
except (ValueError, SyntaxError, ParserError, pysource.SourceWalkerError):
sys.stdout.write("\n")

16
uncompyle6/scanner.py
View File

@@ -1,4 +1,4 @@
# Copyright (c) 2016-2017 by Rocky Bernstein
# Copyright (c) 2016-2018 by Rocky Bernstein
# Copyright (c) 2005 by Dan Pascu <dan@windowmaker.org>
# Copyright (c) 2000-2002 by hartmut Goebel <h.goebel@crazy-compilers.com>
# Copyright (c) 1999 John Aycock
@@ -14,7 +14,8 @@ import sys
from uncompyle6 import PYTHON3, IS_PYPY
from uncompyle6.scanners.tok import Token
from xdis.bytecode import op_size
import xdis
from xdis.bytecode import op_size, extended_arg_val
from xdis.magics import py_str2float, canonic_python_version
from xdis.util import code2num
@@ -103,14 +104,13 @@ class Scanner(object):
target += pos + 3
return target
# FIXME: the below can be removed after xdis version 3.6.1 has been released
def extended_arg_val(self, val):
return val << self.opc.EXTENDED_ARG_SHIFT
def get_argument(self, pos):
arg = self.code[pos+1] + self.code[pos+2] * 256
return arg
def next_offset(self, op, offset):
return xdis.next_offset(op, self.opc, offset)
def print_bytecode(self):
for i in self.op_range(0, len(self.code)):
op = self.code[i]
@@ -188,7 +188,7 @@ class Scanner(object):
if op == self.opc.EXTENDED_ARG:
arg = code2num(code, offset+1) | extended_arg
extended_arg = self.extended_arg_val(arg)
extended_arg = extended_arg_val(self.opc, arg)
continue
if op in instr:
@@ -240,7 +240,7 @@ class Scanner(object):
if op == self.opc.EXTENDED_ARG:
arg = code2num(code, offset+1) | extended_arg
extended_arg = self.extended_arg_val(arg)
extended_arg = extended_arg_val(self.opc, arg)
continue
if op in instr:

74
uncompyle6/semantics/fragments.py
View File

@@ -56,6 +56,7 @@ from xdis.code import iscode
from uncompyle6.semantics import pysource
from uncompyle6 import parser
from uncompyle6.scanner import Token, Code, get_scanner
import uncompyle6.parser as python_parser
from uncompyle6.semantics.check_ast import checker
from uncompyle6.show import (
@@ -70,7 +71,7 @@ from uncompyle6.semantics.pysource import (
from uncompyle6.semantics.consts import (
INDENT_PER_LEVEL, NONE, PRECEDENCE,
TABLE_DIRECT, escape, minint, MAP
TABLE_DIRECT, escape, MAP, PASS
)
from spark_parser import DEFAULT_DEBUG as PARSER_DEFAULT_DEBUG
@@ -986,17 +987,28 @@ class FragmentsWalker(pysource.SourceWalker, object):
self.name = old_name
self.return_none = rn
def build_ast(self, tokens, customize, is_lambda=False, noneInNames=False):
# assert type(tokens) == ListType
def build_ast(self, tokens, customize, is_lambda=False,
noneInNames=False, isTopLevel=False):
# FIXME: DRY with pysource.py
# assert isinstance(tokens[0], Token)
if is_lambda:
for t in tokens:
if t.kind == 'RETURN_END_IF':
t.kind = 'RETURN_END_IF_LAMBDA'
elif t.kind == 'RETURN_VALUE':
t.kind = 'RETURN_VALUE_LAMBDA'
tokens.append(Token('LAMBDA_MARKER'))
try:
ast = parser.parse(self.p, tokens, customize)
except parser.ParserError(e):
raise ParserError(e, tokens)
except AssertionError(e):
# FIXME: have p.insts update in a better way
# modularity is broken here
p_insts = self.p.insts
self.p.insts = self.scanner.insts
ast = python_parser.parse(self.p, tokens, customize)
self.p.insts = p_insts
except (parser.ParserError(e), AssertionError(e)):
raise ParserError(e, tokens)
maybe_show_ast(self.showast, ast)
return ast
@@ -1011,19 +1023,30 @@ class FragmentsWalker(pysource.SourceWalker, object):
#
# NOTE: this differs from behavior in pysource.py
if len(tokens) >= 2 and not noneInNames:
if tokens[-1].kind == 'RETURN_VALUE':
if tokens[-2].kind != 'LOAD_CONST':
tokens.append(Token('RETURN_LAST'))
if len(tokens) == 0:
return
if self.hide_internal:
if len(tokens) >= 2 and not noneInNames:
if tokens[-1].kind in ('RETURN_VALUE', 'RETURN_VALUE_LAMBDA'):
# Python 3.4's classes can add a "return None" which is
# invalid syntax.
if tokens[-2].kind == 'LOAD_CONST':
if isTopLevel or tokens[-2].pattr is None:
del tokens[-2:]
else:
tokens.append(Token('RETURN_LAST'))
else:
tokens.append(Token('RETURN_LAST'))
if len(tokens) == 0:
return PASS
# Build AST from disassembly.
try:
# FIXME: have p.insts update in a better way
# modularity is broken here
p_insts = self.p.insts
self.p.insts = self.scanner.insts
ast = parser.parse(self.p, tokens, customize)
except parser.ParserError(e):
raise ParserError(e, tokens)
except AssertionError(e):
self.p.insts = p_insts
except (parser.ParserError(e), AssertionError(e)):
raise ParserError(e, tokens)
maybe_show_ast(self.showast, ast)
@@ -1303,6 +1326,7 @@ class FragmentsWalker(pysource.SourceWalker, object):
sep = INDENT_PER_LEVEL[:-1]
start = len(self.f.getvalue())
self.write('{')
self.set_pos_info(node[0], start, start+1)
if self.version > 3.0:
if node[0].kind.startswith('kvlist'):
@@ -1371,9 +1395,6 @@ class FragmentsWalker(pysource.SourceWalker, object):
sep = line_seperator
self.write('}')
finish = len(self.f.getvalue())
for n in node:
n.parent = node
self.set_pos_info(n, start, finish)
self.set_pos_info(node, start, finish)
self.indent_less(INDENT_PER_LEVEL)
self.prec = p
@@ -1626,7 +1647,8 @@ class FragmentsWalker(pysource.SourceWalker, object):
pass
def deparse_code(version, co, out=StringIO(), showasm=False, showast=False,
showgrammar=False, is_pypy=False, walker=FragmentsWalker):
showgrammar=False, code_objects={}, compile_mode='exec',
is_pypy=False, walker=FragmentsWalker):
"""
Convert the code object co into a python source fragment.
@@ -1654,7 +1676,8 @@ def deparse_code(version, co, out=StringIO(), showasm=False, showast=False,
# store final output stream for case of error
scanner = get_scanner(version, is_pypy=is_pypy)
tokens, customize = scanner.ingest(co)
tokens, customize = scanner.ingest(co, code_objects=code_objects,
show_asm=showasm)
tokens, customize = scanner.ingest(co)
maybe_show_asm(showasm, tokens)
@@ -1667,7 +1690,8 @@ def deparse_code(version, co, out=StringIO(), showasm=False, showast=False,
# Build AST from disassembly.
# deparsed = pysource.FragmentsWalker(out, scanner, showast=showast)
deparsed = walker(version, scanner, showast=showast,
debug_parser=debug_parser)
debug_parser=debug_parser, compile_mode=compile_mode,
is_pypy=is_pypy)
deparsed.ast = deparsed.build_ast(tokens, customize)
@@ -1771,7 +1795,7 @@ if __name__ == '__main__':
return
def deparse_test_around(offset, name, co, is_pypy=IS_PYPY):
sys_version = sys.version_info.major + (sys.version_info.minor / 10.0)
sys_version = sys.version_info[0] + (sys.version_info[1] / 10.0)
walk = deparse_code_around_offset(name, offset, sys_version, co, showasm=False, showast=False,
showgrammar=False, is_pypy=IS_PYPY)
print("deparsed source")
@@ -1800,6 +1824,8 @@ if __name__ == '__main__':
return
def get_code_for_fn(fn):
if hasattr(fn, 'func_code'):
return fn.func_code
return fn.__code__
def test():
@@ -1822,5 +1848,5 @@ if __name__ == '__main__':
# deparse_test(get_code_for_fn(FragmentsWalker.fixup_offsets))
# deparse_test(get_code_for_fn(FragmentsWalker.n_list))
print('=' * 30)
deparse_test_around(408, 'n_list', get_code_for_fn(FragmentsWalker.n_build_list))
deparse_test_around(408, 'n_list', get_code_for_fn(FragmentsWalker.n_list))
# deparse_test(inspect.currentframe().f_code)

2
uncompyle6/semantics/pysource.py
View File

@@ -2540,6 +2540,8 @@ class SourceWalker(GenericASTTraversal, object):
def build_ast(self, tokens, customize, is_lambda=False,
noneInNames=False, isTopLevel=False):
# FIXME: DRY with fragments.py
# assert isinstance(tokens[0], Token)
if is_lambda: