jinx/python-uncompyle6
1
Fork 0
mirror of https://github.com/rocky/python-uncompyle6.git synced 2025-08-03 08:49:51 +08:00
Code Issues Packages Projects Releases Wiki Activity

Merge branch 'master' into python-2.4

Browse Source
This commit is contained in:
rocky
2017-12-13 08:58:18 -05:00
parent 204612ca85 54b36bc2d1
commit 5ef2d5cd9f
23 changed files with 138 additions and 49 deletions
Download Patch File Download Diff File Expand all files Collapse all files

BIN
test/bytecode_2.6/01_triple_compare.pyc Normal file
View File

Binary file not shown.

BIN
test/bytecode_2.6/02_ifelse_lambda.pyc Normal file
View File

Binary file not shown.

BIN
test/bytecode_2.7/01_triple_compare.pyc
View File

Binary file not shown.

BIN
test/bytecode_2.7/02_ifelse_lambda.pyc
View File

Binary file not shown.

BIN
test/bytecode_3.2/01_triple_compare.pyc
View File

Binary file not shown.

BIN
test/bytecode_3.3/01_triple_compare.pyc
View File

Binary file not shown.

BIN
test/bytecode_3.6/02_ifelse_lambda.pyc
View File

Binary file not shown.

6
test/simple_source/branching/02_ifelse_lambda.py
View File

@@ -3,8 +3,6 @@
f = lambda x: 1 if x<2 else 3
f(5)
<<<<<<< HEAD
=======
# If that wasn't enough ...
# Python will create dead code
@@ -16,4 +14,6 @@ g()
h = lambda: 1 if False else 3
h()
>>>>>>> master
# From 2.7 test_builtin
lambda c: 'a' <= c <= 'z', 'Hello World'

12
test/simple_source/bug33/01_triple_compare.py
View File

@@ -1,5 +1,13 @@
# In Python 3.3+ this uses grammar rule
# compare_chained2 ::= expr COMPARE_OP RETURN_VALUE
def _is_valid_netmask(self, netmask):
return 0 <= netmask <= self._max_prefixlen
# Seen in Python 3.3 ipaddress.py
def _is_valid_netmask(netmask):
return 0 <= netmask <= 10
# There were also bugs in 2.6- involving the use of "or" twice in its "or"
# detections
# See in 2.6.9 quopri.py ishex():
'0' <= __file__ <= '9' or 'a' <= __file__ <= 'f' or 'A' <= __file__ <= 'F'

47
test/stdlib/runtests.sh
View File

@@ -61,17 +61,48 @@ case $PYVERSION in
[test_ftplib.py]=1 # Control flow?
[test_funcattrs.py]=1 # Control flow?
[test_grp.py]=1 # Long test - might work Control flow?
[test_imp.py]=1
[test_int.py]=1
[test_long.py]=1
[test_pty.py]=1
[test_pwd.py]=1 # Long test - might work? Control flow?
[test_queue.py]=1 # Control flow?
[test_re.py]=1 # Probably Control flow?
[test_strftime.py]=1
[test_trace.py]=1 # Line numbers are expected to be different
# .pyenv/versions/2.6.9/lib/python2.6/lib2to3/refactor.pyc
# .pyenv/versions/2.6.9/lib/python2.6/mailbox.pyc
# .pyenv/versions/2.6.9/lib/python2.6/markupbase.pyc
# .pyenv/versions/2.6.9/lib/python2.6/pstats.pyc
# .pyenv/versions/2.6.9/lib/python2.6/pyclbr.pyc
# .pyenv/versions/2.6.9/lib/python2.6/quopri.pyc -- look at ishex, is short
# .pyenv/versions/2.6.9/lib/python2.6/random.pyc
# .pyenv/versions/2.6.9/lib/python2.6/smtpd.pyc
# .pyenv/versions/2.6.9/lib/python2.6/sre_parse.pyc
# .pyenv/versions/2.6.9/lib/python2.6/tabnanny.pyc
# .pyenv/versions/2.6.9/lib/python2.6/tarfile.pyc
)
;;
2.7)
SKIP_TESTS=(
[test_builtin.py]=1 # Syntax error, look at
[test_dis.py]=1 # We change line numbers - duh!
[test_grammar.py]=1 # Too many stmts. Handle large stmts
[test_io.py]=1 # Test takes too long to run
[test_ioctl.py]=1 # Test takes too long to run
[test_itertools.py]=1 # Syntax error - look at!
[test_memoryio.py]=1 # ?
[test_multiprocessing.py]=1 # ?
[test_pep352.py]=1 # ?
[test_re.py]=1 # ?
[test_sys_settrace.py]=1 # Line numbers are expected to be different
[test_strtod.py]=1
[test_traceback.py]=1
[test_unicode.py]=1
# Syntax errors:
# .pyenv/versions/2.7.14/lib/python2.7/mimify.pyc
# .pyenv/versions/2.7.14/lib/python2.7/netrc.pyc
# .pyenv/versions/2.7.14/lib/python2.7/pyclbr.pyc
# .pyenv/versions/2.7.14/lib/python2.7/sre_compile.pyc
)
;;
*)
@@ -101,13 +132,22 @@ if [[ -n $1 ]] ; then
files=$1
SKIP_TESTS=()
else
files=test_[m]*.py
files=test_*.py
fi
for file in $files; do
[[ -v SKIP_TESTS[$file] ]] && continue
# If the fails *before* decompiling, skip it!
typeset -i STARTTIME=$(date +%s)
if ! python $file >/dev/null 2>&1 ; then
echo "Skipping test $file -- it fails on its own"
continue
fi
typeset -i ENDTIME=$(date +%s)
typeset -i time_diff
(( time_diff = ENDTIME - STARTTIME))
if (( time_diff > 10 )) ; then
echo "Skipping test $file -- test takes too long to run: $time_diff seconds"
continue
fi
@@ -117,10 +157,11 @@ for file in $files; do
decompiled_file=$short_name-${PYVERSION}.pyc
$fulldir/compile-file.py $file && \
mv $file{,.orig} && \
echo ========== $(date +%X) Decompiling $file ===========
$fulldir/../../bin/uncompyle6 $decompiled_file > $file
rc=$?
if (( rc == 0 )) ; then
echo ========== Running $file ===========
echo ========== $(date +%X) Running $file ===========
python $file
rc=$?
else

3
test/test_pyenvlib.py
View File

@@ -101,6 +101,9 @@ def do_tests(src_dir, patterns, target_dir, start_with=None, do_verify=False):
files = [file for file in files if not 'site-packages' in file]
files = [file for file in files if not 'test' in file]
if len(files) > 200:
# print("Numer of files %d - truncating to last 200" % len(files))
# files = files[-200:]
print("Numer of files %d - truncating to first 200" % len(files))
files = files[:200]
print(time.ctime())

14
uncompyle6/parser.py
View File

@@ -266,7 +266,7 @@ class PythonParser(GenericASTBuilder):
stmt ::= while1stmt
stmt ::= whileelsestmt
stmt ::= while1elsestmt
stmt ::= forstmt
stmt ::= for
stmt ::= forelsestmt
stmt ::= try_except
stmt ::= tryelsestmt
@@ -374,20 +374,20 @@ class PythonParser(GenericASTBuilder):
def p_forstmt(self, args):
"""
_for ::= GET_ITER FOR_ITER
for_iter ::= GET_ITER FOR_ITER
for_block ::= l_stmts_opt _come_froms JUMP_BACK
forstmt ::= SETUP_LOOP expr _for store
for_block POP_BLOCK _come_froms
for ::= SETUP_LOOP expr for_iter store
for_block POP_BLOCK _come_froms
forelsestmt ::= SETUP_LOOP expr _for store
forelsestmt ::= SETUP_LOOP expr for_iter store
for_block POP_BLOCK else_suite _come_froms
forelselaststmt ::= SETUP_LOOP expr _for store
forelselaststmt ::= SETUP_LOOP expr for_iter store
for_block POP_BLOCK else_suitec _come_froms
forelselaststmtl ::= SETUP_LOOP expr _for store
forelselaststmtl ::= SETUP_LOOP expr for_iter store
for_block POP_BLOCK else_suitel _come_froms
"""

2
uncompyle6/parsers/parse2.py
View File

@@ -244,7 +244,7 @@ class Python2Parser(PythonParser):
stmt ::= assign2_pypy
assign3_pypy ::= expr expr expr store store store
assign2_pypy ::= expr expr store store
list_comp ::= expr BUILD_LIST_FROM_ARG _for store list_iter
list_comp ::= expr BUILD_LIST_FROM_ARG for_iter store list_iter
JUMP_BACK
""", nop_func)

6
uncompyle6/parsers/parse21.py
View File

@@ -13,11 +13,11 @@ class Python21Parser(Python22Parser):
def p_forstmt21(self, args):
"""
_for ::= LOAD_CONST FOR_LOOP
forstmt ::= SETUP_LOOP expr _for store
for_iter ::= LOAD_CONST FOR_LOOP
for ::= SETUP_LOOP expr for_iter store
return_stmts
POP_BLOCK COME_FROM
forstmt ::= SETUP_LOOP expr _for store
for ::= SETUP_LOOP expr for_iter store
l_stmts_opt _jump_back
POP_BLOCK COME_FROM

4
uncompyle6/parsers/parse22.py
View File

@@ -13,10 +13,10 @@ class Python22Parser(Python23Parser):
def p_misc22(self, args):
'''
_for ::= LOAD_CONST FOR_LOOP
for_iter ::= LOAD_CONST FOR_LOOP
list_iter ::= list_if JUMP_FORWARD
COME_FROM POP_TOP COME_FROM
list_for ::= expr _for store list_iter CONTINUE JUMP_FORWARD
list_for ::= expr for_iter store list_iter CONTINUE JUMP_FORWARD
COME_FROM POP_TOP COME_FROM
'''

2
uncompyle6/parsers/parse23.py
View File

@@ -36,7 +36,7 @@ class Python23Parser(Python24Parser):
COME_FROM POP_TOP POP_BLOCK COME_FROM
list_comp ::= BUILD_LIST_0 DUP_TOP LOAD_ATTR store list_iter del_stmt
list_for ::= expr _for store list_iter JUMP_BACK come_froms POP_TOP JUMP_BACK
list_for ::= expr for_iter store list_iter JUMP_BACK come_froms POP_TOP JUMP_BACK
lc_body ::= LOAD_NAME expr CALL_FUNCTION_1 POP_TOP
lc_body ::= LOAD_FAST expr CALL_FUNCTION_1 POP_TOP

26
uncompyle6/parsers/parse26.py
View File

@@ -180,23 +180,23 @@ class Python26Parser(Python2Parser):
def p_comp26(self, args):
'''
list_for ::= expr _for store list_iter JUMP_BACK come_froms POP_TOP
list_for ::= expr for_iter store list_iter JUMP_BACK come_froms POP_TOP
# The JUMP FORWARD below jumps to the JUMP BACK. It seems to happen
# in rare cases that may have to with length of code
# FIXME: we can add a reduction check for this
list_for ::= expr _for store list_iter JUMP_FORWARD come_froms POP_TOP
list_for ::= expr for_iter store list_iter JUMP_FORWARD come_froms POP_TOP
COME_FROM JUMP_BACK
list_for ::= expr _for store list_iter jb_cont
list_for ::= expr for_iter store list_iter jb_cont
# This is for a really funky:
# [ x for x in range(10) if x % 2 if x % 3 ]
# the JUMP_ABSOLUTE is to the instruction after the last POP_TOP
# we have a reduction check for this
list_for ::= expr _for store list_iter JUMP_ABSOLUTE come_froms
list_for ::= expr for_iter store list_iter JUMP_ABSOLUTE come_froms
POP_TOP jb_pop
list_iter ::= list_if JUMP_BACK
@@ -208,7 +208,7 @@ class Python26Parser(Python2Parser):
lc_body ::= LOAD_NAME expr LIST_APPEND
lc_body ::= LOAD_FAST expr LIST_APPEND
comp_for ::= SETUP_LOOP expr _for store comp_iter jb_pb_come_from
comp_for ::= SETUP_LOOP expr for_iter store comp_iter jb_pb_come_from
comp_body ::= gen_comp_body
@@ -259,10 +259,24 @@ class Python26Parser(Python2Parser):
# compare_chained is like x <= y <= z
compare_chained ::= expr compare_chained1 ROT_TWO COME_FROM POP_TOP _come_froms
compare_chained1 ::= expr DUP_TOP ROT_THREE COMPARE_OP
jmp_false_then compare_chained1 _come_froms
compare_chained1 ::= expr DUP_TOP ROT_THREE COMPARE_OP
jmp_false_then compare_chained2 _come_froms
compare_chained1 ::= expr DUP_TOP ROT_THREE COMPARE_OP
jmp_false compare_chained1 _come_froms
compare_chained1 ::= expr DUP_TOP ROT_THREE COMPARE_OP
jmp_false compare_chained2 _come_froms
compare_chained1 ::= expr DUP_TOP ROT_THREE COMPARE_OP
jmp_false_then compare_chained2 _come_froms
return_lambda ::= RETURN_VALUE
return_lambda ::= RETURN_END_IF
return_lambda ::= RETURN_END_IF_LAMBDA
return_lambda ::= RETURN_VALUE_LAMBDA
compare_chained2 ::= expr COMPARE_OP return_lambda
return_if_lambda ::= RETURN_END_IF_LAMBDA POP_TOP
stmt ::= conditional_lambda
conditional_lambda ::= expr jmp_false_then expr return_if_lambda
@@ -314,7 +328,7 @@ class Python26Parser(Python2Parser):
tokens[last].pattr == jmp_false.pattr)
elif rule == (
'list_for',
('expr', '_for', 'store', 'list_iter',
('expr', 'for_iter', 'store', 'list_iter',
'JUMP_ABSOLUTE', 'come_froms', 'POP_TOP', 'jb_pop')):
# The JUMP_ABSOLUTE has to be to the last POP_TOP or this is invalid
ja_attr = ast[4].attr

11
uncompyle6/parsers/parse27.py
View File

@@ -14,7 +14,7 @@ class Python27Parser(Python2Parser):
def p_comprehension27(self, args):
"""
list_for ::= expr _for store list_iter JUMP_BACK
list_for ::= expr for_iter store list_iter JUMP_BACK
list_comp ::= BUILD_LIST_0 list_iter
lc_body ::= expr LIST_APPEND
@@ -33,7 +33,7 @@ class Python27Parser(Python2Parser):
comp_body ::= dict_comp_body
comp_body ::= set_comp_body
comp_for ::= expr _for store comp_iter JUMP_BACK
comp_for ::= expr for_iter store comp_iter JUMP_BACK
dict_comp_body ::= expr expr MAP_ADD
set_comp_body ::= expr SET_ADD
@@ -82,7 +82,12 @@ class Python27Parser(Python2Parser):
compare_chained1 COME_FROM
compare_chained1 ::= expr DUP_TOP ROT_THREE COMPARE_OP JUMP_IF_FALSE_OR_POP
compare_chained2 COME_FROM
compare_chained2 ::= expr COMPARE_OP RETURN_VALUE
return_lambda ::= RETURN_VALUE
return_lambda ::= RETURN_VALUE_LAMBDA
compare_chained2 ::= expr COMPARE_OP return_lambda
compare_chained2 ::= expr COMPARE_OP return_lambda
# conditional_true are for conditions which always evaluate true
# There is dead or non-optional remnants of the condition code though,

14
uncompyle6/parsers/parse3.py
View File

@@ -36,8 +36,8 @@ class Python3Parser(PythonParser):
# one may be a continue - sometimes classifies a JUMP_BACK
# as a CONTINUE. The two are kind of the same in a comprehension.
comp_for ::= expr _for store comp_iter CONTINUE
comp_for ::= expr _for store comp_iter JUMP_BACK
comp_for ::= expr for_iter store comp_iter CONTINUE
comp_for ::= expr for_iter store comp_iter JUMP_BACK
list_comp ::= BUILD_LIST_0 list_iter
lc_body ::= expr LIST_APPEND
@@ -326,16 +326,16 @@ class Python3Parser(PythonParser):
def p_loop_stmt3(self, args):
"""
forstmt ::= SETUP_LOOP expr _for store for_block POP_BLOCK
for ::= SETUP_LOOP expr for_iter store for_block POP_BLOCK
COME_FROM_LOOP
forelsestmt ::= SETUP_LOOP expr _for store for_block POP_BLOCK else_suite
forelsestmt ::= SETUP_LOOP expr for_iter store for_block POP_BLOCK else_suite
COME_FROM_LOOP
forelselaststmt ::= SETUP_LOOP expr _for store for_block POP_BLOCK else_suitec
forelselaststmt ::= SETUP_LOOP expr for_iter store for_block POP_BLOCK else_suitec
COME_FROM_LOOP
forelselaststmtl ::= SETUP_LOOP expr _for store for_block POP_BLOCK else_suitel
forelselaststmtl ::= SETUP_LOOP expr for_iter store for_block POP_BLOCK else_suitel
COME_FROM_LOOP
whilestmt ::= SETUP_LOOP testexpr l_stmts_opt COME_FROM JUMP_BACK POP_BLOCK
@@ -370,7 +370,7 @@ class Python3Parser(PythonParser):
COME_FROM_LOOP
whileTruestmt ::= SETUP_LOOP l_stmts_opt JUMP_BACK POP_BLOCK NOP
COME_FROM_LOOP
forstmt ::= SETUP_LOOP expr _for store for_block POP_BLOCK NOP
for ::= SETUP_LOOP expr for_iter store for_block POP_BLOCK NOP
COME_FROM_LOOP
"""

18
uncompyle6/scanners/scanner2.py
View File

@@ -715,18 +715,22 @@ class Scanner2(Scanner):
# rocky: if we have a conditional jump to the next instruction, then
# possibly I am "skipping over" a "pass" or null statement.
test_target = target
if self.version < 2.7:
op_testset = set([self.opc.POP_TOP,
self.opc.JUMP_IF_TRUE, self.opc.JUMP_IF_FALSE])
# Before 2.6 we have to deal with the fact that there is an extra
# POP_TOP that is logically associated with the JUMP_IF's (even though
# the instance set is called "self.pop_jump_if")
if code[pre[test_target]] == self.opc.POP_TOP:
test_target = pre[test_target]
test_set = self.pop_jump_if
else:
op_testset = self.pop_jump_if_or_pop | self.pop_jump_if
test_set = self.pop_jump_if_or_pop | self.pop_jump_if
if ( code[pre[target]] in op_testset
and (target > offset) ):
if ( code[pre[test_target]] in test_set and target > offset ):
self.fixed_jumps[offset] = pre[target]
self.structs.append({'type': 'and/or',
'start': start,
'end': pre[target]})
'start': start,
'end': pre[target]})
return
# The op offset just before the target jump offset is important

18
uncompyle6/scanners/scanner3.py
View File

@@ -196,8 +196,12 @@ class Scanner3(Scanner):
# 'LOAD_ASSERT' is used in assert statements.
self.load_asserts = set()
self.insts = list(bytecode)
self.offset2inst_index = {}
n = len(self.insts)
for i, inst in enumerate(self.insts):
self.offset2inst_index[inst.offset] = i
# We need to detect the difference between:
# raise AssertionError
# and
@@ -953,8 +957,18 @@ class Scanner3(Scanner):
self.fixed_jumps[offset] = rtarget
self.not_continue.add(pre_rtarget)
else:
# For now, we'll only tag forward jump.
if self.version >= 3.6:
# FIXME: this is very convoluted and based on rather hacky
# empirical evidence. It should go a way when
# we have better control-flow analysis
normal_jump = self.version >= 3.6
if self.version == 3.5:
j = self.offset2inst_index[target]
if j+2 < len(self.insts) and self.insts[j+2].is_jump_target:
normal_jump = self.insts[j+1].opname == 'POP_BLOCK'
if normal_jump:
# For now, we'll only tag forward jump.
if target > offset:
self.fixed_jumps[offset] = target
pass

2
uncompyle6/semantics/consts.py
View File

@@ -258,7 +258,7 @@ TABLE_DIRECT = {
'while1elsestmt': ( '%|while 1:\n%+%c%-%|else:\n%+%c%-\n\n', 1, -2 ),
'whileelsestmt': ( '%|while %c:\n%+%c%-%|else:\n%+%c%-\n\n', 1, 2, -2 ),
'whileelselaststmt': ( '%|while %c:\n%+%c%-%|else:\n%+%c%-', 1, 2, -2 ),
'forstmt': ( '%|for %c in %c:\n%+%c%-\n\n', 3, 1, 4 ),
'for': ( '%|for %c in %c:\n%+%c%-\n\n', 3, 1, 4 ),
'forelsestmt': (
'%|for %c in %c:\n%+%c%-%|else:\n%+%c%-\n\n', 3, 1, 4, -2 ),
'forelselaststmt': (

2
uncompyle6/semantics/fragments.py
View File

@@ -105,7 +105,7 @@ TABLE_DIRECT_FRAGMENT = {
'whilestmt': ( '%|while%b %c:\n%+%c%-\n\n', 0, 1, 2 ),
'whileelsestmt': ( '%|while%b %c:\n%+%c%-%|else:\n%+%c%-\n\n', 0, 1, 2, -2 ),
'whileelselaststmt': ( '%|while%b %c:\n%+%c%-%|else:\n%+%c%-', 0, 1, 2, -2 ),
'forstmt': ( '%|for%b %c in %c:\n%+%c%-\n\n', 0, 3, 1, 4 ),
'for': ( '%|for%b %c in %c:\n%+%c%-\n\n', 0, 3, 1, 4 ),
}