jinx/python-uncompyle6
1
Fork 0
mirror of https://github.com/rocky/python-uncompyle6.git synced 2025-08-02 16:44:46 +08:00
Code Issues Packages Projects Releases Wiki Activity

Misc: long lists, DRY 2/3 grammars, '%' count

Browse Source 
parse{2,3,r}.py: DRY Python expressions between Python 2 and 3
pysource.py, fragment.py, parser.py: handle long lists by grouping in chunks of 32
and 256
bin/uncompyle6: count %s properly
This commit is contained in:
rocky
2016-05-02 21:20:17 -04:00
parent ceb47aba9c
commit feec241da8
10 changed files with 210 additions and 291 deletions
Download Patch File Download Diff File Expand all files Collapse all files

2
bin/uncompyle6
View File

@@ -47,7 +47,7 @@ Extensions of generated files:
'.pyc_dis' '.pyo_dis' successfully decompiled (and verified if --verify)
+ '_unverified' successfully decompile but --verify failed
+ '_failed' decompile failed (contact author for enhancement)
""" % ((program,) * 6
""" % ((program,) * 5)
program = os.path.basename(__file__)

BIN
test/bytecode_2.7/05_long_list.pyc Normal file
View File

Binary file not shown.

BIN
test/bytecode_3.5/05_long_list.pyc Normal file
View File

Binary file not shown.

3
test/simple_source/expression/05_long_list.py Normal file
View File

@@ -0,0 +1,3 @@
# Long lists pose a slowdown in uncompiling.
x = [1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1]
print(x)

16
uncompyle6/main.py
View File

@@ -118,14 +118,14 @@ def main(in_base, out_base, files, codes, outfile=None,
os.remove(outfile)
sys.stderr.write("\nLast file: %s " % (infile))
raise
except:
failed_files += 1
if outfile:
outstream.close()
os.rename(outfile, outfile + '_failed')
else:
sys.stderr.write("\n# %s" % sys.exc_info()[1])
sys.stderr.write("\n# Can't uncompile %s\n" % infile)
# except:
# failed_files += 1
# if outfile:
# outstream.close()
# os.rename(outfile, outfile + '_failed')
# else:
# sys.stderr.write("\n# %s" % sys.exc_info()[1])
# sys.stderr.write("\n# Can't uncompile %s\n" % infile)
else: # uncompile successful
if outfile:
outstream.close()

148
uncompyle6/parser.py
View File

@@ -237,6 +237,154 @@ class PythonParser(GenericASTBuilder):
comp_ifnot ::= expr jmp_true comp_iter
"""
def p_expr(self, args):
'''
expr ::= _mklambda
expr ::= SET_LINENO
expr ::= LOAD_FAST
expr ::= LOAD_NAME
expr ::= LOAD_CONST
expr ::= LOAD_GLOBAL
expr ::= LOAD_DEREF
expr ::= load_attr
expr ::= binary_expr
expr ::= binary_expr_na
expr ::= build_list
expr ::= cmp
expr ::= mapexpr
expr ::= and
expr ::= and2
expr ::= or
expr ::= unary_expr
expr ::= call_function
expr ::= unary_not
expr ::= unary_convert
expr ::= binary_subscr
expr ::= binary_subscr2
expr ::= load_attr
expr ::= get_iter
expr ::= slice0
expr ::= slice1
expr ::= slice2
expr ::= slice3
expr ::= buildslice2
expr ::= buildslice3
expr ::= yield
binary_expr ::= expr expr binary_op
binary_op ::= BINARY_ADD
binary_op ::= BINARY_MULTIPLY
binary_op ::= BINARY_AND
binary_op ::= BINARY_OR
binary_op ::= BINARY_XOR
binary_op ::= BINARY_SUBTRACT
binary_op ::= BINARY_DIVIDE
binary_op ::= BINARY_TRUE_DIVIDE
binary_op ::= BINARY_FLOOR_DIVIDE
binary_op ::= BINARY_MODULO
binary_op ::= BINARY_LSHIFT
binary_op ::= BINARY_RSHIFT
binary_op ::= BINARY_POWER
unary_expr ::= expr unary_op
unary_op ::= UNARY_POSITIVE
unary_op ::= UNARY_NEGATIVE
unary_op ::= UNARY_INVERT
unary_not ::= expr UNARY_NOT
unary_convert ::= expr UNARY_CONVERT
binary_subscr ::= expr expr BINARY_SUBSCR
binary_subscr2 ::= expr expr DUP_TOPX_2 BINARY_SUBSCR
load_attr ::= expr LOAD_ATTR
get_iter ::= expr GET_ITER
slice0 ::= expr SLICE+0
slice0 ::= expr DUP_TOP SLICE+0
slice1 ::= expr expr SLICE+1
slice1 ::= expr expr DUP_TOPX_2 SLICE+1
slice2 ::= expr expr SLICE+2
slice2 ::= expr expr DUP_TOPX_2 SLICE+2
slice3 ::= expr expr expr SLICE+3
slice3 ::= expr expr expr DUP_TOPX_3 SLICE+3
buildslice3 ::= expr expr expr BUILD_SLICE_3
buildslice2 ::= expr expr BUILD_SLICE_2
yield ::= expr YIELD_VALUE
_mklambda ::= load_closure mklambda
_mklambda ::= mklambda
or ::= expr JUMP_IF_TRUE_OR_POP expr COME_FROM
or ::= expr jmp_true expr _come_from
and ::= expr jmp_false expr _come_from
and ::= expr JUMP_IF_FALSE_OR_POP expr COME_FROM
and2 ::= _jump jmp_false COME_FROM expr COME_FROM
expr ::= conditional
conditional ::= expr jmp_false expr JUMP_FORWARD expr COME_FROM
conditional ::= expr jmp_false expr JUMP_ABSOLUTE expr
expr ::= conditionalnot
conditionalnot ::= expr jmp_true expr _jump expr COME_FROM
ret_expr ::= expr
ret_expr ::= ret_and
ret_expr ::= ret_or
ret_expr_or_cond ::= ret_expr
ret_expr_or_cond ::= ret_cond
ret_expr_or_cond ::= ret_cond_not
ret_and ::= expr JUMP_IF_FALSE_OR_POP ret_expr_or_cond COME_FROM
ret_or ::= expr JUMP_IF_TRUE_OR_POP ret_expr_or_cond COME_FROM
ret_cond ::= expr POP_JUMP_IF_FALSE expr RETURN_END_IF ret_expr_or_cond
ret_cond_not ::= expr POP_JUMP_IF_TRUE expr RETURN_END_IF ret_expr_or_cond
stmt ::= return_lambda
stmt ::= conditional_lambda
return_lambda ::= ret_expr RETURN_VALUE LAMBDA_MARKER
conditional_lambda ::= expr jmp_false return_if_stmt return_stmt LAMBDA_MARKER
cmp ::= cmp_list
cmp ::= compare
compare ::= expr expr COMPARE_OP
cmp_list ::= expr cmp_list1 ROT_TWO POP_TOP
_come_from
cmp_list1 ::= expr DUP_TOP ROT_THREE
COMPARE_OP JUMP_IF_FALSE_OR_POP
cmp_list1 COME_FROM
cmp_list1 ::= expr DUP_TOP ROT_THREE
COMPARE_OP jmp_false
cmp_list1 _come_from
cmp_list1 ::= expr DUP_TOP ROT_THREE
COMPARE_OP JUMP_IF_FALSE_OR_POP
cmp_list2 COME_FROM
cmp_list1 ::= expr DUP_TOP ROT_THREE
COMPARE_OP jmp_false
cmp_list2 _come_from
cmp_list2 ::= expr COMPARE_OP JUMP_FORWARD
cmp_list2 ::= expr COMPARE_OP RETURN_VALUE
mapexpr ::= BUILD_MAP kvlist
kvlist ::= kvlist kv
kvlist ::= kvlist kv2
kvlist ::= kvlist kv3
kvlist ::=
kv ::= DUP_TOP expr ROT_TWO expr STORE_SUBSCR
kv2 ::= DUP_TOP expr expr ROT_THREE STORE_SUBSCR
kv3 ::= expr expr STORE_MAP
exprlist ::= exprlist expr
exprlist ::= expr
nullexprlist ::=
expr32 ::= expr expr expr expr expr expr expr expr expr expr expr expr expr expr expr expr expr expr expr expr expr expr expr expr expr expr expr expr expr expr expr expr
expr1024 ::= expr32 expr32 expr32 expr32 expr32 expr32 expr32 expr32 expr32 expr32 expr32 expr32 expr32 expr32 expr32 expr32 expr32 expr32 expr32 expr32 expr32 expr32 expr32 expr32 expr32 expr32 expr32 expr32 expr32 expr32 expr32 expr32
'''
def parse(p, tokens, customize):
p.add_custom_rules(tokens, customize)

137
uncompyle6/parsers/parse2.py
View File

@@ -362,69 +362,10 @@ class Python2Parser(PythonParser):
'''
def p_expr(self, args):
def p_expr2(self, args):
'''
expr ::= _mklambda
expr ::= SET_LINENO
expr ::= LOAD_FAST
expr ::= LOAD_NAME
expr ::= LOAD_CONST
expr ::= LOAD_GLOBAL
expr ::= LOAD_DEREF
expr ::= LOAD_LOCALS
expr ::= load_attr
expr ::= binary_expr
expr ::= binary_expr_na
expr ::= build_list
expr ::= cmp
expr ::= mapexpr
expr ::= and
expr ::= and2
expr ::= or
expr ::= unary_expr
expr ::= call_function
expr ::= unary_not
expr ::= unary_convert
expr ::= binary_subscr
expr ::= binary_subscr2
expr ::= load_attr
expr ::= get_iter
expr ::= slice0
expr ::= slice1
expr ::= slice2
expr ::= slice3
expr ::= buildslice2
expr ::= buildslice3
expr ::= yield
binary_expr ::= expr expr binary_op
binary_op ::= BINARY_ADD
binary_op ::= BINARY_MULTIPLY
binary_op ::= BINARY_AND
binary_op ::= BINARY_OR
binary_op ::= BINARY_XOR
binary_op ::= BINARY_SUBTRACT
binary_op ::= BINARY_DIVIDE
binary_op ::= BINARY_TRUE_DIVIDE
binary_op ::= BINARY_FLOOR_DIVIDE
binary_op ::= BINARY_MODULO
binary_op ::= BINARY_LSHIFT
binary_op ::= BINARY_RSHIFT
binary_op ::= BINARY_POWER
unary_expr ::= expr unary_op
unary_op ::= UNARY_POSITIVE
unary_op ::= UNARY_NEGATIVE
unary_op ::= UNARY_INVERT
unary_not ::= expr UNARY_NOT
unary_convert ::= expr UNARY_CONVERT
binary_subscr ::= expr expr BINARY_SUBSCR
binary_subscr2 ::= expr expr DUP_TOPX_2 BINARY_SUBSCR
load_attr ::= expr LOAD_ATTR
get_iter ::= expr GET_ITER
slice0 ::= expr SLICE+0
slice0 ::= expr DUP_TOP SLICE+0
slice1 ::= expr expr SLICE+1
@@ -433,79 +374,6 @@ class Python2Parser(PythonParser):
slice2 ::= expr expr DUP_TOPX_2 SLICE+2
slice3 ::= expr expr expr SLICE+3
slice3 ::= expr expr expr DUP_TOPX_3 SLICE+3
buildslice3 ::= expr expr expr BUILD_SLICE_3
buildslice2 ::= expr expr BUILD_SLICE_2
yield ::= expr YIELD_VALUE
_mklambda ::= load_closure mklambda
_mklambda ::= mklambda
or ::= expr JUMP_IF_TRUE_OR_POP expr COME_FROM
or ::= expr jmp_true expr _come_from
and ::= expr jmp_false expr _come_from
and ::= expr JUMP_IF_FALSE_OR_POP expr COME_FROM
and2 ::= _jump jmp_false COME_FROM expr COME_FROM
expr ::= conditional
conditional ::= expr jmp_false expr JUMP_FORWARD expr COME_FROM
conditional ::= expr jmp_false expr JUMP_ABSOLUTE expr
expr ::= conditionalnot
conditionalnot ::= expr jmp_true expr _jump expr COME_FROM
ret_expr ::= expr
ret_expr ::= ret_and
ret_expr ::= ret_or
ret_expr_or_cond ::= ret_expr
ret_expr_or_cond ::= ret_cond
ret_expr_or_cond ::= ret_cond_not
ret_and ::= expr JUMP_IF_FALSE_OR_POP ret_expr_or_cond COME_FROM
ret_or ::= expr JUMP_IF_TRUE_OR_POP ret_expr_or_cond COME_FROM
ret_cond ::= expr POP_JUMP_IF_FALSE expr RETURN_END_IF ret_expr_or_cond
ret_cond_not ::= expr POP_JUMP_IF_TRUE expr RETURN_END_IF ret_expr_or_cond
stmt ::= return_lambda
stmt ::= conditional_lambda
return_lambda ::= ret_expr RETURN_VALUE LAMBDA_MARKER
conditional_lambda ::= expr jmp_false return_if_stmt return_stmt LAMBDA_MARKER
cmp ::= cmp_list
cmp ::= compare
compare ::= expr expr COMPARE_OP
cmp_list ::= expr cmp_list1 ROT_TWO POP_TOP
_come_from
cmp_list1 ::= expr DUP_TOP ROT_THREE
COMPARE_OP JUMP_IF_FALSE_OR_POP
cmp_list1 COME_FROM
cmp_list1 ::= expr DUP_TOP ROT_THREE
COMPARE_OP jmp_false
cmp_list1 _come_from
cmp_list1 ::= expr DUP_TOP ROT_THREE
COMPARE_OP JUMP_IF_FALSE_OR_POP
cmp_list2 COME_FROM
cmp_list1 ::= expr DUP_TOP ROT_THREE
COMPARE_OP jmp_false
cmp_list2 _come_from
cmp_list2 ::= expr COMPARE_OP JUMP_FORWARD
cmp_list2 ::= expr COMPARE_OP RETURN_VALUE
mapexpr ::= BUILD_MAP kvlist
kvlist ::= kvlist kv
kvlist ::= kvlist kv2
kvlist ::= kvlist kv3
kvlist ::=
kv ::= DUP_TOP expr ROT_TWO expr STORE_SUBSCR
kv2 ::= DUP_TOP expr expr ROT_THREE STORE_SUBSCR
kv3 ::= expr expr STORE_MAP
exprlist ::= exprlist expr
exprlist ::= expr
nullexprlist ::=
'''
def add_custom_rules(self, tokens, customize):
@@ -535,7 +403,8 @@ class Python2Parser(PythonParser):
op = k[:k.rfind('_')]
if op in ('BUILD_LIST', 'BUILD_TUPLE', 'BUILD_SET'):
rule = 'build_list ::= ' + 'expr '*v + k
rule = ('build_list ::= ' + 'expr1024 '*(v//1024) +
'expr32 '*((v//32)%32) + 'expr '*(v%32) + k)
elif op in ('UNPACK_TUPLE', 'UNPACK_SEQUENCE'):
rule = 'unpack ::= ' + k + ' designator'*v
elif op == 'UNPACK_LIST':

141
uncompyle6/parsers/parse3.py
View File

@@ -371,145 +371,10 @@ class Python3Parser(PythonParser):
'''
def p_expr(self, args):
def p_expr3(self, args):
'''
expr ::= _mklambda
expr ::= SET_LINENO
expr ::= LOAD_FAST
expr ::= LOAD_NAME
expr ::= LOAD_CONST
expr ::= LOAD_GLOBAL
expr ::= LOAD_DEREF
expr ::= LOAD_LOCALS
expr ::= LOAD_CLASSNAME
expr ::= load_attr
expr ::= binary_expr
expr ::= binary_expr_na
expr ::= build_list
expr ::= cmp
expr ::= mapexpr
expr ::= and
expr ::= and2
expr ::= or
expr ::= unary_expr
expr ::= call_function
expr ::= unary_not
expr ::= unary_convert
expr ::= binary_subscr
expr ::= binary_subscr2
expr ::= load_attr
expr ::= get_iter
expr ::= slice0
expr ::= slice1
expr ::= slice2
expr ::= slice3
expr ::= buildslice2
expr ::= buildslice3
expr ::= yield
binary_expr ::= expr expr binary_op
binary_op ::= BINARY_ADD
binary_op ::= BINARY_MULTIPLY
binary_op ::= BINARY_AND
binary_op ::= BINARY_OR
binary_op ::= BINARY_XOR
binary_op ::= BINARY_SUBTRACT
binary_op ::= BINARY_DIVIDE
binary_op ::= BINARY_TRUE_DIVIDE
binary_op ::= BINARY_FLOOR_DIVIDE
binary_op ::= BINARY_MODULO
binary_op ::= BINARY_LSHIFT
binary_op ::= BINARY_RSHIFT
binary_op ::= BINARY_POWER
unary_expr ::= expr unary_op
unary_op ::= UNARY_POSITIVE
unary_op ::= UNARY_NEGATIVE
unary_op ::= UNARY_INVERT
unary_not ::= expr UNARY_NOT
unary_convert ::= expr UNARY_CONVERT
binary_subscr ::= expr expr BINARY_SUBSCR
binary_subscr2 ::= expr expr DUP_TOPX_2 BINARY_SUBSCR
load_attr ::= expr LOAD_ATTR
get_iter ::= expr GET_ITER
# Python3 drops slice0..slice3
buildslice3 ::= expr expr expr BUILD_SLICE_3
buildslice2 ::= expr expr BUILD_SLICE_2
yield ::= expr YIELD_VALUE
_mklambda ::= load_closure mklambda
_mklambda ::= mklambda
or ::= expr jmp_true expr _come_from
or ::= expr JUMP_IF_TRUE_OR_POP expr COME_FROM
and ::= expr jmp_false expr _come_from
and ::= expr JUMP_IF_FALSE_OR_POP expr COME_FROM
and2 ::= _jump jmp_false COME_FROM expr COME_FROM
expr ::= conditional
conditional ::= expr jmp_false expr JUMP_FORWARD expr COME_FROM
conditional ::= expr jmp_false expr JUMP_ABSOLUTE expr
expr ::= conditionalnot
conditionalnot ::= expr jmp_true expr _jump expr COME_FROM
ret_expr ::= expr
ret_expr ::= ret_and
ret_expr ::= ret_or
ret_expr_or_cond ::= ret_expr
ret_expr_or_cond ::= ret_cond
ret_expr_or_cond ::= ret_cond_not
ret_and ::= expr JUMP_IF_FALSE_OR_POP ret_expr_or_cond COME_FROM
ret_or ::= expr JUMP_IF_TRUE_OR_POP ret_expr_or_cond COME_FROM
ret_cond ::= expr POP_JUMP_IF_FALSE expr RETURN_END_IF ret_expr_or_cond
ret_cond_not ::= expr POP_JUMP_IF_TRUE expr RETURN_END_IF ret_expr_or_cond
stmt ::= return_lambda
stmt ::= conditional_lambda
return_lambda ::= ret_expr RETURN_VALUE LAMBDA_MARKER
conditional_lambda ::= expr jmp_false return_if_stmt return_stmt LAMBDA_MARKER
cmp ::= cmp_list
cmp ::= compare
compare ::= expr expr COMPARE_OP
cmp_list ::= expr cmp_list1 ROT_TWO POP_TOP
_come_from
cmp_list1 ::= expr DUP_TOP ROT_THREE
COMPARE_OP JUMP_IF_FALSE_OR_POP
cmp_list1 COME_FROM
cmp_list1 ::= expr DUP_TOP ROT_THREE
COMPARE_OP jmp_false
cmp_list1 _come_from
cmp_list1 ::= expr DUP_TOP ROT_THREE
COMPARE_OP JUMP_IF_FALSE_OR_POP
cmp_list2 COME_FROM
cmp_list1 ::= expr DUP_TOP ROT_THREE
COMPARE_OP jmp_false
cmp_list2 _come_from
cmp_list2 ::= expr COMPARE_OP JUMP_FORWARD
cmp_list2 ::= expr COMPARE_OP RETURN_VALUE
mapexpr ::= BUILD_MAP kvlist
kvlist ::= kvlist kv
kvlist ::= kvlist kv2
kvlist ::= kvlist kv3
kvlist ::=
kv ::= DUP_TOP expr ROT_TWO expr STORE_SUBSCR
kv2 ::= DUP_TOP expr expr ROT_THREE STORE_SUBSCR
kv3 ::= expr expr STORE_MAP
exprlist ::= exprlist expr
exprlist ::= expr
nullexprlist ::=
'''
@staticmethod
@@ -613,7 +478,9 @@ class Python3Parser(PythonParser):
elif opname == 'LOAD_BUILD_CLASS':
self.custom_build_class_rule(opname, i, token, tokens, customize)
elif opname_base in ('BUILD_LIST', 'BUILD_TUPLE', 'BUILD_SET'):
rule = 'build_list ::= ' + 'expr ' * token.attr + opname
v = token.attr
rule = ('build_list ::= ' + 'expr1024 ' * int(v//1024) +
'expr32 ' * int((v//32)%32) + 'expr '*(v%32) + opname)
self.add_unique_rule(rule, opname, token.attr, customize)
elif self.version >= 3.5 and opname_base == 'BUILD_MAP':
kvlist_n = "kvlist_%s" % token.attr

18
uncompyle6/semantics/fragments.py
View File

@@ -973,13 +973,29 @@ class FragmentsWalker(pysource.SourceWalker, object):
else:
raise RuntimeError('Internal Error: n_build_list expects list or tuple')
flat_elems = []
for elem in node:
if elem == 'expr1024':
for subelem in elem:
for subsubelem in subelem:
flat_elems.append(subsubelem)
elif elem == 'expr32':
for subelem in elem:
flat_elems.append(subelem)
else:
flat_elems.append(elem)
self.indentMore(INDENT_PER_LEVEL)
if len(node) > 3:
line_separator = ',\n' + self.indent
else:
line_separator = ', '
sep = INDENT_PER_LEVEL[:-1]
for elem in node:
# FIXME:
# if flat_elems > some_number, then group
# do automatic wrapping
for elem in flat_elems:
if (elem == 'ROT_THREE'):
continue

36
uncompyle6/semantics/pysource.py
View File

@@ -1195,33 +1195,49 @@ class SourceWalker(GenericASTTraversal, object):
"""
p = self.prec
self.prec = 100
lastnode = node.pop().type
if lastnode.startswith('BUILD_LIST'):
lastnode = node.pop()
lastnodetype = lastnode.type
if lastnodetype.startswith('BUILD_LIST'):
self.write('['); endchar = ']'
elif lastnode.startswith('BUILD_TUPLE'):
elif lastnodetype.startswith('BUILD_TUPLE'):
self.write('('); endchar = ')'
elif lastnode.startswith('BUILD_SET'):
elif lastnodetype.startswith('BUILD_SET'):
self.write('{'); endchar = '}'
elif lastnode.startswith('ROT_TWO'):
elif lastnodetype.startswith('ROT_TWO'):
self.write('('); endchar = ')'
else:
raise 'Internal Error: n_build_list expects list or tuple'
flat_elems = []
for elem in node:
if elem == 'expr1024':
for subelem in elem:
for subsubelem in subelem:
flat_elems.append(subsubelem)
elif elem == 'expr32':
for subelem in elem:
flat_elems.append(subelem)
else:
flat_elems.append(elem)
self.indentMore(INDENT_PER_LEVEL)
if len(node) > 3:
if lastnode.attr > 3:
line_separator = ',\n' + self.indent
else:
line_separator = ', '
sep = INDENT_PER_LEVEL[:-1]
for elem in node:
if (elem == 'ROT_THREE'):
continue
# FIXME:
# if flat_elems > some_number, then group
# do automatic wrapping
for elem in flat_elems:
if elem == 'ROT_THREE':
continue
assert elem == 'expr'
value = self.traverse(elem)
self.write(sep, value)
sep = line_separator
if len(node) == 1 and lastnode.startswith('BUILD_TUPLE'):
if lastnode.attr == 1 and lastnodetype.startswith('BUILD_TUPLE'):
self.write(',')
self.write(endchar)
self.indentLess(INDENT_PER_LEVEL)