jinx/python-uncompyle6
1
Fork 0
mirror of https://github.com/rocky/python-uncompyle6.git synced 2025-08-03 00:45:53 +08:00
Code Issues Packages Projects Releases Wiki Activity

Merge branch 'master' into python-2.4

Browse Source
This commit is contained in:
rocky
2018-03-24 10:55:43 -04:00
parent 1462a8beb0 2ae9cd7d08
commit b2c832e19f
15 changed files with 170 additions and 119 deletions
Download Patch File Download Diff File Expand all files Collapse all files

1
pytest/test_grammar.py
View File

@@ -51,7 +51,6 @@ def test_grammar():
else:
expect_right_recursive.add((('l_stmts',
('lastl_stmt', 'COME_FROM', 'l_stmts'))))
# expect_lhs.add('kwargs1')
pass
pass
pass

BIN
test/bytecode_3.4_run/04_call_function.pyc Normal file
View File

Binary file not shown.

BIN
test/bytecode_3.5/04_call_function.pyc
View File

Binary file not shown.

BIN
test/bytecode_3.6/04_call_function.pyc
View File

Binary file not shown.

BIN
test/bytecode_3.6/04_try_finally.pyc
View File

Binary file not shown.

BIN
test/bytecode_3.6_run/04_call_function.pyc Normal file
View File

Binary file not shown.

11
test/simple_source/bug35/04_call_function.py
View File

@@ -10,3 +10,14 @@ def tometadata(self, metadata, schema, Table, args, name=None):
def _strptime_datetime(cls, args):
return cls(*args)
# From 3.5.5 imaplib
# Bug is in parsing *date_time[:6] parameter
from datetime import datetime, timezone, timedelta
import time
def Time2Internaldate(date_time):
delta = timedelta(seconds=0)
return datetime(*date_time[:6], tzinfo=timezone(delta))
assert Time2Internaldate(time.localtime())

15
test/simple_source/bug36/04_try_finally.py
View File

@@ -31,3 +31,18 @@ def handle_read(self):
return why
return data
# From 3.6 contextlib
# Bug is indentation of "return exc"
# Also there are extra statements to remove exec,
# which we hide (unless doing fragments).
# Note: The indentation bug may be a result of using improper
# grammar.
def __exit__(self, type, value, traceback):
try:
value()
except StopIteration as exc:
return exc
except RuntimeError as exc:
return exc
return

3
test/stdlib/runtests.sh
View File

@@ -78,7 +78,8 @@ case $PYVERSION in
[test_curses.py]=1 # Possibly fails on its own but not detected
[test_dis.py]=1 # We change line numbers - duh!
[test_doctest.py]=1 # Fails on its own
[test_grammar.py]=1 # Too many stmts. Handle large stmts
[test_generators.py]=1 # control flow. uncompyle2 has problem here too
[test_grammar.py]=1 # Too many stmts. Handle large stmts
[test_io.py]=1 # Test takes too long to run
[test_ioctl.py]=1 # Test takes too long to run
[test_itertools.py]=1 # Fix erroneous reduction to "conditional_true".

63
uncompyle6/parsers/parse3.py
View File

@@ -113,9 +113,9 @@ class Python3Parser(PythonParser):
continues ::= continue
kwarg ::= LOAD_CONST expr
kwargs ::= kwarg*
kwargs1 ::= kwarg+
kwarg ::= LOAD_CONST expr
kwargs ::= kwarg+
classdef ::= build_class store
@@ -393,9 +393,6 @@ class Python3Parser(PythonParser):
'''
load_genexpr ::= LOAD_GENEXPR
load_genexpr ::= BUILD_TUPLE_1 LOAD_GENEXPR LOAD_CONST
# Is there something general going on here?
dict_comp ::= load_closure LOAD_DICTCOMP LOAD_CONST MAKE_CLOSURE_0 expr GET_ITER CALL_FUNCTION_1
'''
def p_expr3(self, args):
@@ -531,7 +528,7 @@ class Python3Parser(PythonParser):
subclassing is, well, is pretty base. And we want it that way: lean and
mean so that parsing will go faster.
Here, we add additional grammra rules based on specific instructions
Here, we add additional grammar rules based on specific instructions
that are in the instruction/token stream. In classes that
inherit from from here and other versions, grammar rules may
also be removed.
@@ -575,6 +572,10 @@ class Python3Parser(PythonParser):
seen_LOAD_BUILD_CLASS = False
seen_GET_AWAITABLE_YIELD_FROM = False
# This is used in parse36.py as well as here
self.seen_LOAD_DICTCOMP = False
# Loop over instructions adding custom grammar rules based on
# a specific instruction seen.
@@ -656,9 +657,9 @@ class Python3Parser(PythonParser):
('dict ' * token.attr) +
'BUILD_MAP_UNPACK')
else:
rule = kvlist_n + ' ::= ' + 'expr ' * (token.attr*2)
rule = "%s ::= %s %s" % (kvlist_n, 'expr ' * (token.attr*2), opname)
self.add_unique_rule(rule, opname, token.attr, customize)
rule = "dict ::= %s %s" % (kvlist_n, opname)
rule = "dict ::= %s" % kvlist_n
else:
rule = kvlist_n + ' ::= ' + 'expr expr STORE_MAP ' * token.attr
self.add_unique_rule(rule, opname, token.attr, customize)
@@ -784,6 +785,7 @@ class Python3Parser(PythonParser):
self.addRule("expr ::= LOAD_CLASSNAME", nop_func)
custom_ops_seen.add(opname)
elif opname == 'LOAD_DICTCOMP':
self.seen_LOAD_DICTCOMP = True
if has_get_iter_call_function1:
rule_pat = ("dict_comp ::= LOAD_DICTCOMP %sMAKE_FUNCTION_0 expr "
"GET_ITER CALL_FUNCTION_1")
@@ -816,6 +818,17 @@ class Python3Parser(PythonParser):
# DRY with MAKE_FUNCTION
# Note: this probably doesn't handle kwargs proprerly
if opname == 'MAKE_CLOSURE_0' and self.seen_LOAD_DICTCOMP:
# Is there something general going on here?
# Note that 3.6+ doesn't do this, but we'll remove
# this rule in parse36.py
rule = """
dict_comp ::= load_closure LOAD_DICTCOMP LOAD_CONST
MAKE_CLOSURE_0 expr
GET_ITER CALL_FUNCTION_1
"""
self.addRule(rule, nop_func)
args_pos, args_kw, annotate_args = token.attr
# FIXME: Fold test into add_make_function_rule
@@ -854,13 +867,13 @@ class Python3Parser(PythonParser):
opname, token.attr, customize)
if args_kw > 0:
kwargs_str = 'kwargs1 '
kwargs_str = 'kwargs '
else:
kwargs_str = ''
# Note order of kwargs and pos args changed between 3.3-3.4
if self.version <= 3.2:
rule = ('mkfunc ::= %s%sload_closure LOAD_CONST kwargs %s'
rule = ('mkfunc ::= %s%sload_closure LOAD_CONST %s'
% (kwargs_str, 'expr ' * args_pos, opname))
elif self.version == 3.3:
rule = ('mkfunc ::= %s%sload_closure LOAD_CONST LOAD_CONST %s'
@@ -968,29 +981,39 @@ class Python3Parser(PythonParser):
opname))
self.add_make_function_rule(rule_pat, opname, token.attr, customize)
if args_kw == 0:
kwargs = 'no_kwargs'
self.add_unique_rule("no_kwargs ::=", opname, token.attr, customize)
else:
kwargs = 'kwargs'
if self.version < 3.3:
# positional args after keyword args
rule = ('mkfunc ::= kwargs %s%s %s' %
rule = ('mkfunc ::= %s %s%s%s' %
(kwargs, 'pos_arg ' * args_pos, 'LOAD_CONST ',
opname))
self.add_unique_rule(rule, opname, token.attr, customize)
rule = ('mkfunc ::= %s%s%s' %
('pos_arg ' * args_pos, 'LOAD_CONST ',
opname))
elif self.version == 3.3:
# positional args after keyword args
rule = ('mkfunc ::= kwargs %s%s %s' %
('pos_arg ' * args_pos, 'LOAD_CONST '*2,
rule = ('mkfunc ::= %s %s%s%s' %
(kwargs, 'pos_arg ' * args_pos, 'LOAD_CONST '*2,
opname))
elif self.version > 3.5:
# positional args before keyword args
rule = ('mkfunc ::= %skwargs1 %s %s' %
('pos_arg ' * args_pos, 'LOAD_CONST '*2,
rule = ('mkfunc ::= %s%s %s%s' %
('pos_arg ' * args_pos, kwargs, 'LOAD_CONST '*2,
opname))
elif self.version > 3.3:
# positional args before keyword args
rule = ('mkfunc ::= %skwargs %s %s' %
('pos_arg ' * args_pos, 'LOAD_CONST '*2,
rule = ('mkfunc ::= %s%s %s%s' %
('pos_arg ' * args_pos, kwargs, 'LOAD_CONST '*2,
opname))
else:
rule = ('mkfunc ::= kwargs %sexpr %s' %
('pos_arg ' * args_pos, opname))
rule = ('mkfunc ::= %s%sexpr %s' %
(kwargs, 'pos_arg ' * args_pos, opname))
self.add_unique_rule(rule, opname, token.attr, customize)
if opname.startswith('MAKE_FUNCTION_A'):
if self.version >= 3.6:

11
uncompyle6/parsers/parse36.py
View File

@@ -35,9 +35,6 @@ class Python36Parser(Python35Parser):
# 3.6 redoes how return_closure works. FIXME: Isolate to LOAD_CLOSURE
return_closure ::= LOAD_CLOSURE DUP_TOP STORE_NAME RETURN_VALUE RETURN_LAST
# Is there something general going on here? FIXME: Isolate to LOAD_DICTCOMP
dict_comp ::= load_closure LOAD_DICTCOMP LOAD_CONST MAKE_FUNCTION_8 expr GET_ITER CALL_FUNCTION_1
stmt ::= conditional_lambda
conditional_lambda ::= expr jmp_false expr return_if_lambda
return_stmt_lambda LAMBDA_MARKER
@@ -105,6 +102,14 @@ class Python36Parser(Python35Parser):
fstring_single ::= expr FORMAT_VALUE
"""
self.add_unique_doc_rules(rules_str, customize)
elif opname == 'MAKE_FUNCTION_8' and self.seen_LOAD_DICTCOMP:
# Is there something general going on here?
rule = """
dict_comp ::= load_closure LOAD_DICTCOMP LOAD_CONST
MAKE_FUNCTION_8 expr
GET_ITER CALL_FUNCTION_1
"""
self.addRule(rule, nop_func)
elif opname == 'BEFORE_ASYNC_WITH':
rules_str = """
stmt ::= async_with_stmt

4
uncompyle6/semantics/consts.py
View File

@@ -288,7 +288,10 @@ TABLE_DIRECT = {
'except': ( '%|except:\n%+%c%-', 3 ),
'except_cond1': ( '%|except %c:\n', 1 ),
'except_suite': ( '%+%c%-%C', 0, (1, maxint, '') ),
# In Python 3.6, this is more complicated in the presence of "returns"
'except_suite_finalize': ( '%+%c%-%C', 1, (3, maxint, '') ),
'pass': ( '%|pass\n', ),
'STORE_FAST': ( '%{pattr}', ),
'kv': ( '%c: %c', 3, 1 ),
@@ -381,7 +384,6 @@ PRECEDENCE = {
'ret_cond_not': 28,
'_mklambda': 30,
'call_kw': 100, # 100 seems to to be module/function precidence
'yield': 101,
'yield_from': 101
}

46
uncompyle6/semantics/customize.py
View File

@@ -17,7 +17,7 @@
"""
from uncompyle6.semantics.consts import (
INDENT_PER_LEVEL, TABLE_R, TABLE_DIRECT)
PRECEDENCE, INDENT_PER_LEVEL, TABLE_R, TABLE_DIRECT)
from uncompyle6.semantics.make_function import (
make_function3_annotate,
@@ -239,6 +239,7 @@ def customize_for_version(self, is_pypy, version):
TABLE_DIRECT.update({
'LOAD_CLASSDEREF': ( '%{pattr}', ),
})
########################
# Python 3.5+ Additions
#######################
@@ -312,7 +313,7 @@ def customize_for_version(self, is_pypy, version):
template = ('*%P)', (0, len(args_node)-1, ', *', 100))
self.template_engine(template, args_node)
else:
template = ('*%c)', -2)
template = ('*%c, %C)', 1, (2, -1, ', '))
self.template_engine(template, node)
self.prune()
@@ -353,6 +354,10 @@ def customize_for_version(self, is_pypy, version):
# Python 3.6+ Additions
#######################
# Value 100 is important; it is exactly
# module/function precidence.
PRECEDENCE['call_kw'] = 100
TABLE_DIRECT.update({
'tryfinally36': ( '%|try:\n%+%c%-%|finally:\n%+%c%-\n\n',
(1, 'returns'), 3 ),
@@ -484,7 +489,7 @@ def customize_for_version(self, is_pypy, version):
self.n_call_ex_kw3 = call_ex_kw3
def call_ex_kw4(node):
"""Handle CALL_FUNCTION_EX 2 (have KW) but without
"""Handle CALL_FUNCTION_EX {1 or 2} (have KW) but without
BUILD_{MAP,TUPLE}_UNPACK_WITH_CALL"""
self.preorder(node[0])
self.write('(')
@@ -503,8 +508,13 @@ def customize_for_version(self, is_pypy, version):
kwargs = node[2]
if kwargs == 'expr':
kwargs = kwargs[0]
self.write('**')
self.preorder(kwargs)
call_function_ex = node[-1]
assert call_function_ex == 'CALL_FUNCTION_EX_KW'
if call_function_ex.attr & 1 and not isinstance(kwargs, Token):
self.call36_dict(kwargs)
else:
self.write('**')
self.preorder(kwargs)
self.write(')')
self.prune()
self.n_call_ex_kw4 = call_ex_kw4
@@ -559,8 +569,16 @@ def customize_for_version(self, is_pypy, version):
kv_node = node[0]
l = list(kv_node)
i = 0
length = len(l)
# FIXME: Parser-speed improved grammars will have BUILD_MAP
# at the end. So in the future when everything is
# complete, we can do an "assert" instead of "if".
if kv_node[-1].kind.startswith("BUILD_MAP"):
length -= 1
# Respect line breaks from source
while i < len(l):
while i < length:
self.write(sep)
name = self.traverse(l[i], indent='')
# Strip off beginning and trailing quotes in name
@@ -605,6 +623,22 @@ def customize_for_version(self, is_pypy, version):
FSTRING_CONVERSION_MAP = {1: '!s', 2: '!r', 3: '!a'}
def n_except_suite_finalize(node):
if node[1] == 'returns' and self.hide_internal:
# Process node[1] only.
# The code after "returns", e.g. node[3], is dead code.
# Adding it is wrong as it dedents and another
# exception handler "except_stmt" afterwards.
# Note it is also possible that the grammar is wrong here.
# and this should not be "except_stmt".
self.indent_more()
self.preorder(node[1])
self.indent_less()
else:
self.default(node)
self.prune()
self.n_except_suite_finalize = n_except_suite_finalize
def n_formatted_value(node):
if node[0] == 'LOAD_CONST':
self.write(node[0].attr)

20
uncompyle6/semantics/make_function.py
View File

@@ -428,10 +428,7 @@ def make_function2(self, node, is_lambda, nested=1, codeNode=None):
code, self.version)
# Python 2 doesn't support the "nonlocal" statement
try:
assert self.version >= 3.0 or not nonlocals
except:
from trepan.api import debug; debug()
assert self.version >= 3.0 or not nonlocals
for g in sorted((all_globals & self.mod_globs) | globals):
self.println(self.indent, 'global ', g)
@@ -512,15 +509,22 @@ def make_function3(self, node, is_lambda, nested=1, codeNode=None):
if isinstance(args_node.attr, tuple):
pos_args, kw_args, annotate_argc = args_node.attr
# FIXME: there is probably a better way to classify this.
have_kwargs = node[0].kind.startswith('kwarg') or node[0] == 'no_kwargs'
if len(node) >= 4:
lc_index = -4
else:
lc_index = -3
pass
if (self.version <= 3.3 and len(node) > 2 and
node[lambda_index] != 'LOAD_LAMBDA' and
(node[0].kind.startswith('kwarg') or node[-4].kind != 'load_closure')):
node[lambda_index] != 'LOAD_LAMBDA' and
(have_kwargs or node[lc_index].kind != 'load_closure')):
# args are after kwargs; kwargs are bundled as one node
defparams = node[1:args_node.attr[0]+1]
else:
# args are before kwargs; kwags as bundled as one node
defparams = node[:args_node.attr[0]]
pos_args, kw_args, annotate_argc = args_node.attr
pass
else:
if self.version < 3.6:
defparams = node[:args_node.attr]
@@ -695,7 +699,7 @@ def make_function3(self, node, is_lambda, nested=1, codeNode=None):
for n in node:
if n == 'pos_arg':
continue
elif self.version >= 3.4 and not (n.kind in ('kwargs', 'kwargs1', 'kwarg')):
elif self.version >= 3.4 and not (n.kind in ('kwargs', 'no_kwargs', 'kwarg')):
continue
else:
self.preorder(n)

115
uncompyle6/semantics/pysource.py
View File

@@ -404,7 +404,7 @@ class SourceWalker(GenericASTTraversal, object):
def n_mkfunc_annotate(node):
if self.version >= 3.3 or node[-2] == 'kwargs':
if self.version >= 3.3 or node[-2] in ('kwargs', 'no_kwargs'):
# LOAD_CONST code object ..
# LOAD_CONST 'x0' if >= 3.3
# EXTENDED_ARG
@@ -468,73 +468,6 @@ class SourceWalker(GenericASTTraversal, object):
})
def async_call(node):
self.f.write('async ')
node.kind == 'call'
p = self.prec
self.prec = 80
self.template_engine(('%c(%P)', 0,
(1, -4, ', ', 100)), node)
self.prec = p
node.kind == 'async_call'
self.prune()
self.n_async_call = async_call
self.n_build_list_unpack = self.n_list
if version == 3.5:
def n_call(node):
mapping = self._get_mapping(node)
table = mapping[0]
key = node
for i in mapping[1:]:
key = key[i]
pass
if key.kind.startswith('CALL_FUNCTION_VAR_KW'):
# Python 3.5 changes the stack position of *args. kwargs come
# after *args whereas in earlier Pythons, *args is at the end
# which simpilfiies things from our perspective.
# Python 3.6+ replaces CALL_FUNCTION_VAR_KW with CALL_FUNCTION_EX
# We will just swap the order to make it look like earlier Python 3.
entry = table[key.kind]
kwarg_pos = entry[2][1]
args_pos = kwarg_pos - 1
# Put last node[args_pos] after subsequent kwargs
while node[kwarg_pos] == 'kwarg' and kwarg_pos < len(node):
# swap node[args_pos] with node[kwargs_pos]
node[kwarg_pos], node[args_pos] = node[args_pos], node[kwarg_pos]
args_pos = kwarg_pos
kwarg_pos += 1
self.default(node)
self.n_call = n_call
def n_function_def(node):
if self.version == 3.6:
code_node = node[0][0]
else:
code_node = node[0][1]
is_code = hasattr(code_node, 'attr') and iscode(code_node.attr)
if (is_code and
(code_node.attr.co_flags & COMPILER_FLAG_BIT['COROUTINE'])):
self.template_engine(('\n\n%|async def %c\n',
-2), node)
else:
self.template_engine(('\n\n%|def %c\n', -2),
node)
self.prune()
self.n_function_def = n_function_def
def unmapexpr(node):
last_n = node[0][-1]
for n in node[0]:
self.preorder(n)
if n != last_n:
self.f.write(', **')
pass
pass
self.prune()
pass
self.n_unmapexpr = unmapexpr
pass # version >= 3.4
pass # version >= 3.0
@@ -1041,7 +974,7 @@ class SourceWalker(GenericASTTraversal, object):
def n_mkfunc(self, node):
if self.version >= 3.3 or node[-2] == 'kwargs':
if self.version >= 3.3 or node[-2] in ('kwargs', 'no_kwargs'):
# LOAD_CONST code object ..
# LOAD_CONST 'x0' if >= 3.3
# MAKE_FUNCTION ..
@@ -1057,7 +990,7 @@ class SourceWalker(GenericASTTraversal, object):
self.write(func_name)
self.indent_more()
self.make_function(node, is_lambda=False, codeNode=code_node)
self.make_function(node, is_lambda=False, code_node=code_node)
if len(self.param_stack) > 1:
self.write('\n\n')
@@ -1067,14 +1000,14 @@ class SourceWalker(GenericASTTraversal, object):
self.prune() # stop recursing
def make_function(self, node, is_lambda, nested=1,
codeNode=None, annotate=None):
code_node=None, annotate=None):
if self.version >= 3.0:
make_function3(self, node, is_lambda, nested, codeNode)
make_function3(self, node, is_lambda, nested, code_node)
else:
make_function2(self, node, is_lambda, nested, codeNode)
make_function2(self, node, is_lambda, nested, code_node)
def n_mklambda(self, node):
self.make_function(node, is_lambda=True, codeNode=node[-2])
self.make_function(node, is_lambda=True, code_node=node[-2])
self.prune() # stop recursing
def n_list_comp(self, node):
@@ -1563,7 +1496,7 @@ class SourceWalker(GenericASTTraversal, object):
assert 'mkfunc' == build_class[1]
mkfunc = build_class[1]
if mkfunc[0] == 'kwargs':
if mkfunc[0] in ('kwargs', 'no_kwargs'):
if 3.0 <= self.version <= 3.2:
for n in mkfunc:
if hasattr(n, 'attr') and iscode(n.attr):
@@ -1608,8 +1541,15 @@ class SourceWalker(GenericASTTraversal, object):
subclass_info = node
subclass_code = build_class[1][0].attr
elif not subclass_info:
subclass_code = build_class[1][0].attr
subclass_info = node[0]
if mkfunc[0] in ('no_kwargs', 'kwargs'):
subclass_code = mkfunc[1].attr
else:
subclass_code = mkfunc[0].attr
if node == 'classdefdeco2':
subclass_info = node
else:
subclass_info = node[0]
else:
if node == 'classdefdeco2':
build_class = node
@@ -1680,7 +1620,7 @@ class SourceWalker(GenericASTTraversal, object):
self.write(')')
def print_super_classes3(self, node):
n = len(node)-1
n = len(node) - 1
if node.kind != 'expr':
assert node[n].kind.startswith('CALL_FUNCTION')
@@ -1764,9 +1704,17 @@ class SourceWalker(GenericASTTraversal, object):
# Python 3.5+ style key/value list in dict
kv_node = node[0]
l = list(kv_node)
length = len(l)
# FIXME: Parser-speed improved grammars will have BUILD_MAP
# at the end. So in the future when everything is
# complete, we can do an "assert" instead of "if".
if kv_node[-1].kind.startswith("BUILD_MAP"):
length -= 1
i = 0
# Respect line breaks from source
while i < len(l):
while i < length:
self.write(sep)
name = self.traverse(l[i], indent='')
if i > 0:
@@ -2196,6 +2144,15 @@ class SourceWalker(GenericASTTraversal, object):
entry = ('%c(*%C, %c)', 0, p2, -2)
elif str == '%c(%C':
entry = ('%c(*%C)', 0, (1, 100, ''))
elif self.version == 3.4:
# CALL_FUNCTION_VAR's top element of the stack contains
# the variable argument list
if v == 0:
str = '%c(*%c)'
entry = (str, 0, -2)
else:
str = '%c(*%c, %C)'
entry = (str, 0, -2, p2)
else:
str += '*%c)'
entry = (str, 0, p2, -2)