jinx/python-uncompyle6
1
Fork 0
mirror of https://github.com/rocky/python-uncompyle6.git synced 2025-08-04 09:22:40 +08:00
Code Issues Packages Projects Releases Wiki Activity

Fix parser slowness in decompiling 3.x locale.py..

Browse Source 
And remove grammar inefficiency in adding extraneous kwargs in <= 3.2
kwargs was nullable so it might not have been wasn't wrong, just inefficient.
This commit is contained in:
rocky
2018-03-23 11:59:04 -04:00
parent 1b2b45642b
commit 35a60e0274
4 changed files with 68 additions and 35 deletions
Download Patch File Download Diff File Expand all files Collapse all files

43
uncompyle6/parsers/parse3.py
View File

@@ -112,9 +112,10 @@ class Python3Parser(PythonParser):
continues ::= continue continues ::= continue
kwarg ::= LOAD_CONST expr kwarg ::= LOAD_CONST expr
kwargs ::= kwarg* kwargs ::= kwarg+
kwargs1 ::= kwarg+ no_kwargs ::=
classdef ::= build_class store classdef ::= build_class store
@@ -527,7 +528,7 @@ class Python3Parser(PythonParser):
subclassing is, well, is pretty base. And we want it that way: lean and subclassing is, well, is pretty base. And we want it that way: lean and
mean so that parsing will go faster. mean so that parsing will go faster.
Here, we add additional grammra rules based on specific instructions Here, we add additional grammar rules based on specific instructions
that are in the instruction/token stream. In classes that that are in the instruction/token stream. In classes that
inherit from from here and other versions, grammar rules may inherit from from here and other versions, grammar rules may
also be removed. also be removed.
@@ -656,9 +657,9 @@ class Python3Parser(PythonParser):
('dict ' * token.attr) + ('dict ' * token.attr) +
'BUILD_MAP_UNPACK') 'BUILD_MAP_UNPACK')
else: else:
rule = kvlist_n + ' ::= ' + 'expr ' * (token.attr*2) rule = "%s ::= %s %s" % (kvlist_n, 'expr ' * (token.attr*2), opname)
self.add_unique_rule(rule, opname, token.attr, customize) self.add_unique_rule(rule, opname, token.attr, customize)
rule = "dict ::= %s %s" % (kvlist_n, opname) rule = "dict ::= %s" % kvlist_n
else: else:
rule = kvlist_n + ' ::= ' + 'expr expr STORE_MAP ' * token.attr rule = kvlist_n + ' ::= ' + 'expr expr STORE_MAP ' * token.attr
self.add_unique_rule(rule, opname, token.attr, customize) self.add_unique_rule(rule, opname, token.attr, customize)
@@ -866,13 +867,13 @@ class Python3Parser(PythonParser):
opname, token.attr, customize) opname, token.attr, customize)
if args_kw > 0: if args_kw > 0:
kwargs_str = 'kwargs1 ' kwargs_str = 'kwargs '
else: else:
kwargs_str = '' kwargs_str = ''
# Note order of kwargs and pos args changed between 3.3-3.4 # Note order of kwargs and pos args changed between 3.3-3.4
if self.version <= 3.2: if self.version <= 3.2:
rule = ('mkfunc ::= %s%sload_closure LOAD_CONST kwargs %s' rule = ('mkfunc ::= %s%sload_closure LOAD_CONST %s'
% (kwargs_str, 'expr ' * args_pos, opname)) % (kwargs_str, 'expr ' * args_pos, opname))
elif self.version == 3.3: elif self.version == 3.3:
rule = ('mkfunc ::= %s%sload_closure LOAD_CONST LOAD_CONST %s' rule = ('mkfunc ::= %s%sload_closure LOAD_CONST LOAD_CONST %s'
@@ -980,29 +981,37 @@ class Python3Parser(PythonParser):
opname)) opname))
self.add_make_function_rule(rule_pat, opname, token.attr, customize) self.add_make_function_rule(rule_pat, opname, token.attr, customize)
if args_kw == 0:
kwargs = 'no_kwargs'
else:
kwargs = 'kwargs'
if self.version < 3.3: if self.version < 3.3:
# positional args after keyword args # positional args after keyword args
rule = ('mkfunc ::= kwargs %s%s %s' % rule = ('mkfunc ::= %s %s%s%s' %
(kwargs, 'pos_arg ' * args_pos, 'LOAD_CONST ',
opname))
self.add_unique_rule(rule, opname, token.attr, customize)
rule = ('mkfunc ::= %s%s%s' %
('pos_arg ' * args_pos, 'LOAD_CONST ', ('pos_arg ' * args_pos, 'LOAD_CONST ',
opname)) opname))
elif self.version == 3.3: elif self.version == 3.3:
# positional args after keyword args # positional args after keyword args
rule = ('mkfunc ::= kwargs %s%s %s' % rule = ('mkfunc ::= %s %s%s%s' %
('pos_arg ' * args_pos, 'LOAD_CONST '*2, (kwargs, 'pos_arg ' * args_pos, 'LOAD_CONST '*2,
opname)) opname))
elif self.version > 3.5: elif self.version > 3.5:
# positional args before keyword args # positional args before keyword args
rule = ('mkfunc ::= %skwargs1 %s %s' % rule = ('mkfunc ::= %s%s %s%s' %
('pos_arg ' * args_pos, 'LOAD_CONST '*2, ('pos_arg ' * args_pos, kwargs, 'LOAD_CONST '*2,
opname)) opname))
elif self.version > 3.3: elif self.version > 3.3:
# positional args before keyword args # positional args before keyword args
rule = ('mkfunc ::= %skwargs %s %s' % rule = ('mkfunc ::= %s%s %s%s' %
('pos_arg ' * args_pos, 'LOAD_CONST '*2, ('pos_arg ' * args_pos, kwargs, 'LOAD_CONST '*2,
opname)) opname))
else: else:
rule = ('mkfunc ::= kwargs %sexpr %s' % rule = ('mkfunc ::= %s%sexpr %s' %
('pos_arg ' * args_pos, opname)) (kwargs, 'pos_arg ' * args_pos, opname))
self.add_unique_rule(rule, opname, token.attr, customize) self.add_unique_rule(rule, opname, token.attr, customize)
if opname.startswith('MAKE_FUNCTION_A'): if opname.startswith('MAKE_FUNCTION_A'):
if self.version >= 3.6: if self.version >= 3.6:

10
uncompyle6/semantics/customize.py
View File

@@ -563,8 +563,16 @@ def customize_for_version(self, is_pypy, version):
kv_node = node[0] kv_node = node[0]
l = list(kv_node) l = list(kv_node)
i = 0 i = 0
length = len(l)
# FIXME: Parser-speed improved grammars will have BUILD_MAP
# at the end. So in the future when everything is
# complete, we can do an "assert" instead of "if".
if kv_node[-1].kind.startswith("BUILD_MAP"):
length -= 1
# Respect line breaks from source # Respect line breaks from source
while i < len(l): while i < length:
self.write(sep) self.write(sep)
name = self.traverse(l[i], indent='') name = self.traverse(l[i], indent='')
# Strip off beginning and trailing quotes in name # Strip off beginning and trailing quotes in name

19
uncompyle6/semantics/make_function.py
View File

@@ -431,10 +431,7 @@ def make_function2(self, node, is_lambda, nested=1, codeNode=None):
code, self.version) code, self.version)
# Python 2 doesn't support the "nonlocal" statement # Python 2 doesn't support the "nonlocal" statement
try: assert self.version >= 3.0 or not nonlocals
assert self.version >= 3.0 or not nonlocals
except:
from trepan.api import debug; debug()
for g in sorted((all_globals & self.mod_globs) | globals): for g in sorted((all_globals & self.mod_globs) | globals):
self.println(self.indent, 'global ', g) self.println(self.indent, 'global ', g)
@@ -516,14 +513,22 @@ def make_function3(self, node, is_lambda, nested=1, codeNode=None):
if isinstance(args_node.attr, tuple): if isinstance(args_node.attr, tuple):
pos_args, kw_args, annotate_argc = args_node.attr pos_args, kw_args, annotate_argc = args_node.attr
# FIXME: there is probably a better way to classify this. # FIXME: there is probably a better way to classify this.
have_kwargs = node[0].kind.startswith('kwarg') or node[0] == 'no_kwargs'
if len(node) >= 4:
lc_index = -4
else:
lc_index = -3
pass
if (self.version <= 3.3 and len(node) > 2 and if (self.version <= 3.3 and len(node) > 2 and
node[lambda_index] != 'LOAD_LAMBDA' and node[lambda_index] != 'LOAD_LAMBDA' and
(node[0].kind.startswith('kwarg') or node[-4].kind != 'load_closure')): (have_kwargs or node[lc_index].kind != 'load_closure')):
# args are after kwargs; kwargs are bundled as one node # args are after kwargs; kwargs are bundled as one node
defparams = node[1:args_node.attr[0]+1] defparams = node[1:args_node.attr[0]+1]
else: else:
# args are before kwargs; kwags as bundled as one node # args are before kwargs; kwags as bundled as one node
defparams = node[:args_node.attr[0]] defparams = node[:args_node.attr[0]]
pass
else: else:
if self.version < 3.6: if self.version < 3.6:
defparams = node[:args_node.attr] defparams = node[:args_node.attr]
@@ -653,7 +658,7 @@ def make_function3(self, node, is_lambda, nested=1, codeNode=None):
for n in node: for n in node:
if n == 'pos_arg': if n == 'pos_arg':
continue continue
elif self.version >= 3.4 and not (n.kind in ('kwargs', 'kwargs1', 'kwarg')): elif self.version >= 3.4 and not (n.kind in ('kwargs', 'no_kwargs', 'kwarg')):
continue continue
else: else:
self.preorder(n) self.preorder(n)

31
uncompyle6/semantics/pysource.py
View File

@@ -404,7 +404,7 @@ class SourceWalker(GenericASTTraversal, object):
def n_mkfunc_annotate(node): def n_mkfunc_annotate(node):
if self.version >= 3.3 or node[-2] == 'kwargs': if self.version >= 3.3 or node[-2] in ('kwargs', 'no_kwargs'):
# LOAD_CONST code object .. # LOAD_CONST code object ..
# LOAD_CONST 'x0' if >= 3.3 # LOAD_CONST 'x0' if >= 3.3
# EXTENDED_ARG # EXTENDED_ARG
@@ -1041,7 +1041,7 @@ class SourceWalker(GenericASTTraversal, object):
def n_mkfunc(self, node): def n_mkfunc(self, node):
if self.version >= 3.3 or node[-2] == 'kwargs': if self.version >= 3.3 or node[-2] in ('kwargs', 'no_kwargs'):
# LOAD_CONST code object .. # LOAD_CONST code object ..
# LOAD_CONST 'x0' if >= 3.3 # LOAD_CONST 'x0' if >= 3.3
# MAKE_FUNCTION .. # MAKE_FUNCTION ..
@@ -1057,7 +1057,7 @@ class SourceWalker(GenericASTTraversal, object):
self.write(func_name) self.write(func_name)
self.indent_more() self.indent_more()
self.make_function(node, is_lambda=False, codeNode=code_node) self.make_function(node, is_lambda=False, code_node=code_node)
if len(self.param_stack) > 1: if len(self.param_stack) > 1:
self.write('\n\n') self.write('\n\n')
@@ -1067,14 +1067,14 @@ class SourceWalker(GenericASTTraversal, object):
self.prune() # stop recursing self.prune() # stop recursing
def make_function(self, node, is_lambda, nested=1, def make_function(self, node, is_lambda, nested=1,
codeNode=None, annotate=None): code_node=None, annotate=None):
if self.version >= 3.0: if self.version >= 3.0:
make_function3(self, node, is_lambda, nested, codeNode) make_function3(self, node, is_lambda, nested, code_node)
else: else:
make_function2(self, node, is_lambda, nested, codeNode) make_function2(self, node, is_lambda, nested, code_node)
def n_mklambda(self, node): def n_mklambda(self, node):
self.make_function(node, is_lambda=True, codeNode=node[-2]) self.make_function(node, is_lambda=True, code_node=node[-2])
self.prune() # stop recursing self.prune() # stop recursing
def n_list_comp(self, node): def n_list_comp(self, node):
@@ -1563,7 +1563,7 @@ class SourceWalker(GenericASTTraversal, object):
assert 'mkfunc' == build_class[1] assert 'mkfunc' == build_class[1]
mkfunc = build_class[1] mkfunc = build_class[1]
if mkfunc[0] == 'kwargs': if mkfunc[0] in ('kwargs', 'no_kwargs'):
if 3.0 <= self.version <= 3.2: if 3.0 <= self.version <= 3.2:
for n in mkfunc: for n in mkfunc:
if hasattr(n, 'attr') and iscode(n.attr): if hasattr(n, 'attr') and iscode(n.attr):
@@ -1608,7 +1608,10 @@ class SourceWalker(GenericASTTraversal, object):
subclass_info = node subclass_info = node
subclass_code = build_class[1][0].attr subclass_code = build_class[1][0].attr
elif not subclass_info: elif not subclass_info:
subclass_code = build_class[1][0].attr if mkfunc[0] in ('no_kwargs', 'kwargs'):
subclass_code = mkfunc[1].attr
else:
subclass_code = mkfunc[0].attr
subclass_info = node[0] subclass_info = node[0]
else: else:
if node == 'classdefdeco2': if node == 'classdefdeco2':
@@ -1764,9 +1767,17 @@ class SourceWalker(GenericASTTraversal, object):
# Python 3.5+ style key/value list in dict # Python 3.5+ style key/value list in dict
kv_node = node[0] kv_node = node[0]
l = list(kv_node) l = list(kv_node)
length = len(l)
# FIXME: Parser-speed improved grammars will have BUILD_MAP
# at the end. So in the future when everything is
# complete, we can do an "assert" instead of "if".
if kv_node[-1].kind.startswith("BUILD_MAP"):
length -= 1
i = 0 i = 0
# Respect line breaks from source # Respect line breaks from source
while i < len(l): while i < length:
self.write(sep) self.write(sep)
name = self.traverse(l[i], indent='') name = self.traverse(l[i], indent='')
if i > 0: if i > 0: