Python 3.4: MAKE_FUNCTION starts to work.

This commit is contained in:
rocky
2015-12-19 11:15:48 -05:00
parent c0fcb5fc52
commit 93ab1f0281
12 changed files with 75 additions and 33 deletions

BIN
test/bytecode_2.7/def0.pyc Normal file

Binary file not shown.

BIN
test/bytecode_3.4/def0.pyc Normal file

Binary file not shown.

BIN
test/bytecode_3.4/def1.pyc Normal file

Binary file not shown.

View File

@@ -0,0 +1,2 @@
def x0():
pass

View File

@@ -0,0 +1,2 @@
def x1(a):
pass

View File

@@ -191,7 +191,6 @@ def uncompyle_file(filename, outstream=None, showasm=False, showast=False):
if sys.platform.startswith('linux') and os.uname()[2][:2] in ['2.', '3.', '4.']: if sys.platform.startswith('linux') and os.uname()[2][:2] in ['2.', '3.', '4.']:
def __memUsage(): def __memUsage():
mi = open('/proc/self/stat', 'r') mi = open('/proc/self/stat', 'r')
from trepan.api import debug; debug()
mu = mi.readline().split()[22] mu = mi.readline().split()[22]
mi.close() mi.close()
return int(mu) / 1000000 return int(mu) / 1000000

View File

@@ -443,10 +443,20 @@ class Traverser(walker.Walker, object):
def n_mkfunc(self, node): def n_mkfunc(self, node):
start = len(self.f.getvalue()) start = len(self.f.getvalue())
old_name = self.name old_name = self.name
self.name = node[-2].attr.co_name # code.co_name if PYTHON3:
# LOAD_CONST code object ..
# LOAD_CONST 'x0'
# MAKE_FUNCTION ..
self.name = node[-2].attr
code_index = -3
else:
# LOAD_CONST code object ..
# MAKE_FUNCTION ..
self.name = node[-2].attr.co_name
code_index = -2
self.write(self.name) self.write(self.name)
self.indentMore() self.indentMore()
self.make_function(node, isLambda=0) self.make_function(node, isLambda=False, code_index=code_index)
self.name = old_name self.name = old_name
self.set_pos_info(node, start, len(self.f.getvalue())) self.set_pos_info(node, start, len(self.f.getvalue()))
if len(self.__param_stack) > 1: if len(self.__param_stack) > 1:
@@ -1018,7 +1028,7 @@ class Traverser(walker.Walker, object):
self.set_pos_info(last_node, startnode_start, self.last_finish) self.set_pos_info(last_node, startnode_start, self.last_finish)
return return
def make_function(self, node, isLambda, nested=1): def make_function(self, node, isLambda, nested=1, code_index=-2):
"""Dump function defintion, doc string, and function body.""" """Dump function defintion, doc string, and function body."""
def build_param(ast, name, default): def build_param(ast, name, default):
@@ -1050,8 +1060,9 @@ class Traverser(walker.Walker, object):
else: else:
return name return name
defparams = node[:node[-1].attr] # node[-1] == MAKE_xxx_n # node[-1] == MAKE_xxx_n
code = node[-2].attr defparams = node[:node[-1].attr]
code = node[code_index].attr
assert type(code) == CodeType assert type(code) == CodeType
code = Code(code, self.scanner, self.currentclass) code = Code(code, self.scanner, self.currentclass)

View File

@@ -656,6 +656,7 @@ class Python2Parser(PythonParser):
unpack ::= UNPACK_TUPLE {expr}^n unpack ::= UNPACK_TUPLE {expr}^n
unpack ::= UNPACK_SEQEUENE {expr}^n unpack ::= UNPACK_SEQEUENE {expr}^n
mkfunc ::= {expr}^n LOAD_CONST MAKE_FUNCTION_n mkfunc ::= {expr}^n LOAD_CONST MAKE_FUNCTION_n
mklambda ::= {expr}^n LOAD_LAMBDA MAKE_FUNCTION_n
mkfunc ::= {expr}^n load_closure LOAD_CONST MAKE_FUNCTION_n mkfunc ::= {expr}^n load_closure LOAD_CONST MAKE_FUNCTION_n
expr ::= expr {expr}^n CALL_FUNCTION_n expr ::= expr {expr}^n CALL_FUNCTION_n
expr ::= expr {expr}^n CALL_FUNCTION_VAR_n POP_TOP expr ::= expr {expr}^n CALL_FUNCTION_VAR_n POP_TOP

View File

@@ -27,6 +27,17 @@ class Python3Parser(PythonParser):
self.added_rules = set() self.added_rules = set()
GenericASTBuilder.__init__(self, AST, 'stmts') GenericASTBuilder.__init__(self, AST, 'stmts')
self.customized = {} self.customized = {}
self.new_rules = set()
def add_unique_rule(self, rule, opname, count, customize):
"""Add rule to grammar, but only if it hasn't been added previously
"""
if rule not in self.new_rules:
self.new_rules.add(rule)
self.addRule(rule, nop_func)
customize[opname] = count
pass
return
def p_funcdef(self, args): def p_funcdef(self, args):
''' '''
@@ -651,37 +662,40 @@ class Python3Parser(PythonParser):
Special handling for opcodes that take a variable number Special handling for opcodes that take a variable number
of arguments -- we add a new rule for each: of arguments -- we add a new rule for each:
expr ::= {expr}^n BUILD_LIST_n expr ::= {expr}^n BUILD_LIST_n
expr ::= {expr}^n BUILD_TUPLE_n expr ::= {expr}^n BUILD_TUPLE_n
unpack_list ::= UNPACK_LIST {expr}^n unpack_list ::= UNPACK_LIST {expr}^n
unpack ::= UNPACK_TUPLE {expr}^n unpack ::= UNPACK_TUPLE {expr}^n
unpack ::= UNPACK_SEQEUENE {expr}^n unpack ::= UNPACK_SEQEUENE {expr}^n
mkfunc ::= {expr}^n LOAD_CONST MAKE_FUNCTION_n mkfunc ::= {expr}^n LOAD_CONST MAKE_FUNCTION_n
mkfunc ::= {expr}^n load_closure LOAD_CONST MAKE_FUNCTION_n mklambda ::= {expr}^n LOAD_LAMBDA MAKE_FUNCTION_n
mkfunc ::= {expr}^n load_closure LOAD_CONST MAKE_FUNCTION_n
expr ::= expr {expr}^n CALL_FUNCTION_n expr ::= expr {expr}^n CALL_FUNCTION_n
expr ::= expr {expr}^n CALL_FUNCTION_VAR_n POP_TOP expr ::= expr {expr}^n CALL_FUNCTION_VAR_n POP_TOP
expr ::= expr {expr}^n CALL_FUNCTION_VAR_KW_n POP_TOP expr ::= expr {expr}^n CALL_FUNCTION_VAR_KW_n POP_TOP
expr ::= expr {expr}^n CALL_FUNCTION_KW_n POP_TOP expr ::= expr {expr}^n CALL_FUNCTION_KW_n POP_TOP
""" """
new_rules = set()
for token in tokens: for token in tokens:
if token.type in ('CALL_FUNCTION', 'CALL_FUNCTION_VAR', opname = token.type
'CALL_FUNCTION_VAR_KW', 'CALL_FUNCTION_KW'): if opname in ('CALL_FUNCTION', 'CALL_FUNCTION_VAR',
'CALL_FUNCTION_VAR_KW', 'CALL_FUNCTION_KW'):
# Low byte indicates number of positional paramters, # Low byte indicates number of positional paramters,
# high byte number of positional parameters # high byte number of positional parameters
args_pos = token.attr & 0xff args_pos = token.attr & 0xff
args_kw = (token.attr >> 8) & 0xff args_kw = (token.attr >> 8) & 0xff
nak = ( len(token.type)-len('CALL_FUNCTION') ) // 3 nak = ( len(opname)-len('CALL_FUNCTION') ) // 3
token.type = 'CALL_FUNCTION_%i' % token.attr token.type = 'CALL_FUNCTION_%i' % token.attr
rule = ('call_function ::= expr ' rule = ('call_function ::= expr '
+ ('expr ' * args_pos) + ('expr ' * args_pos)
+ ('kwarg ' * args_kw) + ('kwarg ' * args_kw)
+ 'expr ' * nak + token.type) + 'expr ' * nak + token.type)
# Make sure we do not add the same rule twice self.add_unique_rule(rule, token.type, args_pos, customize)
if rule not in new_rules: elif opname.startswith('MAKE_FUNCTION_'):
new_rules.add(rule) # from trepan.api import debug
self.addRule(rule, nop_func) # debug(start_opts={'startup-profile': True})
customize[token.type] = args_pos self.addRule('mklambda ::= %s LOAD_LAMBDA %s' %
pass ('expr ' * token.attr, opname), nop_func)
rule = 'mkfunc ::= %s LOAD_CONST LOAD_CONST %s' % ('expr ' * token.attr, opname)
self.add_unique_rule(rule, opname, token.attr, customize)
pass pass
return return

View File

@@ -41,10 +41,10 @@ class Scanner34(scan.Scanner):
arg = bytecode[pos+1] + bytecode[pos+2] * 256 arg = bytecode[pos+1] + bytecode[pos+2] * 256
return arg return arg
def disassemble(self, co): def disassemble(self, co, classname=None):
fn = self.disassemble_built_in if PYTHON_VERSION == 3.4 \ fn = self.disassemble_built_in if PYTHON_VERSION == 3.4 \
else self.disassemble_cross_version else self.disassemble_cross_version
return fn(co) return fn(co, classname)
def disassemble_built_in(self, co, classname=None): def disassemble_built_in(self, co, classname=None):
# Container for tokens # Container for tokens

View File

@@ -8,7 +8,7 @@ byte-code verification
from __future__ import print_function from __future__ import print_function
import dis, inspect, operator, types import dis, inspect, operator
import uncompyle6 import uncompyle6
import uncompyle6.scanner as scanner import uncompyle6.scanner as scanner
@@ -303,8 +303,8 @@ def cmp_code_objects(version, code_obj1, code_obj2, name=''):
elif member == 'co_consts': elif member == 'co_consts':
# partial optimization can make the co_consts look different, # partial optimization can make the co_consts look different,
# so we'll just compare the code consts # so we'll just compare the code consts
codes1 = ( c for c in code_obj1.co_consts if isinstance(c, types.CodeType) ) codes1 = ( c for c in code_obj1.co_consts if inspect.iscode(c) )
codes2 = ( c for c in code_obj2.co_consts if isinstance(c, types.CodeType) ) codes2 = ( c for c in code_obj2.co_consts if inspect.iscode(c) )
for c1, c2 in zip(codes1, codes2): for c1, c2 in zip(codes1, codes2):
cmp_code_objects(version, c1, c2, name=name) cmp_code_objects(version, c1, c2, name=name)

View File

@@ -877,9 +877,21 @@ class Walker(GenericASTTraversal, object):
n_importstar = n_importfrom n_importstar = n_importfrom
def n_mkfunc(self, node): def n_mkfunc(self, node):
self.write(node[-2].attr.co_name) # = code.co_name if PYTHON3:
# LOAD_CONST code object ..
# LOAD_CONST 'x0'
# MAKE_FUNCTION ..
func_name = node[-2].attr
code_index = -3
else:
# LOAD_CONST code object ..
# MAKE_FUNCTION ..
func_name = node[-2].attr.co_name
code_index = -2
self.write(func_name)
self.indentMore() self.indentMore()
self.make_function(node, isLambda=0) self.make_function(node, isLambda=False, code_index=code_index)
if len(self.param_stack) > 1: if len(self.param_stack) > 1:
self.write('\n\n') self.write('\n\n')
else: else:
@@ -888,7 +900,7 @@ class Walker(GenericASTTraversal, object):
self.prune() # stop recursing self.prune() # stop recursing
def n_mklambda(self, node): def n_mklambda(self, node):
self.make_function(node, isLambda=1) self.make_function(node, isLambda=True)
self.prune() # stop recursing self.prune() # stop recursing
def n_list_compr(self, node): def n_list_compr(self, node):
@@ -1251,7 +1263,7 @@ class Walker(GenericASTTraversal, object):
# return self.traverse(node[1]) # return self.traverse(node[1])
raise Exception("Can't find tuple parameter " + name) raise Exception("Can't find tuple parameter " + name)
def make_function(self, node, isLambda, nested=1): def make_function(self, node, isLambda, nested=1, code_index=-2):
"""Dump function defintion, doc string, and function body.""" """Dump function defintion, doc string, and function body."""
def build_param(ast, name, default): def build_param(ast, name, default):
@@ -1276,8 +1288,9 @@ class Walker(GenericASTTraversal, object):
return result return result
else: else:
return name return name
defparams = node[:node[-1].attr] # node[-1] == MAKE_xxx_n # node[-1] == MAKE_xxx_n
code = node[-2].attr defparams = node[:node[-1].attr]
code = node[code_index].attr
assert inspect.iscode(code) assert inspect.iscode(code)
code = Code(code, self.scanner, self.currentclass) code = Code(code, self.scanner, self.currentclass)