From 93ab1f02813a01874cd812461fc09da226ce6144 Mon Sep 17 00:00:00 2001 From: rocky Date: Sat, 19 Dec 2015 11:15:48 -0500 Subject: [PATCH] Python 3.4: MAKE_FUNCTION starts to work. --- test/bytecode_2.7/def0.pyc | Bin 0 -> 226 bytes test/bytecode_3.4/def0.pyc | Bin 0 -> 200 bytes test/bytecode_3.4/def1.pyc | Bin 0 -> 200 bytes test/simple-source/def/def0.py | 2 ++ test/simple-source/def/def1.py | 2 ++ uncompyle6/__init__.py | 1 - uncompyle6/deparser.py | 21 ++++++++++---- uncompyle6/parsers/parse2.py | 1 + uncompyle6/parsers/parse3.py | 46 ++++++++++++++++++++----------- uncompyle6/scanners/scanner34.py | 4 +-- uncompyle6/verify.py | 6 ++-- uncompyle6/walker.py | 25 +++++++++++++---- 12 files changed, 75 insertions(+), 33 deletions(-) create mode 100644 test/bytecode_2.7/def0.pyc create mode 100644 test/bytecode_3.4/def0.pyc create mode 100644 test/bytecode_3.4/def1.pyc create mode 100644 test/simple-source/def/def0.py create mode 100644 test/simple-source/def/def1.py diff --git a/test/bytecode_2.7/def0.pyc b/test/bytecode_2.7/def0.pyc new file mode 100644 index 0000000000000000000000000000000000000000..86692f85c18f3d641e627fdec0f6db6f1c7611e1 GIT binary patch literal 226 zcmZSn%*(|QQ5u%a00oRd+5w1*d4WU<149b~Llls~$Plc-1QbD0=L}NE0#O&N0n+KG z0VEjEh+;{gSaD`9UF&P;Bz(xgu*rb`n T0n}lWo1apelWGSx4`c)Y`VJ!D literal 0 HcmV?d00001 diff --git a/test/bytecode_3.4/def0.pyc b/test/bytecode_3.4/def0.pyc new file mode 100644 index 0000000000000000000000000000000000000000..2e9b036ebc152976b354300bf971cb59862aa927 GIT binary patch literal 200 zcmaFI!^_1HQ5q)9z`*brh~a<<$Z`PUVgVqL0z`}qEes4%Kn4>-uqJadP!tXrfh=br zE@lA|K()b|jD9N_ia^2$;+JG`W^O@Fs%~+9X;E^jeoAT@2pi}XRNi8$FaRko23g9$ gSi}q@{WKvo!T<;hq~sQdO>TZlX-=vgSQkhW0Pidwu>b%7 literal 0 HcmV?d00001 diff --git a/test/bytecode_3.4/def1.pyc b/test/bytecode_3.4/def1.pyc new file mode 100644 index 0000000000000000000000000000000000000000..14ae6877c7c72e7c76ac37b66fe15f474f4afaef GIT binary patch literal 200 zcmaFI!^<@-zBEjPfq~&M5W@izkmUfx#R5Pg1&A0KS{N9jfD9&vU`^&^MxZbVfM^hQ z2I68CAOTbxtjXxNl0lR47Gq)&NDfT=k}S^5EyzjLEzU13N>0^JNlgP`L%o8^TTB&( nAa%tcOBoo8K*soKLTH2m5Ee5~?iPnlZhlH>PO2SP7f2HT$W|ah literal 0 HcmV?d00001 diff --git a/test/simple-source/def/def0.py b/test/simple-source/def/def0.py new file mode 100644 index 00000000..be094890 --- /dev/null +++ b/test/simple-source/def/def0.py @@ -0,0 +1,2 @@ +def x0(): + pass diff --git a/test/simple-source/def/def1.py b/test/simple-source/def/def1.py new file mode 100644 index 00000000..b5c84c57 --- /dev/null +++ b/test/simple-source/def/def1.py @@ -0,0 +1,2 @@ +def x1(a): + pass diff --git a/uncompyle6/__init__.py b/uncompyle6/__init__.py index 6f606e72..a4b4a2e7 100644 --- a/uncompyle6/__init__.py +++ b/uncompyle6/__init__.py @@ -191,7 +191,6 @@ def uncompyle_file(filename, outstream=None, showasm=False, showast=False): if sys.platform.startswith('linux') and os.uname()[2][:2] in ['2.', '3.', '4.']: def __memUsage(): mi = open('/proc/self/stat', 'r') - from trepan.api import debug; debug() mu = mi.readline().split()[22] mi.close() return int(mu) / 1000000 diff --git a/uncompyle6/deparser.py b/uncompyle6/deparser.py index 5451334c..6306fe29 100644 --- a/uncompyle6/deparser.py +++ b/uncompyle6/deparser.py @@ -443,10 +443,20 @@ class Traverser(walker.Walker, object): def n_mkfunc(self, node): start = len(self.f.getvalue()) old_name = self.name - self.name = node[-2].attr.co_name # code.co_name + if PYTHON3: + # LOAD_CONST code object .. + # LOAD_CONST 'x0' + # MAKE_FUNCTION .. + self.name = node[-2].attr + code_index = -3 + else: + # LOAD_CONST code object .. + # MAKE_FUNCTION .. + self.name = node[-2].attr.co_name + code_index = -2 self.write(self.name) self.indentMore() - self.make_function(node, isLambda=0) + self.make_function(node, isLambda=False, code_index=code_index) self.name = old_name self.set_pos_info(node, start, len(self.f.getvalue())) if len(self.__param_stack) > 1: @@ -1018,7 +1028,7 @@ class Traverser(walker.Walker, object): self.set_pos_info(last_node, startnode_start, self.last_finish) return - def make_function(self, node, isLambda, nested=1): + def make_function(self, node, isLambda, nested=1, code_index=-2): """Dump function defintion, doc string, and function body.""" def build_param(ast, name, default): @@ -1050,8 +1060,9 @@ class Traverser(walker.Walker, object): else: return name - defparams = node[:node[-1].attr] # node[-1] == MAKE_xxx_n - code = node[-2].attr + # node[-1] == MAKE_xxx_n + defparams = node[:node[-1].attr] + code = node[code_index].attr assert type(code) == CodeType code = Code(code, self.scanner, self.currentclass) diff --git a/uncompyle6/parsers/parse2.py b/uncompyle6/parsers/parse2.py index 0ee5cf8d..52924520 100644 --- a/uncompyle6/parsers/parse2.py +++ b/uncompyle6/parsers/parse2.py @@ -656,6 +656,7 @@ class Python2Parser(PythonParser): unpack ::= UNPACK_TUPLE {expr}^n unpack ::= UNPACK_SEQEUENE {expr}^n mkfunc ::= {expr}^n LOAD_CONST MAKE_FUNCTION_n + mklambda ::= {expr}^n LOAD_LAMBDA MAKE_FUNCTION_n mkfunc ::= {expr}^n load_closure LOAD_CONST MAKE_FUNCTION_n expr ::= expr {expr}^n CALL_FUNCTION_n expr ::= expr {expr}^n CALL_FUNCTION_VAR_n POP_TOP diff --git a/uncompyle6/parsers/parse3.py b/uncompyle6/parsers/parse3.py index b41800fa..f341c77c 100644 --- a/uncompyle6/parsers/parse3.py +++ b/uncompyle6/parsers/parse3.py @@ -27,6 +27,17 @@ class Python3Parser(PythonParser): self.added_rules = set() GenericASTBuilder.__init__(self, AST, 'stmts') self.customized = {} + self.new_rules = set() + + def add_unique_rule(self, rule, opname, count, customize): + """Add rule to grammar, but only if it hasn't been added previously + """ + if rule not in self.new_rules: + self.new_rules.add(rule) + self.addRule(rule, nop_func) + customize[opname] = count + pass + return def p_funcdef(self, args): ''' @@ -651,37 +662,40 @@ class Python3Parser(PythonParser): Special handling for opcodes that take a variable number of arguments -- we add a new rule for each: - expr ::= {expr}^n BUILD_LIST_n - expr ::= {expr}^n BUILD_TUPLE_n + expr ::= {expr}^n BUILD_LIST_n + expr ::= {expr}^n BUILD_TUPLE_n unpack_list ::= UNPACK_LIST {expr}^n - unpack ::= UNPACK_TUPLE {expr}^n - unpack ::= UNPACK_SEQEUENE {expr}^n - mkfunc ::= {expr}^n LOAD_CONST MAKE_FUNCTION_n - mkfunc ::= {expr}^n load_closure LOAD_CONST MAKE_FUNCTION_n + unpack ::= UNPACK_TUPLE {expr}^n + unpack ::= UNPACK_SEQEUENE {expr}^n + mkfunc ::= {expr}^n LOAD_CONST MAKE_FUNCTION_n + mklambda ::= {expr}^n LOAD_LAMBDA MAKE_FUNCTION_n + mkfunc ::= {expr}^n load_closure LOAD_CONST MAKE_FUNCTION_n expr ::= expr {expr}^n CALL_FUNCTION_n expr ::= expr {expr}^n CALL_FUNCTION_VAR_n POP_TOP expr ::= expr {expr}^n CALL_FUNCTION_VAR_KW_n POP_TOP expr ::= expr {expr}^n CALL_FUNCTION_KW_n POP_TOP """ - new_rules = set() for token in tokens: - if token.type in ('CALL_FUNCTION', 'CALL_FUNCTION_VAR', - 'CALL_FUNCTION_VAR_KW', 'CALL_FUNCTION_KW'): + opname = token.type + if opname in ('CALL_FUNCTION', 'CALL_FUNCTION_VAR', + 'CALL_FUNCTION_VAR_KW', 'CALL_FUNCTION_KW'): # Low byte indicates number of positional paramters, # high byte number of positional parameters args_pos = token.attr & 0xff args_kw = (token.attr >> 8) & 0xff - nak = ( len(token.type)-len('CALL_FUNCTION') ) // 3 + nak = ( len(opname)-len('CALL_FUNCTION') ) // 3 token.type = 'CALL_FUNCTION_%i' % token.attr rule = ('call_function ::= expr ' + ('expr ' * args_pos) + ('kwarg ' * args_kw) + 'expr ' * nak + token.type) - # Make sure we do not add the same rule twice - if rule not in new_rules: - new_rules.add(rule) - self.addRule(rule, nop_func) - customize[token.type] = args_pos - pass + self.add_unique_rule(rule, token.type, args_pos, customize) + elif opname.startswith('MAKE_FUNCTION_'): + # from trepan.api import debug + # debug(start_opts={'startup-profile': True}) + self.addRule('mklambda ::= %s LOAD_LAMBDA %s' % + ('expr ' * token.attr, opname), nop_func) + rule = 'mkfunc ::= %s LOAD_CONST LOAD_CONST %s' % ('expr ' * token.attr, opname) + self.add_unique_rule(rule, opname, token.attr, customize) pass return diff --git a/uncompyle6/scanners/scanner34.py b/uncompyle6/scanners/scanner34.py index cdbd0c93..e720bdff 100644 --- a/uncompyle6/scanners/scanner34.py +++ b/uncompyle6/scanners/scanner34.py @@ -41,10 +41,10 @@ class Scanner34(scan.Scanner): arg = bytecode[pos+1] + bytecode[pos+2] * 256 return arg - def disassemble(self, co): + def disassemble(self, co, classname=None): fn = self.disassemble_built_in if PYTHON_VERSION == 3.4 \ else self.disassemble_cross_version - return fn(co) + return fn(co, classname) def disassemble_built_in(self, co, classname=None): # Container for tokens diff --git a/uncompyle6/verify.py b/uncompyle6/verify.py index 79cdf03d..3c804104 100755 --- a/uncompyle6/verify.py +++ b/uncompyle6/verify.py @@ -8,7 +8,7 @@ byte-code verification from __future__ import print_function -import dis, inspect, operator, types +import dis, inspect, operator import uncompyle6 import uncompyle6.scanner as scanner @@ -303,8 +303,8 @@ def cmp_code_objects(version, code_obj1, code_obj2, name=''): elif member == 'co_consts': # partial optimization can make the co_consts look different, # so we'll just compare the code consts - codes1 = ( c for c in code_obj1.co_consts if isinstance(c, types.CodeType) ) - codes2 = ( c for c in code_obj2.co_consts if isinstance(c, types.CodeType) ) + codes1 = ( c for c in code_obj1.co_consts if inspect.iscode(c) ) + codes2 = ( c for c in code_obj2.co_consts if inspect.iscode(c) ) for c1, c2 in zip(codes1, codes2): cmp_code_objects(version, c1, c2, name=name) diff --git a/uncompyle6/walker.py b/uncompyle6/walker.py index 90d39bb4..10fc7ac5 100644 --- a/uncompyle6/walker.py +++ b/uncompyle6/walker.py @@ -877,9 +877,21 @@ class Walker(GenericASTTraversal, object): n_importstar = n_importfrom def n_mkfunc(self, node): - self.write(node[-2].attr.co_name) # = code.co_name + if PYTHON3: + # LOAD_CONST code object .. + # LOAD_CONST 'x0' + # MAKE_FUNCTION .. + func_name = node[-2].attr + code_index = -3 + else: + # LOAD_CONST code object .. + # MAKE_FUNCTION .. + func_name = node[-2].attr.co_name + code_index = -2 + self.write(func_name) + self.indentMore() - self.make_function(node, isLambda=0) + self.make_function(node, isLambda=False, code_index=code_index) if len(self.param_stack) > 1: self.write('\n\n') else: @@ -888,7 +900,7 @@ class Walker(GenericASTTraversal, object): self.prune() # stop recursing def n_mklambda(self, node): - self.make_function(node, isLambda=1) + self.make_function(node, isLambda=True) self.prune() # stop recursing def n_list_compr(self, node): @@ -1251,7 +1263,7 @@ class Walker(GenericASTTraversal, object): # return self.traverse(node[1]) raise Exception("Can't find tuple parameter " + name) - def make_function(self, node, isLambda, nested=1): + def make_function(self, node, isLambda, nested=1, code_index=-2): """Dump function defintion, doc string, and function body.""" def build_param(ast, name, default): @@ -1276,8 +1288,9 @@ class Walker(GenericASTTraversal, object): return result else: return name - defparams = node[:node[-1].attr] # node[-1] == MAKE_xxx_n - code = node[-2].attr + # node[-1] == MAKE_xxx_n + defparams = node[:node[-1].attr] + code = node[code_index].attr assert inspect.iscode(code) code = Code(code, self.scanner, self.currentclass)