From a08ece371e3cdc03b454f1c708a1754ffe570f19 Mon Sep 17 00:00:00 2001 From: rocky Date: Wed, 18 May 2016 10:27:29 -0400 Subject: [PATCH] pydisassemble improvements; DRY scannners disas.py: - disassembles *all* code objects found scanner*.py: - no longer need to pass in version numbers; this is obtained from the class name - no longer pass in opcodes; this is done at initialization from the scanner name - all Pythoin 3 scanners support native disassembly --- pytest/test_load.py | 11 ++++++--- pytest/testdata/if-2.7.right | 1 - pytest/testdata/ifelse-2.7.right | 1 - uncompyle6/disas.py | 31 ++++++++++++++++------- uncompyle6/scanner.py | 21 +++++++++------- uncompyle6/scanners/scanner25.py | 21 ++++++---------- uncompyle6/scanners/scanner26.py | 24 ++++++++++++------ uncompyle6/scanners/scanner27.py | 9 +++---- uncompyle6/scanners/scanner3.py | 42 ++++++++++++++++---------------- uncompyle6/scanners/scanner32.py | 36 +++++++++++++++++---------- uncompyle6/scanners/scanner33.py | 31 +++++++++++++++-------- uncompyle6/scanners/scanner34.py | 31 ++++++++++++++--------- uncompyle6/scanners/scanner35.py | 26 +++++++++++++------- uncompyle6/scanners/tok.py | 8 ++++-- uncompyle6/verify.py | 12 ++++----- 15 files changed, 182 insertions(+), 123 deletions(-) diff --git a/pytest/test_load.py b/pytest/test_load.py index eafb7aee..63602265 100644 --- a/pytest/test_load.py +++ b/pytest/test_load.py @@ -1,10 +1,13 @@ -import sys +import os, sys from uncompyle6.load import load_file, check_object_path, load_module def test_load(): """Basic test of load_file, check_object_path and load_module""" co = load_file(__file__) obj_path = check_object_path(__file__) - version, timestamp, magic_int, co2 = load_module(obj_path) - assert sys.version[0:3] == str(version) - assert co == co2 + if os.path.exists(obj_path): + version, timestamp, magic_int, co2 = load_module(obj_path) + assert sys.version[0:3] == str(version) + assert co == co2 + else: + assert True, "Skipped because we can't find %s" % obj_path diff --git a/pytest/testdata/if-2.7.right b/pytest/testdata/if-2.7.right index 3a8f7768..1560c9a2 100644 --- a/pytest/testdata/if-2.7.right +++ b/pytest/testdata/if-2.7.right @@ -9,4 +9,3 @@ 12 JUMP_FORWARD 0 '15' 15 LOAD_CONST 0 '' 18 RETURN_VALUE '' - diff --git a/pytest/testdata/ifelse-2.7.right b/pytest/testdata/ifelse-2.7.right index ddf804ba..623c3a44 100644 --- a/pytest/testdata/ifelse-2.7.right +++ b/pytest/testdata/ifelse-2.7.right @@ -12,4 +12,3 @@ 18 STORE_NAME 2 'd' 21 LOAD_CONST 2 '' 24 RETURN_VALUE '' - diff --git a/uncompyle6/disas.py b/uncompyle6/disas.py index 85d8c929..3fc9f71d 100644 --- a/uncompyle6/disas.py +++ b/uncompyle6/disas.py @@ -19,6 +19,7 @@ want to run on Python 2.7. from __future__ import print_function import os, sys +from collections import deque import uncompyle6 from uncompyle6.code import iscode @@ -40,17 +41,29 @@ def disco(version, co, out=None, use_uncompyle6_format=False): file=real_out) scanner = get_scanner(version) - if (not use_uncompyle6_format) and hasattr(scanner, 'disassemble_native'): - tokens, customize = scanner.disassemble_native(co, True) - else: - tokens, customize = scanner.disassemble(co) - # FIXME: This should go in another routine and - # a queue should be kept of code to disassemble. - for t in tokens: - print(t.format(), file=real_out) - print(file=out) + disasm = scanner.disassemble_native \ + if (not use_uncompyle6_format) and hasattr(scanner, 'disassemble_native') \ + else scanner.disassemble + queue = deque([co]) + disco_loop(disasm, queue, real_out, use_uncompyle6_format) + + +def disco_loop(disasm, queue, real_out, use_uncompyle6_format): + while len(queue) > 0: + co = queue.popleft() + if co.co_name != '': + print('\n# %s line %d of %s' % + (co.co_name, co.co_firstlineno, co.co_filename), + file=real_out) + tokens, customize = disasm(co, use_uncompyle6_format) + for t in tokens: + if iscode(t.pattr): + queue.append(t.pattr) + print(t.format(), file=real_out) + pass + pass def disassemble_file(filename, outstream=None, native=False): """ diff --git a/uncompyle6/scanner.py b/uncompyle6/scanner.py index 5e443270..5af7801a 100755 --- a/uncompyle6/scanner.py +++ b/uncompyle6/scanner.py @@ -50,6 +50,7 @@ class Code(object): class Scanner(object): def __init__(self, version): + self.version = version # FIXME: DRY if version == 2.7: self.opc = opcode_27 @@ -68,6 +69,7 @@ class Scanner(object): else: raise TypeError("%s is not a Python version I know about" % version) + self.opname = self.opc.opname # FIXME: This weird Python2 behavior is not Python3 self.resetTokenClass() @@ -96,9 +98,9 @@ class Scanner(object): op = self.code[i] if op in self.opc.hasjabs+self.opc.hasjrel: dest = self.get_target(i, op) - print('%i\t%s\t%i' % (i, self.opc.opname[op], dest)) + print('%i\t%s\t%i' % (i, self.opname[op], dest)) else: - print('%i\t%s\t' % (i, self.opc.opname[op])) + print('%i\t%s\t' % (i, self.opname[op])) def first_instr(self, start, end, instr, target=None, exact=True): """ @@ -291,27 +293,28 @@ def get_scanner(version): # from trepan.api import debug; # debug(start_opts={'startup-profile': True}) + # FIXME: see if we can do better if version == 2.7: import uncompyle6.scanners.scanner27 as scan - scanner = scan.Scanner27(version) + scanner = scan.Scanner27() elif version == 2.6: import uncompyle6.scanners.scanner26 as scan - scanner = scan.Scanner26(version) + scanner = scan.Scanner26() elif version == 2.5: import uncompyle6.scanners.scanner25 as scan - scanner = scan.Scanner25(version) + scanner = scan.Scanner25() elif version == 3.2: import uncompyle6.scanners.scanner32 as scan - scanner = scan.Scanner32(version) + scanner = scan.Scanner32() elif version == 3.3: import uncompyle6.scanners.scanner33 as scan - scanner = scan.Scanner33(version) + scanner = scan.Scanner33() elif version == 3.4: import uncompyle6.scanners.scanner34 as scan - scanner = scan.Scanner34(version) + scanner = scan.Scanner34() elif version == 3.5: import uncompyle6.scanners.scanner35 as scan - scanner = scan.Scanner35(version) + scanner = scan.Scanner35() else: raise RuntimeError("Unsupported Python version %d" % version) return scanner diff --git a/uncompyle6/scanners/scanner25.py b/uncompyle6/scanners/scanner25.py index 5deb5f95..e068d6ee 100755 --- a/uncompyle6/scanners/scanner25.py +++ b/uncompyle6/scanners/scanner25.py @@ -1,8 +1,7 @@ - -# Copyright (c) 1999 John Aycock -# Copyright (c) 2000-2002 by hartmut Goebel +# Copyright (c) 2015-2016 by Rocky Bernstein # Copyright (c) 2005 by Dan Pascu -# Copyright (c) 2015 by Rocky Bernstein +# Copyright (c) 2000-2002 by hartmut Goebel +# Copyright (c) 1999 John Aycock # """ Python 2.5 bytecode scanner/deparser @@ -20,8 +19,8 @@ from uncompyle6.opcodes.opcode_25 import * import uncompyle6.scanner as scan class Scanner25(scan.Scanner): - def __init__(self, version): - scan.Scanner.__init__(self, 2.5) # check + def __init__(self): + scan.Scanner.__init__(self, 2.5) def disassemble(self, co, classname=None, code_objects={}): ''' @@ -132,7 +131,7 @@ class Scanner25(scan.Scanner): extended_arg = 0 for offset in self.op_range(0, codelen): op = self.code[offset] - op_name = opname[op] + op_name = self.opname[op] oparg = None; pattr = None if offset in cf: @@ -913,10 +912,4 @@ class Scanner25(scan.Scanner): label = self.fixed_jumps[i] targets[label] = targets.get(label, []) + [i] return targets - -if __name__ == "__main__": - import inspect - co = inspect.currentframe().f_code - tokens, customize = Scanner25().disassemble(co) - for t in tokens: - print(t) + pass diff --git a/uncompyle6/scanners/scanner26.py b/uncompyle6/scanners/scanner26.py index 4c6ffa0a..6ea0d73a 100755 --- a/uncompyle6/scanners/scanner26.py +++ b/uncompyle6/scanners/scanner26.py @@ -19,7 +19,7 @@ import dis import uncompyle6.scanner as scan class Scanner26(scan.Scanner): - def __init__(self, version): + def __init__(self): scan.Scanner.__init__(self, 2.6) def disassemble(self, co, classname=None, code_objects={}): @@ -130,7 +130,7 @@ class Scanner26(scan.Scanner): extended_arg = 0 for offset in self.op_range(0, codelen): op = self.code[offset] - op_name = opname[op] + op_name = self.opname[op] oparg = None; pattr = None if offset in cf: @@ -180,7 +180,10 @@ class Scanner26(scan.Scanner): elif op in haslocal: pattr = varnames[oparg] elif op in hascompare: - pattr = cmp_op[oparg] + try: + pattr = cmp_op[oparg] + except: + from trepan.api import debug; debug() elif op in hasfree: pattr = free[oparg] if offset in self.toChange: @@ -917,8 +920,13 @@ class Scanner26(scan.Scanner): return targets if __name__ == "__main__": - import inspect - co = inspect.currentframe().f_code - tokens, customize = Scanner26().disassemble(co) - for t in tokens: - print(t) + from uncompyle6 import PYTHON_VERSION + if PYTHON_VERSION == 2.6: + import inspect + co = inspect.currentframe().f_code + tokens, customize = Scanner26().disassemble(co) + for t in tokens: + print(t.format()) + else: + print("Need to be Python 2.6 to demo; I am %s." % + PYTHON_VERSION) diff --git a/uncompyle6/scanners/scanner27.py b/uncompyle6/scanners/scanner27.py index 02bc3f27..9f24b574 100755 --- a/uncompyle6/scanners/scanner27.py +++ b/uncompyle6/scanners/scanner27.py @@ -22,8 +22,8 @@ from uncompyle6.opcodes.opcode_27 import * # NOQA import uncompyle6.scanner as scan class Scanner27(scan.Scanner): - def __init__(self, version): - scan.Scanner.__init__(self, 2.7) # check + def __init__(self): + scan.Scanner.__init__(self, 2.7) def disassemble(self, co, classname=None, code_objects={}): """ @@ -192,7 +192,7 @@ class Scanner27(scan.Scanner): tokens.append(Token(replace[offset], oparg, pattr, offset, linestart)) return tokens, customize - def disassemble_native(self, co, opnames, classname=None, code_objects={}): + def disassemble_native(self, co, classname=None, code_objects={}): """ Like disassemble3 but doesn't try to adjust any opcodes. """ @@ -728,8 +728,7 @@ class Scanner27(scan.Scanner): if __name__ == "__main__": co = inspect.currentframe().f_code - from uncompyle6 import PYTHON_VERSION - tokens, customize = Scanner27(PYTHON_VERSION).disassemble(co) + tokens, customize = Scanner27().disassemble(co) for t in tokens: print(t) pass diff --git a/uncompyle6/scanners/scanner3.py b/uncompyle6/scanners/scanner3.py index 12b90e8c..b2f4f7f1 100644 --- a/uncompyle6/scanners/scanner3.py +++ b/uncompyle6/scanners/scanner3.py @@ -45,17 +45,15 @@ import uncompyle6.scanner as scan class Scanner3(scan.Scanner): - ## FIXME opnames should be passed in here def __init__(self, version): - self.version = version - self.opnames = {} # will eventually get passed in - scan.Scanner.__init__(self, version) + if PYTHON3: + super().__init__(version) + else: + super(Scanner3, self).__init__(version) - - ## FIXME opnames should be moved to init - def disassemble3(self, co, opnames, classname=None, code_objects={}): + def disassemble3(self, co, classname=None, code_objects={}): """ - Disassemble a Python 3 ode object, returning a list of 'Token'. + Disassemble a Python 3 code object, returning a list of 'Token'. Various tranformations are made to assist the deparsing grammar. For example: - various types of LOAD_CONST's are categorized in terms of what they load @@ -65,8 +63,6 @@ class Scanner3(scan.Scanner): dis.disassemble(). """ - self.opnames = opnames # will eventually disasppear - # import dis; dis.disassemble(co) # DEBUG # Container for tokens @@ -76,7 +72,7 @@ class Scanner3(scan.Scanner): self.build_lines_data(co) self.build_prev_op() - bytecode = dis3.Bytecode(co, opnames) + bytecode = dis3.Bytecode(co, self.opname) # Scan for assertions. Later we will # turn 'LOAD_GLOBAL' to 'LOAD_ASSERT' for those @@ -164,7 +160,7 @@ class Scanner3(scan.Scanner): pattr = inst.argval target = self.get_target(inst.offset) if target < inst.offset: - next_opname = opnames[self.code[inst.offset+3]] + next_opname = self.opname[self.code[inst.offset+3]] if (inst.offset in self.stmts and next_opname not in ('END_FINALLY', 'POP_BLOCK') and inst.offset not in self.not_continue): @@ -187,7 +183,7 @@ class Scanner3(scan.Scanner): pass return tokens, {} - def disassemble3_native(self, co, opnames, classname=None, code_objects={}): + def disassemble3_native(self, co, classname=None, code_objects={}): """ Like disassemble3 but doesn't try to adjust any opcodes. """ @@ -196,7 +192,7 @@ class Scanner3(scan.Scanner): self.code = array('B', co.co_code) - bytecode = dis3.Bytecode(co, opnames) + bytecode = dis3.Bytecode(co, self.opname) for inst in bytecode: pattr = inst.argrepr @@ -292,7 +288,7 @@ class Scanner3(scan.Scanner): pass op = code[offset] - op_name = op3.opname[op] + op_name = self.opname[op] oparg = None; pattr = None @@ -907,11 +903,15 @@ class Scanner3(scan.Scanner): return filtered if __name__ == "__main__": - import inspect - co = inspect.currentframe().f_code from uncompyle6 import PYTHON_VERSION - from opcode import opname - tokens, customize = Scanner3(PYTHON_VERSION).disassemble3(co, opname) - for t in tokens: - print(t) + if PYTHON_VERSION >= 3.2: + import inspect + co = inspect.currentframe().f_code + from uncompyle6 import PYTHON_VERSION + tokens, customize = Scanner3(PYTHON_VERSION).disassemble3(co) + for t in tokens: + print(t.format()) + else: + print("Need to be Python 3.2 or greater to demo; I am %s." % + PYTHON_VERSION) pass diff --git a/uncompyle6/scanners/scanner32.py b/uncompyle6/scanners/scanner32.py index a648c4c3..83631e65 100644 --- a/uncompyle6/scanners/scanner32.py +++ b/uncompyle6/scanners/scanner32.py @@ -2,28 +2,38 @@ """ Python 3.2 bytecode scanner/deparser -This overlaps various Python3.2's dis module, but it can be run from -Python versions other than the version running this code. Notably, -run from Python version 2. See doc comments in scanner3.py for more information. +This sets up opcodes Python's 3.2 and calls a generalized +scanner routine for Python 3. """ from __future__ import print_function -import uncompyle6.scanners.scanner3 as scan3 +import uncompyle6 -import uncompyle6.opcodes.opcode_32 -# verify uses JUMP_OPs from here +# bytecode verification, verify(), uses JUMP_OPs from here JUMP_OPs = uncompyle6.opcodes.opcode_32.JUMP_OPs -class Scanner32(scan3.Scanner3): +from uncompyle6.scanners.scanner3 import Scanner3 +class Scanner32(Scanner3): + + def __init__(self): + super(Scanner3, self).__init__(3.2) def disassemble(self, co, classname=None, code_objects={}): return self.disassemble_generic(co, classname, code_objects=code_objects) + def disassemble_native(self, co, classname=None, code_objects={}): + return self.disassemble3_native(co, classname, code_objects) + if __name__ == "__main__": - import inspect - co = inspect.currentframe().f_code - tokens, customize = Scanner32(3.2).disassemble(co) - for t in tokens: - print(t) - pass + from uncompyle6 import PYTHON_VERSION + if PYTHON_VERSION == 3.2: + import inspect + co = inspect.currentframe().f_code + tokens, customize = Scanner32().disassemble(co) + for t in tokens: + print(t) + pass + else: + print("Need to be Python 3.2 to demo; I am %s." % + PYTHON_VERSION) diff --git a/uncompyle6/scanners/scanner33.py b/uncompyle6/scanners/scanner33.py index d78427c9..616d1f2a 100644 --- a/uncompyle6/scanners/scanner33.py +++ b/uncompyle6/scanners/scanner33.py @@ -9,21 +9,32 @@ for later use in deparsing. from __future__ import print_function -import uncompyle6.scanners.scanner3 as scan3 +import uncompyle6 -import uncompyle6.opcodes.opcode_33 -# verify uses JUMP_OPs from here +# bytecode verification, verify(), uses JUMP_OPs from here JUMP_OPs = uncompyle6.opcodes.opcode_33.JUMP_OPs -class Scanner33(scan3.Scanner3): +from uncompyle6.scanners.scanner3 import Scanner3 +class Scanner33(Scanner3): + + def __init__(self): + super(Scanner3, self).__init__(3.3) def disassemble(self, co, classname=None, code_objects={}): return self.disassemble_generic(co, classname, code_objects=code_objects) + def disassemble_native(self, co, classname=None, code_objects={}): + return self.disassemble3_native(co, classname, code_objects) + if __name__ == "__main__": - import inspect - co = inspect.currentframe().f_code - tokens, customize = Scanner33(3.3).disassemble(co) - for t in tokens: - print(t) - pass + from uncompyle6 import PYTHON_VERSION + if PYTHON_VERSION == 3.3: + import inspect + co = inspect.currentframe().f_code + tokens, customize = Scanner33().disassemble(co) + for t in tokens: + print(t) + pass + else: + print("Need to be Python 3.3 to demo; I am %s." % + PYTHON_VERSION) diff --git a/uncompyle6/scanners/scanner34.py b/uncompyle6/scanners/scanner34.py index ee87e9ad..404a9e04 100644 --- a/uncompyle6/scanners/scanner34.py +++ b/uncompyle6/scanners/scanner34.py @@ -2,29 +2,38 @@ """ Python 3.4 bytecode scanner/deparser -This sets up opcodes Python's 3.5 and calls a generalized +This sets up opcodes Python's 3.4 and calls a generalized scanner routine for Python 3. """ from __future__ import print_function -from uncompyle6.scanners.scanner3 import Scanner3 -from uncompyle6.opcodes.opcode_34 import opname as opnames +import uncompyle6 # bytecode verification, verify(), uses JUMP_OPs from here from uncompyle6.opcodes.opcode_34 import JUMP_OPs +from uncompyle6.scanners.scanner3 import Scanner3 class Scanner34(Scanner3): + + def __init__(self): + super(Scanner3, self).__init__(3.4) + def disassemble(self, co, classname=None, code_objects={}): - return self.disassemble3(co, opnames, classname, code_objects) + return self.disassemble3(co, classname, code_objects) def disassemble_native(self, co, classname=None, code_objects={}): - return self.disassemble3_native(co, opnames, classname, code_objects) + return self.disassemble3_native(co, classname, code_objects) if __name__ == "__main__": - import inspect - co = inspect.currentframe().f_code - tokens, customize = Scanner34(3.4).disassemble(co) - for t in tokens: - print(t) - pass + from uncompyle6 import PYTHON_VERSION + if PYTHON_VERSION >= 3.2: + import inspect + co = inspect.currentframe().f_code + tokens, customize = Scanner34(3.4).disassemble(co) + for t in tokens: + print(t) + pass + else: + print("Need to be Python 3.2 or greater to demo; I am %s." % + PYTHON_VERSION) diff --git a/uncompyle6/scanners/scanner35.py b/uncompyle6/scanners/scanner35.py index b009603c..b3ab1c09 100644 --- a/uncompyle6/scanners/scanner35.py +++ b/uncompyle6/scanners/scanner35.py @@ -9,22 +9,30 @@ scanner routine for Python 3. from __future__ import print_function from uncompyle6.scanners.scanner3 import Scanner3 -from uncompyle6.opcodes.opcode_35 import opname as opnames # bytecode verification, verify(), uses JUMP_OPs from here from uncompyle6.opcodes.opcode_35 import JUMP_OPs class Scanner35(Scanner3): + + def __init__(self): + super(Scanner3, self).__init__(3.5) + def disassemble(self, co, classname=None, code_objects={}): - return self.disassemble3(co, opnames, classname, code_objects) + return self.disassemble3(co, classname, code_objects) def disassemble_native(self, co, classname=None, code_objects={}): - return self.disassemble3_native(co, opnames, classname, code_objects) + return self.disassemble3_native(co, classname, code_objects) if __name__ == "__main__": - import inspect - co = inspect.currentframe().f_code - tokens, customize = Scanner35(3.5).disassemble(co) - for t in tokens: - print(t) - pass + from uncompyle6 import PYTHON_VERSION + if PYTHON_VERSION == 3.5: + import inspect + co = inspect.currentframe().f_code + tokens, customize = Scanner35(3.5).disassemble(co) + for t in tokens: + print(t.format()) + pass + else: + print("Need to be Python 3.5to demo; I am %s." % + PYTHON_VERSION) diff --git a/uncompyle6/scanners/tok.py b/uncompyle6/scanners/tok.py index e70605bc..c2999e75 100644 --- a/uncompyle6/scanners/tok.py +++ b/uncompyle6/scanners/tok.py @@ -1,3 +1,7 @@ +# Copyright (c) 2016 by Rocky Bernstein +# Copyright (c) 2000-2002 by hartmut Goebel +# Copyright (c) 1999 John Aycock + import sys from uncompyle6 import PYTHON3 @@ -6,9 +10,9 @@ if PYTHON3: class Token: """ - Class representing a byte-code token. + Class representing a byte-code instruction. - A byte-code token is equivalent to Python 3's dis.instruction or + A byte-code token is equivalent to Python 3's dis.instruction or the contents of one line as output by dis.dis(). """ # FIXME: match Python 3.4's terms: diff --git a/uncompyle6/verify.py b/uncompyle6/verify.py index ddd1893c..f5d73213 100755 --- a/uncompyle6/verify.py +++ b/uncompyle6/verify.py @@ -182,22 +182,22 @@ def cmp_code_objects(version, code_obj1, code_obj2, name=''): elif member == 'co_code': if version == 2.5: import uncompyle6.scanners.scanner25 as scan - scanner = scan.Scanner25(version) + scanner = scan.Scanner25() elif version == 2.6: import uncompyle6.scanners.scanner26 as scan - scanner = scan.Scanner26(version) + scanner = scan.Scanner26() elif version == 2.7: import uncompyle6.scanners.scanner27 as scan - scanner = scan.Scanner27(version) + scanner = scan.Scanner27() elif version == 3.2: import uncompyle6.scanners.scanner32 as scan - scanner = scan.Scanner32(version) + scanner = scan.Scanner32() elif version == 3.3: import uncompyle6.scanners.scanner33 as scan - scanner = scan.Scanner33(version) + scanner = scan.Scanner33() elif version == 3.4: import uncompyle6.scanners.scanner34 as scan - scanner = scan.Scanner34(version) + scanner = scan.Scanner34() global JUMP_OPs JUMP_OPs = list(scan.JUMP_OPs) + ['JUMP_BACK']