pydisassemble improvements; DRY scannners

disas.py:
 - disassembles *all* code objects found

scanner*.py:
 - no longer need to pass in version numbers; this
   is obtained from the class name
 - no longer pass in opcodes; this is done at
   initialization from the scanner name
 - all Pythoin 3 scanners support native disassembly

pytest/test_load.py

@@ -1,10 +1,13 @@
-import sys
+import os, sys
 from uncompyle6.load import load_file, check_object_path, load_module
 
 def test_load():
     """Basic test of load_file, check_object_path and load_module"""
     co = load_file(__file__)
     obj_path = check_object_path(__file__)
-    version, timestamp, magic_int, co2 = load_module(obj_path)
+    if os.path.exists(obj_path):
-    assert sys.version[0:3] == str(version)
+        version, timestamp, magic_int, co2 = load_module(obj_path)
-    assert co == co2
+        assert sys.version[0:3] == str(version)
+        assert co == co2
+    else:
+        assert True, "Skipped because we can't find %s" % obj_path

pytest/testdata/if-2.7.right

@@ -9,4 +9,3 @@
          12	JUMP_FORWARD          0  '15'
          15	LOAD_CONST            0  ''
          18	RETURN_VALUE             ''
-

pytest/testdata/ifelse-2.7.right

@@ -12,4 +12,3 @@
          18	STORE_NAME            2  'd'
          21	LOAD_CONST            2  ''
          24	RETURN_VALUE             ''
-

uncompyle6/disas.py

@@ -19,6 +19,7 @@ want to run on Python 2.7.
 from __future__ import print_function
 
 import os, sys
+from collections import deque
 
 import uncompyle6
 from uncompyle6.code import iscode
@@ -40,17 +41,29 @@ def disco(version, co, out=None, use_uncompyle6_format=False):
               file=real_out)
 
     scanner = get_scanner(version)
-    if (not use_uncompyle6_format) and hasattr(scanner, 'disassemble_native'):
-        tokens, customize = scanner.disassemble_native(co, True)
-    else:
-        tokens, customize = scanner.disassemble(co)
 
-    # FIXME: This should go in another routine and
+    disasm = scanner.disassemble_native \
-    # a queue should be kept of code to disassemble.
+      if (not use_uncompyle6_format) and hasattr(scanner, 'disassemble_native') \
-    for t in tokens:
+      else scanner.disassemble
-        print(t.format(), file=real_out)
-    print(file=out)
 
+    queue = deque([co])
+    disco_loop(disasm, queue, real_out, use_uncompyle6_format)
+
+
+def disco_loop(disasm, queue, real_out, use_uncompyle6_format):
+    while len(queue) > 0:
+        co = queue.popleft()
+        if co.co_name != '<module>':
+            print('\n# %s line %d of %s' %
+                      (co.co_name, co.co_firstlineno, co.co_filename),
+                      file=real_out)
+        tokens, customize = disasm(co, use_uncompyle6_format)
+        for t in tokens:
+            if iscode(t.pattr):
+                queue.append(t.pattr)
+            print(t.format(), file=real_out)
+            pass
+        pass
 
 def disassemble_file(filename, outstream=None, native=False):
     """

uncompyle6/scanner.py

@@ -50,6 +50,7 @@ class Code(object):
 class Scanner(object):
 
     def __init__(self, version):
+        self.version = version
         # FIXME: DRY
         if version == 2.7:
             self.opc = opcode_27
@@ -68,6 +69,7 @@ class Scanner(object):
         else:
             raise TypeError("%s is not a Python version I know about" % version)
 
+        self.opname = self.opc.opname
         # FIXME: This weird Python2 behavior is not Python3
         self.resetTokenClass()
 
@@ -96,9 +98,9 @@ class Scanner(object):
             op = self.code[i]
             if op in self.opc.hasjabs+self.opc.hasjrel:
                 dest = self.get_target(i, op)
-                print('%i\t%s\t%i' % (i, self.opc.opname[op], dest))
+                print('%i\t%s\t%i' % (i, self.opname[op], dest))
             else:
-                print('%i\t%s\t' % (i, self.opc.opname[op]))
+                print('%i\t%s\t' % (i, self.opname[op]))
 
     def first_instr(self, start, end, instr, target=None, exact=True):
         """
@@ -291,27 +293,28 @@ def get_scanner(version):
     # from trepan.api import debug;
     # debug(start_opts={'startup-profile': True})
 
+    # FIXME: see if we can do better
     if version == 2.7:
         import uncompyle6.scanners.scanner27 as scan
-        scanner = scan.Scanner27(version)
+        scanner = scan.Scanner27()
     elif version == 2.6:
         import uncompyle6.scanners.scanner26 as scan
-        scanner = scan.Scanner26(version)
+        scanner = scan.Scanner26()
     elif version == 2.5:
         import uncompyle6.scanners.scanner25 as scan
-        scanner = scan.Scanner25(version)
+        scanner = scan.Scanner25()
     elif version == 3.2:
         import uncompyle6.scanners.scanner32 as scan
-        scanner = scan.Scanner32(version)
+        scanner = scan.Scanner32()
     elif version == 3.3:
         import uncompyle6.scanners.scanner33 as scan
-        scanner = scan.Scanner33(version)
+        scanner = scan.Scanner33()
     elif version == 3.4:
         import uncompyle6.scanners.scanner34 as scan
-        scanner = scan.Scanner34(version)
+        scanner = scan.Scanner34()
     elif version == 3.5:
         import uncompyle6.scanners.scanner35 as scan
-        scanner = scan.Scanner35(version)
+        scanner = scan.Scanner35()
     else:
         raise RuntimeError("Unsupported Python version %d" % version)
     return scanner

uncompyle6/scanners/scanner25.py

@@ -1,8 +1,7 @@
-
+#  Copyright (c) 2015-2016 by Rocky Bernstein
-#  Copyright (c) 1999 John Aycock
-#  Copyright (c) 2000-2002 by hartmut Goebel <h.goebel@crazy-compilers.com>
 #  Copyright (c) 2005 by Dan Pascu <dan@windowmaker.org>
-#  Copyright (c) 2015 by Rocky Bernstein
+#  Copyright (c) 2000-2002 by hartmut Goebel <h.goebel@crazy-compilers.com>
+#  Copyright (c) 1999 John Aycock
 #
 """
 Python 2.5 bytecode scanner/deparser
@@ -20,8 +19,8 @@ from uncompyle6.opcodes.opcode_25 import *
 import uncompyle6.scanner as scan
 
 class Scanner25(scan.Scanner):
-    def __init__(self, version):
+    def __init__(self):
-        scan.Scanner.__init__(self, 2.5) # check
+        scan.Scanner.__init__(self, 2.5)
 
     def disassemble(self, co, classname=None, code_objects={}):
         '''
@@ -132,7 +131,7 @@ class Scanner25(scan.Scanner):
         extended_arg = 0
         for offset in self.op_range(0, codelen):
             op = self.code[offset]
-            op_name = opname[op]
+            op_name = self.opname[op]
             oparg = None; pattr = None
 
             if offset in cf:
@@ -913,10 +912,4 @@ class Scanner25(scan.Scanner):
                 label = self.fixed_jumps[i]
                 targets[label] = targets.get(label, []) + [i]
         return targets
-
+    pass
-if __name__ == "__main__":
-    import inspect
-    co = inspect.currentframe().f_code
-    tokens, customize = Scanner25().disassemble(co)
-    for t in tokens:
-        print(t)

uncompyle6/scanners/scanner26.py

@@ -19,7 +19,7 @@ import dis
 import uncompyle6.scanner as scan
 
 class Scanner26(scan.Scanner):
-    def __init__(self, version):
+    def __init__(self):
         scan.Scanner.__init__(self, 2.6)
 
     def disassemble(self, co, classname=None, code_objects={}):
@@ -130,7 +130,7 @@ class Scanner26(scan.Scanner):
         extended_arg = 0
         for offset in self.op_range(0, codelen):
             op = self.code[offset]
-            op_name = opname[op]
+            op_name = self.opname[op]
             oparg = None; pattr = None
 
             if offset in cf:
@@ -180,7 +180,10 @@ class Scanner26(scan.Scanner):
                 elif op in haslocal:
                     pattr = varnames[oparg]
                 elif op in hascompare:
-                    pattr = cmp_op[oparg]
+                    try:
+                        pattr = cmp_op[oparg]
+                    except:
+                        from trepan.api import debug; debug()
                 elif op in hasfree:
                     pattr = free[oparg]
             if offset in self.toChange:
@@ -917,8 +920,13 @@ class Scanner26(scan.Scanner):
         return targets
 
 if __name__ == "__main__":
-    import inspect
+    from uncompyle6 import PYTHON_VERSION
-    co = inspect.currentframe().f_code
+    if PYTHON_VERSION == 2.6:
-    tokens, customize = Scanner26().disassemble(co)
+        import inspect
-    for t in tokens:
+        co = inspect.currentframe().f_code
-        print(t)
+        tokens, customize = Scanner26().disassemble(co)
+        for t in tokens:
+            print(t.format())
+    else:
+        print("Need to be Python 2.6 to demo; I am %s." %
+              PYTHON_VERSION)

uncompyle6/scanners/scanner27.py

@@ -22,8 +22,8 @@ from uncompyle6.opcodes.opcode_27 import * # NOQA
 import uncompyle6.scanner as scan
 
 class Scanner27(scan.Scanner):
-    def __init__(self, version):
+    def __init__(self):
-        scan.Scanner.__init__(self, 2.7) # check
+        scan.Scanner.__init__(self, 2.7)
 
     def disassemble(self, co, classname=None, code_objects={}):
         """
@@ -192,7 +192,7 @@ class Scanner27(scan.Scanner):
                 tokens.append(Token(replace[offset], oparg, pattr, offset, linestart))
         return tokens, customize
 
-    def disassemble_native(self, co, opnames, classname=None, code_objects={}):
+    def disassemble_native(self, co, classname=None, code_objects={}):
         """
         Like disassemble3 but doesn't try to adjust any opcodes.
         """
@@ -728,8 +728,7 @@ class Scanner27(scan.Scanner):
 
 if __name__ == "__main__":
     co = inspect.currentframe().f_code
-    from uncompyle6 import PYTHON_VERSION
+    tokens, customize = Scanner27().disassemble(co)
-    tokens, customize = Scanner27(PYTHON_VERSION).disassemble(co)
     for t in tokens:
         print(t)
     pass

uncompyle6/scanners/scanner3.py

@@ -45,17 +45,15 @@ import uncompyle6.scanner as scan
 
 class Scanner3(scan.Scanner):
 
-    ## FIXME opnames should be passed in here
     def __init__(self, version):
-        self.version = version
+        if PYTHON3:
-        self.opnames = {} # will eventually get passed in
+            super().__init__(version)
-        scan.Scanner.__init__(self, version)
+        else:
+            super(Scanner3, self).__init__(version)
 
-
+    def disassemble3(self, co, classname=None, code_objects={}):
-    ## FIXME opnames should be moved to init
-    def disassemble3(self, co, opnames, classname=None, code_objects={}):
         """
-        Disassemble a Python 3 ode object, returning a list of 'Token'.
+        Disassemble a Python 3 code object, returning a list of 'Token'.
         Various tranformations are made to assist the deparsing grammar.
         For example:
            -  various types of LOAD_CONST's are categorized in terms of what they load
@@ -65,8 +63,6 @@ class Scanner3(scan.Scanner):
         dis.disassemble().
         """
 
-        self.opnames = opnames # will eventually disasppear
-
         # import dis; dis.disassemble(co) # DEBUG
 
         # Container for tokens
@@ -76,7 +72,7 @@ class Scanner3(scan.Scanner):
         self.build_lines_data(co)
         self.build_prev_op()
 
-        bytecode = dis3.Bytecode(co, opnames)
+        bytecode = dis3.Bytecode(co, self.opname)
 
         # Scan for assertions. Later we will
         # turn 'LOAD_GLOBAL' to 'LOAD_ASSERT' for those
@@ -164,7 +160,7 @@ class Scanner3(scan.Scanner):
                 pattr = inst.argval
                 target = self.get_target(inst.offset)
                 if target < inst.offset:
-                    next_opname = opnames[self.code[inst.offset+3]]
+                    next_opname = self.opname[self.code[inst.offset+3]]
                     if (inst.offset in self.stmts and
                         next_opname not in ('END_FINALLY', 'POP_BLOCK')
                         and inst.offset not in self.not_continue):
@@ -187,7 +183,7 @@ class Scanner3(scan.Scanner):
             pass
         return tokens, {}
 
-    def disassemble3_native(self, co, opnames, classname=None, code_objects={}):
+    def disassemble3_native(self, co, classname=None, code_objects={}):
         """
         Like disassemble3 but doesn't try to adjust any opcodes.
         """
@@ -196,7 +192,7 @@ class Scanner3(scan.Scanner):
 
         self.code = array('B', co.co_code)
 
-        bytecode = dis3.Bytecode(co, opnames)
+        bytecode = dis3.Bytecode(co, self.opname)
 
         for inst in bytecode:
             pattr =  inst.argrepr
@@ -292,7 +288,7 @@ class Scanner3(scan.Scanner):
                 pass
 
             op = code[offset]
-            op_name = op3.opname[op]
+            op_name = self.opname[op]
 
             oparg = None; pattr = None
 
@@ -907,11 +903,15 @@ class Scanner3(scan.Scanner):
         return filtered
 
 if __name__ == "__main__":
-    import inspect
-    co = inspect.currentframe().f_code
     from uncompyle6 import PYTHON_VERSION
-    from opcode import opname
+    if PYTHON_VERSION >= 3.2:
-    tokens, customize = Scanner3(PYTHON_VERSION).disassemble3(co, opname)
+        import inspect
-    for t in tokens:
+        co = inspect.currentframe().f_code
-        print(t)
+        from uncompyle6 import PYTHON_VERSION
+        tokens, customize = Scanner3(PYTHON_VERSION).disassemble3(co)
+        for t in tokens:
+            print(t.format())
+    else:
+        print("Need to be Python 3.2 or greater to demo; I am %s." %
+              PYTHON_VERSION)
     pass

uncompyle6/scanners/scanner32.py

@@ -2,28 +2,38 @@
 """
 Python 3.2 bytecode scanner/deparser
 
-This overlaps various Python3.2's dis module, but it can be run from
+This sets up opcodes Python's 3.2 and calls a generalized
-Python versions other than the version running this code. Notably,
+scanner routine for Python 3.
-run from Python version 2. See doc comments in scanner3.py for more information.
 """
 
 from __future__ import print_function
 
-import uncompyle6.scanners.scanner3 as scan3
+import uncompyle6
 
-import uncompyle6.opcodes.opcode_32
+# bytecode verification, verify(), uses JUMP_OPs from here
-# verify uses JUMP_OPs from here
 JUMP_OPs = uncompyle6.opcodes.opcode_32.JUMP_OPs
 
-class Scanner32(scan3.Scanner3):
+from uncompyle6.scanners.scanner3 import Scanner3
+class Scanner32(Scanner3):
+
+    def __init__(self):
+        super(Scanner3, self).__init__(3.2)
 
     def disassemble(self, co, classname=None, code_objects={}):
         return self.disassemble_generic(co, classname, code_objects=code_objects)
 
+    def disassemble_native(self, co, classname=None, code_objects={}):
+        return self.disassemble3_native(co, classname, code_objects)
+
 if __name__ == "__main__":
-    import inspect
+    from uncompyle6 import PYTHON_VERSION
-    co = inspect.currentframe().f_code
+    if PYTHON_VERSION == 3.2:
-    tokens, customize = Scanner32(3.2).disassemble(co)
+        import inspect
-    for t in tokens:
+        co = inspect.currentframe().f_code
-        print(t)
+        tokens, customize = Scanner32().disassemble(co)
-    pass
+        for t in tokens:
+            print(t)
+        pass
+    else:
+        print("Need to be Python 3.2 to demo; I am %s." %
+              PYTHON_VERSION)

uncompyle6/scanners/scanner33.py

@@ -9,21 +9,32 @@ for later use in deparsing.
 
 from __future__ import print_function
 
-import uncompyle6.scanners.scanner3 as scan3
+import uncompyle6
 
-import uncompyle6.opcodes.opcode_33
+# bytecode verification, verify(), uses JUMP_OPs from here
-# verify uses JUMP_OPs from here
 JUMP_OPs = uncompyle6.opcodes.opcode_33.JUMP_OPs
 
-class Scanner33(scan3.Scanner3):
+from uncompyle6.scanners.scanner3 import Scanner3
+class Scanner33(Scanner3):
+
+    def __init__(self):
+        super(Scanner3, self).__init__(3.3)
 
     def disassemble(self, co, classname=None, code_objects={}):
         return self.disassemble_generic(co, classname, code_objects=code_objects)
 
+    def disassemble_native(self, co, classname=None, code_objects={}):
+        return self.disassemble3_native(co, classname, code_objects)
+
 if __name__ == "__main__":
-    import inspect
+    from uncompyle6 import PYTHON_VERSION
-    co = inspect.currentframe().f_code
+    if PYTHON_VERSION == 3.3:
-    tokens, customize = Scanner33(3.3).disassemble(co)
+        import inspect
-    for t in tokens:
+        co = inspect.currentframe().f_code
-        print(t)
+        tokens, customize = Scanner33().disassemble(co)
-    pass
+        for t in tokens:
+            print(t)
+        pass
+    else:
+        print("Need to be Python 3.3 to demo; I am %s." %
+              PYTHON_VERSION)

uncompyle6/scanners/scanner34.py

@@ -2,29 +2,38 @@
 """
 Python 3.4 bytecode scanner/deparser
 
-This sets up opcodes Python's 3.5 and calls a generalized
+This sets up opcodes Python's 3.4 and calls a generalized
 scanner routine for Python 3.
 """
 
 from __future__ import print_function
 
-from uncompyle6.scanners.scanner3 import Scanner3
+import uncompyle6
-from uncompyle6.opcodes.opcode_34 import opname as opnames
 
 # bytecode verification, verify(), uses JUMP_OPs from here
 from uncompyle6.opcodes.opcode_34 import JUMP_OPs
 
+from uncompyle6.scanners.scanner3 import Scanner3
 class Scanner34(Scanner3):
+
+    def __init__(self):
+        super(Scanner3, self).__init__(3.4)
+
     def disassemble(self, co, classname=None, code_objects={}):
-        return self.disassemble3(co, opnames, classname, code_objects)
+        return self.disassemble3(co, classname, code_objects)
 
     def disassemble_native(self, co, classname=None, code_objects={}):
-        return self.disassemble3_native(co, opnames, classname, code_objects)
+        return self.disassemble3_native(co, classname, code_objects)
 
 if __name__ == "__main__":
-    import inspect
+    from uncompyle6 import PYTHON_VERSION
-    co = inspect.currentframe().f_code
+    if PYTHON_VERSION >= 3.2:
-    tokens, customize = Scanner34(3.4).disassemble(co)
+        import inspect
-    for t in tokens:
+        co = inspect.currentframe().f_code
-        print(t)
+        tokens, customize = Scanner34(3.4).disassemble(co)
-    pass
+        for t in tokens:
+            print(t)
+        pass
+    else:
+        print("Need to be Python 3.2 or greater to demo; I am %s." %
+              PYTHON_VERSION)

uncompyle6/scanners/scanner35.py

@@ -9,22 +9,30 @@ scanner routine for Python 3.
 from __future__ import print_function
 
 from uncompyle6.scanners.scanner3 import Scanner3
-from uncompyle6.opcodes.opcode_35 import opname as opnames
 
 # bytecode verification, verify(), uses JUMP_OPs from here
 from uncompyle6.opcodes.opcode_35 import JUMP_OPs
 
 class Scanner35(Scanner3):
+
+    def __init__(self):
+        super(Scanner3, self).__init__(3.5)
+
     def disassemble(self, co, classname=None, code_objects={}):
-        return self.disassemble3(co, opnames, classname, code_objects)
+        return self.disassemble3(co, classname, code_objects)
 
     def disassemble_native(self, co, classname=None, code_objects={}):
-        return self.disassemble3_native(co, opnames, classname, code_objects)
+        return self.disassemble3_native(co, classname, code_objects)
 
 if __name__ == "__main__":
-    import inspect
+    from uncompyle6 import PYTHON_VERSION
-    co = inspect.currentframe().f_code
+    if PYTHON_VERSION == 3.5:
-    tokens, customize = Scanner35(3.5).disassemble(co)
+        import inspect
-    for t in tokens:
+        co = inspect.currentframe().f_code
-        print(t)
+        tokens, customize = Scanner35(3.5).disassemble(co)
-    pass
+        for t in tokens:
+            print(t.format())
+        pass
+    else:
+        print("Need to be Python 3.5to demo; I am %s." %
+              PYTHON_VERSION)

uncompyle6/scanners/tok.py

@@ -1,3 +1,7 @@
+#  Copyright (c) 2016 by Rocky Bernstein
+#  Copyright (c) 2000-2002 by hartmut Goebel <h.goebel@crazy-compilers.com>
+#  Copyright (c) 1999 John Aycock
+
 import sys
 from uncompyle6 import PYTHON3
 
@@ -6,9 +10,9 @@ if PYTHON3:
 
 class Token:
     """
-    Class representing a byte-code token.
+    Class representing a byte-code instruction.
 
-    A byte-code token is equivalent to Python 3's  dis.instruction or
+    A byte-code token is equivalent to Python 3's dis.instruction or
     the contents of one line as output by dis.dis().
     """
     # FIXME: match Python 3.4's terms:

uncompyle6/verify.py

@@ -182,22 +182,22 @@ def cmp_code_objects(version, code_obj1, code_obj2, name=''):
         elif member == 'co_code':
             if version == 2.5:
                 import uncompyle6.scanners.scanner25 as scan
-                scanner = scan.Scanner25(version)
+                scanner = scan.Scanner25()
             elif version == 2.6:
                 import uncompyle6.scanners.scanner26 as scan
-                scanner = scan.Scanner26(version)
+                scanner = scan.Scanner26()
             elif version == 2.7:
                 import uncompyle6.scanners.scanner27 as scan
-                scanner = scan.Scanner27(version)
+                scanner = scan.Scanner27()
             elif version == 3.2:
                 import uncompyle6.scanners.scanner32 as scan
-                scanner = scan.Scanner32(version)
+                scanner = scan.Scanner32()
             elif version == 3.3:
                 import uncompyle6.scanners.scanner33 as scan
-                scanner = scan.Scanner33(version)
+                scanner = scan.Scanner33()
             elif version == 3.4:
                 import uncompyle6.scanners.scanner34 as scan
-                scanner = scan.Scanner34(version)
+                scanner = scan.Scanner34()
 
             global JUMP_OPs
             JUMP_OPs = list(scan.JUMP_OPs) + ['JUMP_BACK']