Split out marhsal and disassemble code and spell disassemble correctly.

Fix some lint issues

.gitignore

@@ -1,6 +1,7 @@
-*_dis
 *.pyc
+*_dis
 *~
+/.eggs
 /.python-version
 /dist
 /uncompyle6.egg-info

__pkginfo__.py

@@ -30,11 +30,9 @@ modname            = 'uncompyle6'
 packages           = ['uncompyle6', 'uncompyle6.opcodes']
 py_modules         = None
 short_desc         = 'Python byte-code disassembler and source-code converter'
-# scripts            = ['bin/uncompyle6', 'bin/pydissassemble']
+scripts            = ['bin/uncompyle6', 'bin/pydisassemble']
-scripts            = ['bin/uncompyle6']
 
-import os
+import os.path
-import os.path, sys
 
 
 def get_srcdir():

bin/pydisassemble

@@ -0,0 +1,77 @@
+#!/usr/bin/env python
+# Mode: -*- python -*-
+#
+# Copyright (c) 2015 by Rocky Bernstein <rb@dustyfeet.com>
+#
+"""
+Usage: pydisassemble [OPTIONS]... FILE
+
+Examples:
+  pydisassemble      foo.pyc
+  pydisassemble      foo.py
+  pydisassemble -o foo.pydis foo.pyc
+  pydisassemble -o /tmp foo.pyc
+
+Options:
+  -o <path>     output decompiled files to this path:
+                if multiple input files are decompiled, the common prefix
+                is stripped from these names and the remainder appended to
+                <path>
+  --help        show this message
+
+"""
+
+from __future__ import print_function
+
+Usage_short = \
+"pydissassemble [--help] [--verify] [--showasm] [--showast] [-o <path>] FILE|DIR..."
+
+import sys, os, getopt
+import os.path
+
+from uncompyle6.disas import disassemble_files
+
+if sys.version[:3] != '2.7' and sys.version[:3] != '3.4':
+    print('Error:  pydisassemble requires Python 2.7 or 3.4.', file=sys.stderr)
+    sys.exit(-1)
+
+outfile = '-'
+out_base = None
+
+
+try:
+    opts, files = getopt.getopt(sys.argv[1:], 'ho:', ['help'])
+except getopt.GetoptError as e:
+    print('%s: %s' % (os.path.basename(sys.argv[0]), e),  file=sys.stderr)
+    sys.exit(-1)
+
+for opt, val in opts:
+    if opt in ('-h', '--help'):
+        print(__doc__)
+        sys.exit(0)
+    elif opt == '-o':
+        outfile = val
+    else:
+        print(opt)
+        print(Usage_short)
+        sys.exit(1)
+
+# argl, commonprefix works on strings, not on path parts,
+# thus we must handle the case with files in 'some/classes'
+# and 'some/cmds'
+src_base = os.path.commonprefix(files)
+if src_base[-1:] != os.sep:
+    src_base = os.path.dirname(src_base)
+if src_base:
+    sb_len = len( os.path.join(src_base, '') )
+    files = [f[sb_len:] for f in files]
+    del sb_len
+
+if outfile == '-':
+    outfile = None # use stdout
+elif outfile and os.path.isdir(outfile):
+    out_base = outfile; outfile = None
+elif outfile and len(files) > 1:
+    out_base = outfile; outfile = None
+
+disassemble_files(src_base, out_base, files, outfile)

bin/pydissassemble

@@ -1,201 +0,0 @@
-#!/usr/bin/env python
-# Mode: -*- python -*-
-#
-# Copyright (c) 2015 by Rocky Bernstein <rb@dustyfeet.com>
-#
-"""
-Usage: pydisassemble [OPTIONS]... FILE
-
-Examples:
-  pydisassemble      foo.pyc
-  pydisassemble      foo.py
-  pydisassemble -o foo.pydis foo.pyc
-  pydisassemble -o /tmp foo.pyc
-
-Options:
-  -o <path>     output decompiled files to this path:
-                if multiple input files are decompiled, the common prefix
-                is stripped from these names and the remainder appended to
-                <path>
-  --help        show this message
-
-"""
-
-from __future__ import print_function
-
-Usage_short = \
-"pydissassemble [--help] [--verify] [--showasm] [--showast] [-o <path>] FILE|DIR..."
-
-import sys, os, getopt, time, types
-import os.path
-import uncompyle6
-
-def disassemble_code(version, co, out=None):
-    """
-    diassembles and deparses a given code block 'co'
-    """
-
-    assert isinstance(co, types.CodeType)
-
-    # store final output stream for case of error
-    real_out = out or sys.stdout
-    print('# Python %s' % version, file=real_out)
-    if co.co_filename:
-        print('# Embedded file name: %s' % co.co_filename,
-              file=real_out)
-
-    # Pick up appropriate scanner
-    if version == 2.7:
-        import uncompyle6.scanners.scanner27 as scan
-        scanner = scan.Scanner27()
-    elif version == 2.6:
-        import uncompyle6.scanners.scanner26 as scan
-        scanner = scan.Scanner26()
-    elif version == 2.5:
-        import uncompyle6.scanners.scanner25 as scan
-        scanner = scan.Scanner25()
-    elif version == 3.2:
-        import uncompyle6.scanners.scanner32 as scan
-        scanner = scan.Scanner32()
-    elif version == 3.4:
-        import uncompyle6.scanners.scanner34 as scan
-        scanner = scan.Scanner34()
-    scanner.setShowAsm(True, out)
-    tokens, customize = scanner.disassemble(co)
-
-    for t in tokens:
-        print(t, file=real_out)
-    print(file=out)
-
-
-
-def disassemble_file(filename, outstream=None, showasm=False, showast=False):
-    """
-    disassemble Python byte-code file (.pyc)
-    """
-    version, co = uncompyle6.load_module(filename)
-    if type(co) == list:
-        for con in co:
-            disassemble_code(version, con, outstream)
-    else:
-        from trepan.api import debug; debug
-        disassemble_code(version, co, outstream)
-    co = None
-
-def disassemble_files(in_base, out_base, files, outfile=None):
-    """
-    in_base	base directory for input files
-    out_base	base directory for output files (ignored when
-    files	list of filenames to be uncompyled (relative to src_base)
-    outfile	write output to this filename (overwrites out_base)
-
-    For redirecting output to
-    - <filename>		outfile=<filename> (out_base is ignored)
-    - files below out_base	out_base=...
-    - stdout			out_base=None, outfile=None
-    """
-    def _get_outstream(outfile):
-        dir = os.path.dirname(outfile)
-        failed_file = outfile + '_failed'
-        if os.path.exists(failed_file):
-            os.remove(failed_file)
-        try:
-            os.makedirs(dir)
-        except OSError:
-            pass
-        return open(outfile, 'w')
-
-    of = outfile
-    if outfile == '-':
-        outfile = None # use stdout
-    elif outfile and os.path.isdir(outfile):
-        out_base = outfile; outfile = None
-    elif outfile:
-        out_base = outfile; outfile = None
-
-
-    for filename in files:
-        infile = os.path.join(in_base, filename)
-        # print (infile, file=sys.stderr)
-
-        if of: # outfile was given as parameter
-            outstream = _get_outstream(outfile)
-        elif out_base is None:
-            outstream = sys.stdout
-        else:
-            outfile = os.path.join(out_base, file) + '_dis'
-            outstream = _get_outstream(outfile)
-            # print(outfile, file=sys.stderr)
-            pass
-
-        # try to decomyple the input file
-        try:
-            disassemble_file(infile, outstream, showasm=True, showast=False)
-        except KeyboardInterrupt:
-            if outfile:
-                outstream.close()
-                os.remove(outfile)
-            raise
-        except:
-            if outfile:
-                outstream.close()
-                os.rename(outfile, outfile + '_failed')
-            else:
-                sys.stderr.write("\n# Can't disassemble %s\n" % infile)
-                import traceback
-                traceback.print_exc()
-        else: # uncompyle successfull
-            if outfile:
-                outstream.close()
-            if not outfile: print('\n# okay decompyling', infile)
-            sys.stdout.flush()
-
-        if outfile:
-            sys.stdout.write("\n")
-            sys.stdout.flush()
-        return
-
-if sys.version[:3] != '2.7' and sys.version[:3] != '3.4':
-    print('Error:  pydisassemble requires Python 2.7 or 3.4.', file=sys.stderr)
-    sys.exit(-1)
-
-outfile = '-'
-out_base = None
-
-
-try:
-    opts, files = getopt.getopt(sys.argv[1:], 'ho:', ['help'])
-except getopt.GetoptError as e:
-    print('%s: %s' % (os.path.basename(sys.argv[0]), e),  file=sys.stderr)
-    sys.exit(-1)
-
-for opt, val in opts:
-    if opt in ('-h', '--help'):
-        print(__doc__)
-        sys.exit(0)
-    elif opt == '-o':
-        outfile = val
-    else:
-        print(opt)
-        print(Usage_short)
-        sys.exit(1)
-
-# argl, commonprefix works on strings, not on path parts,
-# thus we must handle the case with files in 'some/classes'
-# and 'some/cmds'
-src_base = os.path.commonprefix(files)
-if src_base[-1:] != os.sep:
-    src_base = os.path.dirname(src_base)
-if src_base:
-    sb_len = len( os.path.join(src_base, '') )
-    files = [f[sb_len:] for f in files]
-    del sb_len
-
-if outfile == '-':
-    outfile = None # use stdout
-elif outfile and os.path.isdir(outfile):
-    out_base = outfile; outfile = None
-elif outfile and len(files) > 1:
-    out_base = outfile; outfile = None
-
-disassemble_files(src_base, out_base, files, outfile)

test/test_pythonlib.py

@@ -1,9 +1,7 @@
 #!/usr/bin/env python
 # emacs-mode: -*-python-*-
 
-from __future__ import print_function
+"""
-
-'''
 test_pythonlib.py -- compile, uncompyle, and verify Python libraries
 
 Usage-Examples:
@@ -27,11 +25,13 @@ Step 1) Edit this file and add a new entry to 'test_options', eg.
 Step 2: Run the test:
   test_pythonlib.py --mylib	  # decompile 'mylib'
   test_pythonlib.py --mylib --verify # decompile verify 'mylib'
-'''
+"""
+
+from __future__ import print_function
 
 import getopt, os, py_compile, sys, shutil, tempfile, time
 
-from uncompyle6 import main, verify, PYTHON_VERSION
+from uncompyle6 import main, PYTHON_VERSION
 from fnmatch import fnmatch
 
 def get_srcdir():

uncompyle6/__init__.py

@@ -4,6 +4,7 @@ from __future__ import print_function
 '''
   Copyright (c) 1999 John Aycock
   Copyright (c) 2000 by hartmut Goebel <h.goebel@crazy-compilers.com>
+  Copyright (c) 2015 by Rocky Bernstein
 
   Permission is hereby granted, free of charge, to any person obtaining
   a copy of this software and associated documentation files (the
@@ -33,8 +34,10 @@ from __future__ import print_function
 import os, marshal, sys, types
 
 PYTHON_VERSION = sys.version_info.major + (sys.version_info.minor / 10.0)
+PYTHON3 = (sys.version_info >= (3, 0))
 
-from uncompyle6 import disas, walker, verify, magics
+import uncompyle6
+from uncompyle6 import walker, verify, magics
 
 sys.setrecursionlimit(5000)
 __all__ = ['uncompyle_file', 'main']
@@ -95,7 +98,7 @@ def load_module(filename):
             bytecode = fp.read()
             co = marshal.loads(bytecode)
         else:
-            co = disas.load(fp, magic_int)
+            co = uncompyle6.marsh.load_code(fp, magic_int)
         pass
 
     return version, co
@@ -284,7 +287,7 @@ def main(in_base, out_base, files, codes, outfile=None,
                     verify_failed_files += 1
                     os.rename(outfile, outfile + '_unverified')
                     if not outfile:
-                        print("### Error Verifiying %s" % file,  file=sys.stderr)
+                        print("### Error Verifiying %s" % filename,  file=sys.stderr)
                         print(e, file=sys.stderr)
             else:
                 okay_files += 1

uncompyle6/deparser.py

@@ -1,4 +1,6 @@
-'''
+"""
+  Deparsing saving text fragment information indexed by offset
+
   Copyright (c) 1999 John Aycock
   Copyright (c) 2000-2002 by hartmut Goebel <h.goebel@crazy-compilers.com>
   Copyright (c) 2005 by Dan Pascu <dan@windowmaker.org>
@@ -6,7 +8,6 @@
 
   See main module for license.
 
-
   Decompilation (walking AST)
 
   All table-driven. (rocky: well, mostly. I need to add more format
@@ -44,7 +45,7 @@
 
   The '%' may optionally be followed by a number (C) in square brackets, which
   makes the engine walk down to N[C] before evaluating the escape code.
-'''
+"""
 
 from __future__ import print_function
 

uncompyle6/disas.py

@@ -1,201 +1,153 @@
-from __future__ import print_function
+"""
+CPython magic- and version- independent disassembly routines
 
-"""Disassembler of Python byte code into mnemonics.
+Copyright (c) 1999 John Aycock
+Copyright (c) 2000-2002 by hartmut Goebel <h.goebel@crazy-compilers.com>
+Copyright (c) 2005 by Dan Pascu <dan@windowmaker.org>
+Copyright (c) 2015 by Rocky Bernstein
 
 This is needed when the bytecode extracted is from
 a different version than the currently-running Python.
 
-When the two are the same, you can simply use marshal.loads()
+When the two are the same, you can simply use Python's built-in disassemble
-to prodoce a code object
 """
 
-import imp, marshal, pickle, sys, types
+from __future__ import print_function
-import dis as Mdis
 
-from struct import unpack
+import os, sys, types
 
 import uncompyle6
-from uncompyle6.magics import magic2int
 
-internStrings = []
+def disco(version, co, out=None):
-
-# XXX For backwards compatibility
-disco = Mdis.disassemble
-
-PYTHON3 = (sys.version_info >= (3, 0))
-PYTHON_MAGIC_INT = magic2int(imp.get_magic())
-
-if PYTHON3:
-    def long(n): return n
-
-def compat_str(s):
-    return s.decode('utf-8', errors='ignore') if PYTHON3 else str(s)
-
-def marshalLoad(fp):
-    global internStrings
-    internStrings = []
-    return load(fp)
-
-def load(fp, magic_int):
     """
-    marshal.load() written in Python. When the Python bytecode magic loaded is the
+    diassembles and deparses a given code block 'co'
-    same magic for the running Python interpreter, we can simply use the
-    Python-supplied mashal.load().
-
-    However we need to use this when versions are different since the internal
-    code structures are different. Sigh.
     """
-    global internStrings
 
-    marshalType = fp.read(1).decode('utf-8')
+    assert isinstance(co, types.CodeType)
-    if marshalType == 'c':
-        Code = types.CodeType
 
-        # FIXME If 'i' is deprecated, what would we use?
+    # store final output stream for case of error
-        co_argcount = unpack('i', fp.read(4))[0]
+    real_out = out or sys.stdout
-        co_nlocals = unpack('i', fp.read(4))[0]
+    print('# Python %s' % version, file=real_out)
-        co_stacksize = unpack('i', fp.read(4))[0]
+    if co.co_filename:
-        co_flags = unpack('i', fp.read(4))[0]
+        print('# Embedded file name: %s' % co.co_filename,
-        # FIXME: somewhere between Python 2.7 and python 3.2 there's
+              file=real_out)
-        # another 4 bytes before we get to the bytecode. What's going on?
-        # Again, because magic ints decreased between python 2.7 and 3.0 we need
-        # a range here.
-        if 3000 < magic_int < 20121:
-            fp.read(4)
-        co_code = load(fp, magic_int)
-        co_consts = load(fp, magic_int)
-        co_names = load(fp, magic_int)
-        co_varnames = load(fp, magic_int)
-        co_freevars = load(fp, magic_int)
-        co_cellvars = load(fp, magic_int)
-        co_filename = load(fp, magic_int)
-        co_name = load(fp, magic_int)
-        co_firstlineno = unpack('i', fp.read(4))[0]
-        co_lnotab = load(fp, magic_int)
-        # The Python3 code object is different than Python2's which
-        # we are reading if we get here.
-        # Also various parameters which were strings are now
-            # bytes (which is probably more logical).
-        if PYTHON3:
-            if PYTHON_MAGIC_INT > 3020:
-                # In later Python3 magic_ints, there is a
-                # kwonlyargcount parameter which we set to 0.
-                return Code(co_argcount, 0, co_nlocals, co_stacksize, co_flags,
-                            bytes(co_code, encoding='utf-8'),
-                            co_consts, co_names, co_varnames, co_filename, co_name,
-                            co_firstlineno, bytes(co_lnotab, encoding='utf-8'),
-                            co_freevars, co_cellvars)
-            else:
-                return Code(co_argcount, 0, co_nlocals, co_stacksize, co_flags,
-                            bytes(co_code, encoding='utf-8'),
-                            co_consts, co_names, co_varnames, co_filename, co_name,
-                            co_firstlineno, bytes(co_lnotab, encoding='utf-8'),
-                            co_freevars, co_cellvars)
-        else:
-            return Code(co_argcount, co_nlocals, co_stacksize, co_flags, co_code,
-                        co_consts, co_names, co_varnames, co_filename, co_name,
-                        co_firstlineno, co_lnotab, co_freevars, co_cellvars)
 
-    # const type
+    # Pick up appropriate scanner
-    elif marshalType == '.':
+    if version == 2.7:
-        return Ellipsis
+        import uncompyle6.scanners.scanner27 as scan
-    elif marshalType == '0':
+        scanner = scan.Scanner27()
-        raise KeyError(marshalType)
+    elif version == 2.6:
-        return None
+        import uncompyle6.scanners.scanner26 as scan
-    elif marshalType == 'N':
+        scanner = scan.Scanner26()
-        return None
+    elif version == 2.5:
-    elif marshalType == 'T':
+        import uncompyle6.scanners.scanner25 as scan
-        return True
+        scanner = scan.Scanner25()
-    elif marshalType == 'F':
+    elif version == 3.2:
-        return False
+        import uncompyle6.scanners.scanner32 as scan
-    elif marshalType == 'S':
+        scanner = scan.Scanner32()
-        return StopIteration
+    elif version == 3.4:
-    # number type
+        import uncompyle6.scanners.scanner34 as scan
-    elif marshalType == 'f':
+        scanner = scan.Scanner34()
-        n = fp.read(1)
+    scanner.setShowAsm(True, out)
-        return float(unpack('d', fp.read(n))[0])
+    tokens, customize = scanner.disassemble(co)
-    elif marshalType == 'g':
+
-        return float(unpack('d', fp.read(8))[0])
+    for t in tokens:
-    elif marshalType == 'i':
+        print(t, file=real_out)
-        return int(unpack('i', fp.read(4))[0])
+    print(file=out)
-    elif marshalType == 'I':
+
-        return unpack('q', fp.read(8))[0]
+
-    elif marshalType == 'x':
+def disassemble_file(filename, outstream=None):
-        raise KeyError(marshalType)
+    """
-        return None
+    disassemble Python byte-code file (.pyc)
-    elif marshalType == 'y':
+    """
-        raise KeyError(marshalType)
+    version, co = uncompyle6.load_module(filename)
-        return None
+    if type(co) == list:
-    elif marshalType == 'l':
+        for con in co:
-        n = unpack('i', fp.read(4))[0]
+            disco(version, con, outstream)
-        if n == 0:
-            return long(0)
-        size = abs(n)
-        d = long(0)
-        for j in range(0, size):
-            md = int(unpack('h', fp.read(2))[0])
-            d += md << j*15
-        if n < 0:
-            return long(d*-1)
-        return d
-    # strings type
-    elif marshalType == 'R':
-        refnum = unpack('i', fp.read(4))[0]
-        return internStrings[refnum]
-    elif marshalType == 's':
-        strsize = unpack('i', fp.read(4))[0]
-        return compat_str(fp.read(strsize))
-    elif marshalType == 't':
-        strsize = unpack('i', fp.read(4))[0]
-        interned = compat_str(fp.read(strsize))
-        internStrings.append(interned)
-        return interned
-    elif marshalType == 'u':
-        strsize = unpack('i', fp.read(4))[0]
-        unicodestring = fp.read(strsize)
-        return unicodestring.decode('utf-8')
-    # collection type
-    elif marshalType == '(':
-        tuplesize = unpack('i', fp.read(4))[0]
-        ret = tuple()
-        while tuplesize > 0:
-            ret += load(fp, magic_int),
-            tuplesize -= 1
-        return ret
-    elif marshalType == '[':
-        raise KeyError(marshalType)
-        return None
-    elif marshalType == '{':
-        raise KeyError(marshalType)
-        return None
-    elif marshalType in ['<', '>']:
-        raise KeyError(marshalType)
-        return None
     else:
-        sys.stderr.write("Unknown type %i (hex %x)\n" % (ord(marshalType), ord(marshalType)))
+        disco(version, co, outstream)
+    co = None
+
+def disassemble_files(in_base, out_base, files, outfile=None):
+    """
+    in_base	base directory for input files
+    out_base	base directory for output files (ignored when
+    files	list of filenames to be uncompyled (relative to src_base)
+    outfile	write output to this filename (overwrites out_base)
+
+    For redirecting output to
+    - <filename>		outfile=<filename> (out_base is ignored)
+    - files below out_base	out_base=...
+    - stdout			out_base=None, outfile=None
+    """
+    def _get_outstream(outfile):
+        dir = os.path.dirname(outfile)
+        failed_file = outfile + '_failed'
+        if os.path.exists(failed_file):
+            os.remove(failed_file)
+        try:
+            os.makedirs(dir)
+        except OSError:
+            pass
+        return open(outfile, 'w')
+
+    of = outfile
+    if outfile == '-':
+        outfile = None # use stdout
+    elif outfile and os.path.isdir(outfile):
+        out_base = outfile; outfile = None
+    elif outfile:
+        out_base = outfile; outfile = None
+
+    for filename in files:
+        infile = os.path.join(in_base, filename)
+        # print (infile, file=sys.stderr)
+
+        if of: # outfile was given as parameter
+            outstream = _get_outstream(outfile)
+        elif out_base is None:
+            outstream = sys.stdout
+        else:
+            outfile = os.path.join(out_base, filename) + '_dis'
+            outstream = _get_outstream(outfile)
+            # print(outfile, file=sys.stderr)
+            pass
+
+        # try to decomyple the input file
+        try:
+            disassemble_file(infile, outstream)
+        except KeyboardInterrupt:
+            if outfile:
+                outstream.close()
+                os.remove(outfile)
+            raise
+        except:
+            if outfile:
+                outstream.close()
+                os.rename(outfile, outfile + '_failed')
+            else:
+                sys.stderr.write("\n# Can't disassemble %s\n" % infile)
+                import traceback
+                traceback.print_exc()
+        else: # uncompyle successfull
+            if outfile:
+                outstream.close()
+            if not outfile: print('\n# okay decompyling', infile)
+            sys.stdout.flush()
+
+        if outfile:
+            sys.stdout.write("\n")
+            sys.stdout.flush()
+        return
 
 def _test():
     """Simple test program to disassemble a file."""
-    if sys.argv[1:]:
+    argc = len(sys.argv)
-        if sys.argv[2:]:
+    if argc != 2:
-            sys.stderr.write("usage: python dis.py [-|file]\n")
+        sys.stderr.write("usage: %s [-|CPython compiled file]\n" % __file__)
-            sys.exit(2)
+        sys.exit(2)
-        fn = sys.argv[1]
+    fn = sys.argv[1]
-        if not fn or fn == "-":
+    disassemble_file(fn)
-            fn = None
-    else:
-        fn = None
-    if fn is None:
-        f = sys.stdin
-    else:
-        f = open(fn)
-    source = f.read()
-    if fn is not None:
-        f.close()
-    else:
-        fn = "<stdin>"
-    code = compile(source, fn, "exec")
-    Mdis.dis(code)
 
 if __name__ == "__main__":
     _test()

uncompyle6/magics.py

@@ -2,7 +2,7 @@ from __future__ import print_function
 
 import struct, sys
 
-__all__ = ['magics', 'versions', 'magics2int']
+__all__ = ['magics', 'versions', 'magic2int']
 
 def __build_magic(magic):
     if (sys.version_info >= (3, 0)):
@@ -10,7 +10,7 @@ def __build_magic(magic):
     else:
         return struct.pack('Hcc', magic, '\r', '\n')
 
-def magic2int (magic):
+def magic2int(magic):
     return struct.unpack('Hcc', magic)[0]
 
 by_magic = {}
@@ -73,20 +73,20 @@ versions = {
     __build_magic(3160): '3.2',  # 3.2a0 (add SETUP_WITH)
     __build_magic(3170): '3.2',  # 3.2a1 (add DUP_TOP_TWO, remove DUP_TOPX and ROT_FOUR)
     __build_magic(3180): '3.2',  # 3.2a2 (add DELETE_DEREF)
-    __build_magic(3190): '3.3',  #     Python 3.3a0  3190 __class__ super closure changed
+    __build_magic(3190): '3.3',  # 3.3a0  3190 __class__ super closure changed
-    __build_magic(3200): '3.3',  #     Python 3.3a0  3200 (__qualname__ added)
+    __build_magic(3100): '3.3',  # 3.3a0  3200 (__qualname__ added)
-    __build_magic(3200): '3.3',  #     3210 (added size modulo 2**32 to the pyc header)
+    __build_magic(3210): '3.3',  # 3210 (added size modulo 2**32 to the pyc header)
-    __build_magic(3220): '3.3',  #     Python 3.3a1  3220 (changed PEP 380 implementation)
+    __build_magic(3220): '3.3',  # 3.3a1  3220 (changed PEP 380 implementation)
-    __build_magic(3230): '3.3',  #     Python 3.3a4  3230 (revert changes to implicit __class__ closure)
+    __build_magic(3230): '3.3',  # 3.3a4  3230 (revert changes to implicit __class__ closure)
-    __build_magic(3250): '3.4',  #     Python 3.4a1  3250 (evaluate positional default arg
+    __build_magic(3250): '3.4',  # 3.4a1  3250 (evaluate positional default arg
-                                 #                        keyword-only defaults)
+                                 # keyword-only defaults)
-    __build_magic(3260): '3.4',  #     Python 3.4a1  3260 (add LOAD_CLASSDEREF; allow locals of class to override
+    __build_magic(3260): '3.4',  # 3.4a1  3260 (add LOAD_CLASSDEREF;
-                                 #                        free vars)
+                                 # allow locals of class to override free vars)
-    __build_magic(3270): '3.4',  #     Python 3.4a1  3270 (various tweaks to the __class__ closure)
+    __build_magic(3270): '3.4',  # 3.4a1  3270 (various tweaks to the __class__ closure)
-    __build_magic(3280): '3.4',  #     Python 3.4a1  3280 (remove implicit class argument)
+    __build_magic(3280): '3.4',  # 3.4a1  3280 (remove implicit class argument)
-    __build_magic(3290): '3.4',  #     Python 3.4a4  3290 (changes to __qualname__ computation)
+    __build_magic(3290): '3.4',  # 3.4a4  3290 (changes to __qualname__ computation)
-    __build_magic(3300): '3.4',  #     Python 3.4a4  3300 (more changes to __qualname__ computation)
+    __build_magic(3300): '3.4',  # 3.4a4  3300 (more changes to __qualname__ computation)
-    __build_magic(3310): '3.4',  #     Python 3.4rc2 3310 (alter __qualname__ computation)
+    __build_magic(3310): '3.4',  # 3.4rc2 3310 (alter __qualname__ computation)
 
 }
 

uncompyle6/marsh.py

@@ -0,0 +1,176 @@
+"""
+CPython magic- and version- independent marshal routines
+
+  Copyright (c) 1999 John Aycock
+  Copyright (c) 2000-2002 by hartmut Goebel <h.goebel@crazy-compilers.com>
+  Copyright (c) 2005 by Dan Pascu <dan@windowmaker.org>
+  Copyright (c) 2015 by Rocky Bernstein
+
+This is needed when the bytecode extracted is from
+a different version than the currently-running Python.
+
+When the two are the same, you can simply use Python's built-in marshal.loads()
+to produce a code object
+"""
+
+from __future__ import print_function
+
+import imp, sys, types
+from struct import unpack
+
+from uncompyle6.magics import magic2int
+
+internStrings = []
+
+PYTHON_MAGIC_INT = magic2int(imp.get_magic())
+PYTHON3 = (sys.version_info >= (3, 0))
+
+if PYTHON3:
+    def long(n): return n
+
+def compat_str(s):
+    return s.decode('utf-8', errors='ignore') if PYTHON3 else str(s)
+
+def load_code(fp, magic_int):
+    """
+    marshal.load() written in Python. When the Python bytecode magic loaded is the
+    same magic for the running Python interpreter, we can simply use the
+    Python-supplied mashal.load().
+
+    However we need to use this when versions are different since the internal
+    code structures are different. Sigh.
+    """
+    global internStrings
+    internStrings = []
+    return load_code_internal(fp, magic_int)
+
+def load_code_internal(fp, magic_int):
+    global internStrings
+
+    marshalType = fp.read(1).decode('utf-8')
+    if marshalType == 'c':
+        Code = types.CodeType
+
+        # FIXME If 'i' is deprecated, what would we use?
+        co_argcount = unpack('i', fp.read(4))[0]
+        co_nlocals = unpack('i', fp.read(4))[0]
+        co_stacksize = unpack('i', fp.read(4))[0]
+        co_flags = unpack('i', fp.read(4))[0]
+        # FIXME: somewhere between Python 2.7 and python 3.2 there's
+        # another 4 bytes before we get to the bytecode. What's going on?
+        # Again, because magic ints decreased between python 2.7 and 3.0 we need
+        # a range here.
+        if 3000 < magic_int < 20121:
+            fp.read(4)
+        co_code = load_code(fp, magic_int)
+        co_consts = load_code(fp, magic_int)
+        co_names = load_code(fp, magic_int)
+        co_varnames = load_code(fp, magic_int)
+        co_freevars = load_code(fp, magic_int)
+        co_cellvars = load_code(fp, magic_int)
+        co_filename = load_code(fp, magic_int)
+        co_name = load_code(fp, magic_int)
+        co_firstlineno = unpack('i', fp.read(4))[0]
+        co_lnotab = load_code(fp, magic_int)
+        # The Python3 code object is different than Python2's which
+        # we are reading if we get here.
+        # Also various parameters which were strings are now
+            # bytes (which is probably more logical).
+        if PYTHON3:
+            if PYTHON_MAGIC_INT > 3020:
+                # In later Python3 magic_ints, there is a
+                # kwonlyargcount parameter which we set to 0.
+                return Code(co_argcount, 0, co_nlocals, co_stacksize, co_flags,
+                            bytes(co_code, encoding='utf-8'),
+                            co_consts, co_names, co_varnames, co_filename, co_name,
+                            co_firstlineno, bytes(co_lnotab, encoding='utf-8'),
+                            co_freevars, co_cellvars)
+            else:
+                return Code(co_argcount, 0, co_nlocals, co_stacksize, co_flags,
+                            bytes(co_code, encoding='utf-8'),
+                            co_consts, co_names, co_varnames, co_filename, co_name,
+                            co_firstlineno, bytes(co_lnotab, encoding='utf-8'),
+                            co_freevars, co_cellvars)
+        else:
+            return Code(co_argcount, co_nlocals, co_stacksize, co_flags, co_code,
+                        co_consts, co_names, co_varnames, co_filename, co_name,
+                        co_firstlineno, co_lnotab, co_freevars, co_cellvars)
+
+    # const type
+    elif marshalType == '.':
+        return Ellipsis
+    elif marshalType == '0':
+        raise KeyError(marshalType)
+        return None
+    elif marshalType == 'N':
+        return None
+    elif marshalType == 'T':
+        return True
+    elif marshalType == 'F':
+        return False
+    elif marshalType == 'S':
+        return StopIteration
+    # number type
+    elif marshalType == 'f':
+        n = fp.read(1)
+        return float(unpack('d', fp.read(n))[0])
+    elif marshalType == 'g':
+        return float(unpack('d', fp.read(8))[0])
+    elif marshalType == 'i':
+        return int(unpack('i', fp.read(4))[0])
+    elif marshalType == 'I':
+        return unpack('q', fp.read(8))[0]
+    elif marshalType == 'x':
+        raise KeyError(marshalType)
+        return None
+    elif marshalType == 'y':
+        raise KeyError(marshalType)
+        return None
+    elif marshalType == 'l':
+        n = unpack('i', fp.read(4))[0]
+        if n == 0:
+            return long(0)
+        size = abs(n)
+        d = long(0)
+        for j in range(0, size):
+            md = int(unpack('h', fp.read(2))[0])
+            d += md << j*15
+        if n < 0:
+            return long(d*-1)
+        return d
+    # strings type
+    elif marshalType == 'R':
+        refnum = unpack('i', fp.read(4))[0]
+        return internStrings[refnum]
+    elif marshalType == 's':
+        strsize = unpack('i', fp.read(4))[0]
+        return compat_str(fp.read(strsize))
+    elif marshalType == 't':
+        strsize = unpack('i', fp.read(4))[0]
+        interned = compat_str(fp.read(strsize))
+        internStrings.append(interned)
+        return interned
+    elif marshalType == 'u':
+        strsize = unpack('i', fp.read(4))[0]
+        unicodestring = fp.read(strsize)
+        return unicodestring.decode('utf-8')
+    # collection type
+    elif marshalType == '(':
+        tuplesize = unpack('i', fp.read(4))[0]
+        ret = tuple()
+        while tuplesize > 0:
+            ret += load_code(fp, magic_int),
+            tuplesize -= 1
+        return ret
+    elif marshalType == '[':
+        raise KeyError(marshalType)
+        return None
+    elif marshalType == '{':
+        raise KeyError(marshalType)
+        return None
+    elif marshalType in ['<', '>']:
+        raise KeyError(marshalType)
+        return None
+    else:
+        sys.stderr.write("Unknown type %i (hex %x)\n" % (ord(marshalType), ord(marshalType)))
+    return

uncompyle6/scanners/scanner25.py

@@ -1,16 +1,21 @@
-'''
+"""
+  Python 2.5 bytecode scanner/deparser
+
   Copyright (c) 1999 John Aycock
   Copyright (c) 2000-2002 by hartmut Goebel <h.goebel@crazy-compilers.com>
   Copyright (c) 2005 by Dan Pascu <dan@windowmaker.org>
+  Copyright (c) 2015 by Rocky Bernstein
 
   See main module for license.
-'''
+
+This overlaps Python's 2.5's dis module, but it can be run from
+Python3 and other versions of Python. Also, we save token information
+for later use in deparsing.
+"""
 
 import types
 from collections import namedtuple
 from array import array
-from operator import itemgetter
-from struct import *
 
 import dis
 from uncompyle6.opcodes.opcode_25 import *

uncompyle6/scanners/scanner26.py

@@ -1,15 +1,21 @@
-'''
+"""
+  Python 2.6 bytecode scanner
+
   Copyright (c) 1999 John Aycock
   Copyright (c) 2000-2002 by hartmut Goebel <h.goebel@crazy-compilers.com>
   Copyright (c) 2005 by Dan Pascu <dan@windowmaker.org>
+  Copyright (c) 2015 by Rocky Bernstein
 
   See main module for license.
-'''
+
+This overlaps Python's 2.6's dis module, but it can be run from Python3 and
+other versions of Python. Also, we save token information for later
+use in deparsing.
+"""
 
 import types
 from collections import namedtuple
 from array import array
-from operator import itemgetter
 
 from uncompyle6.opcodes.opcode_26 import *
 import dis

uncompyle6/scanners/scanner27.py

@@ -1,17 +1,23 @@
-from __future__ import print_function
+"""
+  Python 2.5 bytecode scanner/deparser
 
-'''
   Copyright (c) 1999 John Aycock
   Copyright (c) 2000-2002 by hartmut Goebel <h.goebel@crazy-compilers.com>
   Copyright (c) 2005 by Dan Pascu <dan@windowmaker.org>
+  Copyright (c) 2015 by Rocky Bernstein
 
   See main module for license.
-'''
+
+This overlaps Python's 2.7's dis module, but it can be run from
+Python3 and other versions of Python. Also, we save token information
+for later use in deparsing.
+"""
+
+from __future__ import print_function
 
 import dis, types
 from collections import namedtuple
 from array import array
-from operator import itemgetter
 
 from uncompyle6.opcodes.opcode_27 import *
 import uncompyle6.scanner as scan

uncompyle6/scanners/scanner32.py

@@ -1,12 +1,19 @@
-from __future__ import print_function
+"""
+  Python 3.2 bytecode scanner/deparser
 
-'''
   Copyright (c) 1999 John Aycock
   Copyright (c) 2000-2002 by hartmut Goebel <h.goebel@crazy-compilers.com>
   Copyright (c) 2005 by Dan Pascu <dan@windowmaker.org>
+  Copyright (c) 2015 by Rocky Bernstein
 
   See main module for license.
-'''
+
+This overlaps Python's 3.2's dis module, but it can be run from
+Python3 and other versions of Python. Also, we save token information
+for later use in deparsing.
+"""
+
+from __future__ import print_function
 
 import dis, marshal
 from collections import namedtuple

uncompyle6/scanners/scanner34.py

@@ -1,12 +1,15 @@
-from __future__ import print_function
+"""
+  Python 2.5 bytecode scanner/deparser
 
-'''
   Copyright (c) 1999 John Aycock
   Copyright (c) 2000-2002 by hartmut Goebel <h.goebel@crazy-compilers.com>
   Copyright (c) 2005 by Dan Pascu <dan@windowmaker.org>
+  Copyright (c) 2015 by Rocky Bernstein
 
   See main module for license.
-'''
+"""
+
+from __future__ import print_function
 
 import dis, marshal
 from collections import namedtuple