need disas.py for cross version Python compiling

fixup MANIFEST.in
pythonlib.py: store expected python version and don't compile if it
mismatches. Work files now go in temp directory. Start masrhal load in
Python for Python3.

MANIFEST.in

@@ -1,5 +1,5 @@
 include README.rst
 include __pkginfo__.py
-recursive-include uncompyle6
+recursive-include uncompyle6 *.py
 include script/uncompyle6
-recursive-include/test/*
+recursive-include test *.py *.pyc *.pyo

test/test_pythonlib.py

@@ -4,14 +4,21 @@
 from __future__ import print_function
 
 '''
-test_pythonlib.py -- uncompyle and verify Python libraries
+test_pythonlib.py -- compile, uncompyle, and verify Python libraries
 
 Usage-Examples:
 
-  test_pythonlib.py --all		# decompile all tests (suite + libs)
-  test_pythonlib.py --all --verify	# decomyile all tests and verify results
-  test_pythonlib.py --test		# decompile only the testsuite
-  test_pythonlib.py --2.2 --verify	# decompile and verify python lib 2.2
+  # decompile, and verify base set of python 2.7 byte-compiled files
+  test_pythonlib.py --base-2.7 --verify
+
+  # Same as above but compile the base set first
+  test_pythonlib.py --base-2.7 --verify --compile
+
+  # Same as above but use a longer set from the python 2.7 library
+  test_pythonlib.py --ok-2.7 --verify --compile
+
+  # Just deompile the longer set of files
+  test_pythonlib.py --ok-2.7
 
 Adding own test-trees:
 
@@ -24,7 +31,7 @@ Step 2: Run the test:
 
 import getopt, os, py_compile, sys, shutil, tempfile, time
 
-from uncompyle6 import main, verify
+from uncompyle6 import main, verify, PYTHON_VERSION
 from fnmatch import fnmatch
 
 def get_srcdir():
@@ -33,6 +40,7 @@ def get_srcdir():
 
 src_dir = get_srcdir()
 
+
 #----- configure this for your needs
 
 lib_prefix = [src_dir, '/usr/lib/', '/usr/local/lib/']
@@ -45,15 +53,15 @@ PYO = ('*.pyo', )
 PYOC = ('*.pyc', '*.pyo')
 
 test_options = {
-    # name:   (src_basedir, pattern, output_base_suffix)
+    # name:   (src_basedir, pattern, output_base_suffix, pythoin_version)
     'test':      ['test', PYC, 'test'],
-    '2.7':       ['python2.7', PYC, 'python2.7'],
+    '2.7':       ['python2.7', PYC, 'python2.7', '2.7'],
     'ok-2.6':    [os.path.join(src_dir, 'ok_2.6'),
-                 PYC, 'ok-2.6'],
+                 PYC, 'ok-2.6', '2.6'],
     'ok-2.7':    [os.path.join(src_dir, 'ok_2.7'),
-                 PYC, 'ok-2.7'],
+                 PYC, 'ok-2.7', '2.7'],
     'base-2.7':  [os.path.join(src_dir, 'base-tests', 'python2.7'),
-                  PYC, 'base_2.7'],
+                  PYC, 'base_2.7', '2.7'],
 }
 
 #-----
@@ -83,23 +91,39 @@ def do_tests(src_dir, obj_patterns, target_dir, opts):
                         if fnmatch(n, pat)])
 
     files = []
+    # Change directories so use relative rather than
+    # absolute paths. This speeds up things, and allows
+    # main() to write to a relative-path destination.
+    cwd = os.getcwd()
+    os.chdir(src_dir)
+
     if opts['do_compile']:
-        for root, dirs, basenames in os.walk(src_dir):
-            file_matches(files, root, basenames, PY)
-            for sfile in files:
-                py_compile.compile(sfile)
+        compiled_version = opts['compiled_version']
+        if compiled_version and PYTHON_VERSION != compiled_version:
+            print("Not compiling: desired Python version is %s but we are running %s" %
+                  (compiled_version, PYTHON_VERSION), file=sys.stderr)
+        else:
+            for root, dirs, basenames in os.walk(src_dir):
+                file_matches(files, root, basenames, PY)
+                for sfile in files:
+                    py_compile.compile(sfile)
+                    pass
                 pass
+            files = []
             pass
-        files = []
+        pass
 
     for root, dirs, basenames in os.walk(src_dir):
-        file_matches(files, root, basenames, obj_patterns)
+        # Turn root into a relative path
+        dirname = root[len(src_dir)+1:]
+        file_matches(files, dirname, basenames, obj_patterns)
 
     if not files:
         print("Didn't come up with any files to test! Try with --compile?",
               file=sys.stderr)
         exit(1)
 
+    os.chdir(cwd)
     files.sort()
 
     if opts['start_with']:
@@ -154,19 +178,26 @@ if __name__ == '__main__':
             pass
         pass
 
-    for src_dir, pattern, target_dir in test_dirs:
+    last_compile_version = None
+    for src_dir, pattern, target_dir, compiled_version in test_dirs:
         if os.path.isdir(src_dir):
             checked_dirs.append([src_dir, pattern, target_dir])
         else:
             print("Can't find directory %s. Skipping" % src_dir,
                   file=sys.stderr)
-            pass
+            continue
+        if last_compile_version and last_compile_version != compile_version:
+            print("Warning: mixed python version decompylation")
+        else:
+            last_compile_version = compiled_version
         pass
 
     if not checked_dirs:
         print("No directories found to check", file=sys.stderr)
         sys.exit(1)
 
+    test_opts['compiled_version'] = last_compile_version
+
     for src_dir, pattern, target_dir in checked_dirs:
         target_dir = os.path.join(target_base, target_dir)
         if os.path.exists(target_dir):

uncompyle6/__init__.py

@@ -32,10 +32,9 @@ from __future__ import print_function
 
 import os, marshal, sys, types
 
-if (sys.version_info > (3, 0)):
-    from . import walker, verify, magics
-else:
-    import walker, verify, magics
+PYTHON_VERSION = sys.version_info.major + (sys.version_info.minor / 10.0)
+
+from uncompyle6 import disas, walker, verify, magics
 
 sys.setrecursionlimit(5000)
 __all__ = ['uncompyle_file', 'main']
@@ -68,20 +67,25 @@ def _load_module(filename):
     code_object: code_object from this file
     '''
 
-    fp = open(filename, 'rb')
-    magic = fp.read(4)
-    try:
-        version = float(magics.versions[magic])
-    except KeyError:
-        raise ImportError("Unknown magic number %s in %s" % (ord(magic[0])+256*ord(magic[1]), filename))
-    if (version > 2.7) or (version < 2.5):
-        raise ImportError("This is a Python %s file! Only Python 2.5 to 2.7 files are supported." % version)
-    # print version
-    fp.read(4) # timestamp
+    with open(filename, 'rb') as fp:
+        magic = fp.read(4)
+        try:
+            version = float(magics.versions[magic])
+        except KeyError:
+            raise ImportError("Unknown magic number %s in %s" % (ord(magic[0])+256*ord(magic[1]), filename))
+        if (version > 2.7) or (version < 2.5):
+            raise ImportError("This is a Python %s file! Only Python 2.5 to 2.7 files are supported." % version)
+
+        # print version
+        fp.read(4) # timestamp
+
+        if version == PYTHON_VERSION:
+            bytecode = fp.read()
+            co = marshal.loads(bytecode)
+        else:
+            co = disas.load(fp)
+        pass
 
-    bytecode = fp.read()
-    co = marshal.loads(bytecode)
-    fp.close()
     return version, co
 
 def uncompyle(version, co, out=None, showasm=0, showast=0):

uncompyle6/disas.py

@@ -0,0 +1,162 @@
+from __future__ import print_function
+
+"""Disassembler of Python byte code into mnemonics.
+
+This is needed when the bytecode extracted is from
+a different version than the currently-running Python.
+
+When the two are the same, you can simply use marshal.loads()
+to prodoce a code object
+"""
+
+import marshal, pickle, sys, types
+
+import dis as Mdis
+
+from struct import unpack
+
+internStrings = []
+
+# XXX For backwards compatibility
+disco = Mdis.disassemble
+
+if (sys.version_info >= (3, 0)):
+    def long(n):
+        return n
+
+def marshalLoad(fp):
+    global internStrings
+    internStrings = []
+    return load(fp)
+
+def load(fp):
+    """
+    Load marshal
+    """
+    global internStrings
+
+    marshalType = fp.read(1).decode('utf-8')
+    if marshalType == 'c':
+        Code = types.CodeType
+
+        co_argcount = unpack('i', fp.read(4))[0]
+        co_nlocals = unpack('i', fp.read(4))[0]
+        co_stacksize = unpack('i', fp.read(4))[0]
+        co_flags = unpack('i', fp.read(4))[0]
+        co_code = load(fp)
+        co_consts = load(fp)
+        co_names = load(fp)
+        co_varnames = load(fp)
+        co_freevars = load(fp)
+        co_cellvars = load(fp)
+        co_filename = load(fp)
+        co_name = load(fp)
+        co_firstlineno = unpack('i', fp.read(4))[0]
+        co_lnotab = load(fp)
+        return Code(co_argcount, co_nlocals, co_stacksize, co_flags, co_code,
+                    co_consts, co_names, co_varnames, co_filename, co_name,
+                    co_firstlineno, co_lnotab, co_freevars, co_cellvars)
+
+    # const type
+    elif marshalType == '.':
+        return Ellipsis
+    elif marshalType == '0':
+        raise KeyError(marshalType)
+        return None
+    elif marshalType == 'N':
+        return None
+    elif marshalType == 'T':
+        return True
+    elif marshalType == 'F':
+        return False
+    elif marshalType == 'S':
+        return StopIteration
+    # number type
+    elif marshalType == 'f':
+        n = fp.read(1)
+        return float(unpack('d', fp.read(n))[0])
+    elif marshalType == 'g':
+        return float(unpack('d', fp.read(8))[0])
+    elif marshalType == 'i':
+        return int(unpack('i', fp.read(4))[0])
+    elif marshalType == 'I':
+        return unpack('q', fp.read(8))[0]
+    elif marshalType == 'x':
+        raise KeyError(marshalType)
+        return None
+    elif marshalType == 'y':
+        raise KeyError(marshalType)
+        return None
+    elif marshalType == 'l':
+        n = unpack('i', fp.read(4))[0]
+        if n == 0:
+            return long(0)
+        size = abs(n)
+        d = long(0)
+        for j in range(0, size):
+            md = int(unpack('h', fp.read(2))[0])
+            d += md << j*15
+        if n < 0:
+            return long(d*-1)
+        return d
+    # strings type
+    elif marshalType == 'R':
+        refnum = unpack('i', fp.read(4))[0]
+        return internStrings[refnum]
+    elif marshalType == 's':
+        strsize = unpack('i', fp.read(4))[0]
+        return str(fp.read(strsize))
+    elif marshalType == 't':
+        strsize = unpack('i', fp.read(4))[0]
+        interned = str(fp.read(strsize))
+        internStrings.append(interned)
+        return interned
+    elif marshalType == 'u':
+        strsize = unpack('i', fp.read(4))[0]
+        unicodestring = fp.read(strsize)
+        return unicodestring.decode('utf-8')
+    # collection type
+    elif marshalType == '(':
+        tuplesize = unpack('i', fp.read(4))[0]
+        ret = tuple()
+        while tuplesize > 0:
+            ret += load(fp),
+            tuplesize -= 1
+        return ret
+    elif marshalType == '[':
+        raise KeyError(marshalType)
+        return None
+    elif marshalType == '{':
+        raise KeyError(marshalType)
+        return None
+    elif marshalType in ['<', '>']:
+        raise KeyError(marshalType)
+        return None
+    else:
+        sys.stderr.write("Unknown type %i (hex %x)\n" % (ord(marshalType), ord(marshalType)))
+
+def _test():
+    """Simple test program to disassemble a file."""
+    if sys.argv[1:]:
+        if sys.argv[2:]:
+            sys.stderr.write("usage: python dis.py [-|file]\n")
+            sys.exit(2)
+        fn = sys.argv[1]
+        if not fn or fn == "-":
+            fn = None
+    else:
+        fn = None
+    if fn is None:
+        f = sys.stdin
+    else:
+        f = open(fn)
+    source = f.read()
+    if fn is not None:
+        f.close()
+    else:
+        fn = "<stdin>"
+    code = compile(source, fn, "exec")
+    Mdis.dis(code)
+
+if __name__ == "__main__":
+    _test()

uncompyle6/dparser.py

@@ -17,7 +17,7 @@ except ImportError:
 
 import string, sys
 
-if (sys.version_info > (3, 0)):
+if (sys.version_info >= (3, 0)):
     intern = sys.intern
     from collections import UserList
 else:

uncompyle6/magics.py

@@ -5,7 +5,7 @@ import struct, sys
 __all__ = ['magics', 'versions']
 
 def __build_magic(magic):
-    if (sys.version_info > (3, 0)):
+    if (sys.version_info >= (3, 0)):
         return struct.pack('Hcc', magic, bytes('\r', 'utf-8'), bytes('\n', 'utf-8'))
     else:
         return struct.pack('Hcc', magic, '\r', '\n')

uncompyle6/verify.py

@@ -11,7 +11,7 @@ import dis, operator, sys, types
 import uncompyle6
 import uncompyle6.scanner as scanner
 
-if (sys.version_info > (3, 0)):
+if (sys.version_info >= (3, 0)):
     truediv = operator.truediv
 else:
     truediv = operator.div

uncompyle6/walker.py

@@ -45,7 +45,7 @@ from __future__ import print_function
 
 import sys, re
 
-if (sys.version_info > (3, 0)):
+if (sys.version_info >= (3, 0)):
     from io import StringIO
     import uncompyle6
     from .spark import GenericASTTraversal