diff --git a/MANIFEST.in b/MANIFEST.in index 13042ba8..20e43223 100644 --- a/MANIFEST.in +++ b/MANIFEST.in @@ -1,5 +1,5 @@ include README.rst include __pkginfo__.py -recursive-include uncompyle6 +recursive-include uncompyle6 *.py include script/uncompyle6 -recursive-include/test/* +recursive-include test *.py *.pyc *.pyo diff --git a/test/test_pythonlib.py b/test/test_pythonlib.py index 2659da15..85b571d9 100755 --- a/test/test_pythonlib.py +++ b/test/test_pythonlib.py @@ -4,14 +4,21 @@ from __future__ import print_function ''' -test_pythonlib.py -- uncompyle and verify Python libraries +test_pythonlib.py -- compile, uncompyle, and verify Python libraries Usage-Examples: - test_pythonlib.py --all # decompile all tests (suite + libs) - test_pythonlib.py --all --verify # decomyile all tests and verify results - test_pythonlib.py --test # decompile only the testsuite - test_pythonlib.py --2.2 --verify # decompile and verify python lib 2.2 + # decompile, and verify base set of python 2.7 byte-compiled files + test_pythonlib.py --base-2.7 --verify + + # Same as above but compile the base set first + test_pythonlib.py --base-2.7 --verify --compile + + # Same as above but use a longer set from the python 2.7 library + test_pythonlib.py --ok-2.7 --verify --compile + + # Just deompile the longer set of files + test_pythonlib.py --ok-2.7 Adding own test-trees: @@ -24,7 +31,7 @@ Step 2: Run the test: import getopt, os, py_compile, sys, shutil, tempfile, time -from uncompyle6 import main, verify +from uncompyle6 import main, verify, PYTHON_VERSION from fnmatch import fnmatch def get_srcdir(): @@ -33,6 +40,7 @@ def get_srcdir(): src_dir = get_srcdir() + #----- configure this for your needs lib_prefix = [src_dir, '/usr/lib/', '/usr/local/lib/'] @@ -45,15 +53,15 @@ PYO = ('*.pyo', ) PYOC = ('*.pyc', '*.pyo') test_options = { - # name: (src_basedir, pattern, output_base_suffix) + # name: (src_basedir, pattern, output_base_suffix, pythoin_version) 'test': ['test', PYC, 'test'], - '2.7': ['python2.7', PYC, 'python2.7'], + '2.7': ['python2.7', PYC, 'python2.7', '2.7'], 'ok-2.6': [os.path.join(src_dir, 'ok_2.6'), - PYC, 'ok-2.6'], + PYC, 'ok-2.6', '2.6'], 'ok-2.7': [os.path.join(src_dir, 'ok_2.7'), - PYC, 'ok-2.7'], + PYC, 'ok-2.7', '2.7'], 'base-2.7': [os.path.join(src_dir, 'base-tests', 'python2.7'), - PYC, 'base_2.7'], + PYC, 'base_2.7', '2.7'], } #----- @@ -83,23 +91,39 @@ def do_tests(src_dir, obj_patterns, target_dir, opts): if fnmatch(n, pat)]) files = [] + # Change directories so use relative rather than + # absolute paths. This speeds up things, and allows + # main() to write to a relative-path destination. + cwd = os.getcwd() + os.chdir(src_dir) + if opts['do_compile']: - for root, dirs, basenames in os.walk(src_dir): - file_matches(files, root, basenames, PY) - for sfile in files: - py_compile.compile(sfile) + compiled_version = opts['compiled_version'] + if compiled_version and PYTHON_VERSION != compiled_version: + print("Not compiling: desired Python version is %s but we are running %s" % + (compiled_version, PYTHON_VERSION), file=sys.stderr) + else: + for root, dirs, basenames in os.walk(src_dir): + file_matches(files, root, basenames, PY) + for sfile in files: + py_compile.compile(sfile) + pass pass + files = [] pass - files = [] + pass for root, dirs, basenames in os.walk(src_dir): - file_matches(files, root, basenames, obj_patterns) + # Turn root into a relative path + dirname = root[len(src_dir)+1:] + file_matches(files, dirname, basenames, obj_patterns) if not files: print("Didn't come up with any files to test! Try with --compile?", file=sys.stderr) exit(1) + os.chdir(cwd) files.sort() if opts['start_with']: @@ -154,19 +178,26 @@ if __name__ == '__main__': pass pass - for src_dir, pattern, target_dir in test_dirs: + last_compile_version = None + for src_dir, pattern, target_dir, compiled_version in test_dirs: if os.path.isdir(src_dir): checked_dirs.append([src_dir, pattern, target_dir]) else: print("Can't find directory %s. Skipping" % src_dir, file=sys.stderr) - pass + continue + if last_compile_version and last_compile_version != compile_version: + print("Warning: mixed python version decompylation") + else: + last_compile_version = compiled_version pass if not checked_dirs: print("No directories found to check", file=sys.stderr) sys.exit(1) + test_opts['compiled_version'] = last_compile_version + for src_dir, pattern, target_dir in checked_dirs: target_dir = os.path.join(target_base, target_dir) if os.path.exists(target_dir): diff --git a/uncompyle6/__init__.py b/uncompyle6/__init__.py index 5688c3b0..e537a4bb 100644 --- a/uncompyle6/__init__.py +++ b/uncompyle6/__init__.py @@ -32,10 +32,9 @@ from __future__ import print_function import os, marshal, sys, types -if (sys.version_info > (3, 0)): - from . import walker, verify, magics -else: - import walker, verify, magics +PYTHON_VERSION = sys.version_info.major + (sys.version_info.minor / 10.0) + +from uncompyle6 import disas, walker, verify, magics sys.setrecursionlimit(5000) __all__ = ['uncompyle_file', 'main'] @@ -68,20 +67,25 @@ def _load_module(filename): code_object: code_object from this file ''' - fp = open(filename, 'rb') - magic = fp.read(4) - try: - version = float(magics.versions[magic]) - except KeyError: - raise ImportError("Unknown magic number %s in %s" % (ord(magic[0])+256*ord(magic[1]), filename)) - if (version > 2.7) or (version < 2.5): - raise ImportError("This is a Python %s file! Only Python 2.5 to 2.7 files are supported." % version) - # print version - fp.read(4) # timestamp + with open(filename, 'rb') as fp: + magic = fp.read(4) + try: + version = float(magics.versions[magic]) + except KeyError: + raise ImportError("Unknown magic number %s in %s" % (ord(magic[0])+256*ord(magic[1]), filename)) + if (version > 2.7) or (version < 2.5): + raise ImportError("This is a Python %s file! Only Python 2.5 to 2.7 files are supported." % version) + + # print version + fp.read(4) # timestamp + + if version == PYTHON_VERSION: + bytecode = fp.read() + co = marshal.loads(bytecode) + else: + co = disas.load(fp) + pass - bytecode = fp.read() - co = marshal.loads(bytecode) - fp.close() return version, co def uncompyle(version, co, out=None, showasm=0, showast=0): diff --git a/uncompyle6/disas.py b/uncompyle6/disas.py new file mode 100644 index 00000000..56db4ec7 --- /dev/null +++ b/uncompyle6/disas.py @@ -0,0 +1,162 @@ +from __future__ import print_function + +"""Disassembler of Python byte code into mnemonics. + +This is needed when the bytecode extracted is from +a different version than the currently-running Python. + +When the two are the same, you can simply use marshal.loads() +to prodoce a code object +""" + +import marshal, pickle, sys, types + +import dis as Mdis + +from struct import unpack + +internStrings = [] + +# XXX For backwards compatibility +disco = Mdis.disassemble + +if (sys.version_info >= (3, 0)): + def long(n): + return n + +def marshalLoad(fp): + global internStrings + internStrings = [] + return load(fp) + +def load(fp): + """ + Load marshal + """ + global internStrings + + marshalType = fp.read(1).decode('utf-8') + if marshalType == 'c': + Code = types.CodeType + + co_argcount = unpack('i', fp.read(4))[0] + co_nlocals = unpack('i', fp.read(4))[0] + co_stacksize = unpack('i', fp.read(4))[0] + co_flags = unpack('i', fp.read(4))[0] + co_code = load(fp) + co_consts = load(fp) + co_names = load(fp) + co_varnames = load(fp) + co_freevars = load(fp) + co_cellvars = load(fp) + co_filename = load(fp) + co_name = load(fp) + co_firstlineno = unpack('i', fp.read(4))[0] + co_lnotab = load(fp) + return Code(co_argcount, co_nlocals, co_stacksize, co_flags, co_code, + co_consts, co_names, co_varnames, co_filename, co_name, + co_firstlineno, co_lnotab, co_freevars, co_cellvars) + + # const type + elif marshalType == '.': + return Ellipsis + elif marshalType == '0': + raise KeyError(marshalType) + return None + elif marshalType == 'N': + return None + elif marshalType == 'T': + return True + elif marshalType == 'F': + return False + elif marshalType == 'S': + return StopIteration + # number type + elif marshalType == 'f': + n = fp.read(1) + return float(unpack('d', fp.read(n))[0]) + elif marshalType == 'g': + return float(unpack('d', fp.read(8))[0]) + elif marshalType == 'i': + return int(unpack('i', fp.read(4))[0]) + elif marshalType == 'I': + return unpack('q', fp.read(8))[0] + elif marshalType == 'x': + raise KeyError(marshalType) + return None + elif marshalType == 'y': + raise KeyError(marshalType) + return None + elif marshalType == 'l': + n = unpack('i', fp.read(4))[0] + if n == 0: + return long(0) + size = abs(n) + d = long(0) + for j in range(0, size): + md = int(unpack('h', fp.read(2))[0]) + d += md << j*15 + if n < 0: + return long(d*-1) + return d + # strings type + elif marshalType == 'R': + refnum = unpack('i', fp.read(4))[0] + return internStrings[refnum] + elif marshalType == 's': + strsize = unpack('i', fp.read(4))[0] + return str(fp.read(strsize)) + elif marshalType == 't': + strsize = unpack('i', fp.read(4))[0] + interned = str(fp.read(strsize)) + internStrings.append(interned) + return interned + elif marshalType == 'u': + strsize = unpack('i', fp.read(4))[0] + unicodestring = fp.read(strsize) + return unicodestring.decode('utf-8') + # collection type + elif marshalType == '(': + tuplesize = unpack('i', fp.read(4))[0] + ret = tuple() + while tuplesize > 0: + ret += load(fp), + tuplesize -= 1 + return ret + elif marshalType == '[': + raise KeyError(marshalType) + return None + elif marshalType == '{': + raise KeyError(marshalType) + return None + elif marshalType in ['<', '>']: + raise KeyError(marshalType) + return None + else: + sys.stderr.write("Unknown type %i (hex %x)\n" % (ord(marshalType), ord(marshalType))) + +def _test(): + """Simple test program to disassemble a file.""" + if sys.argv[1:]: + if sys.argv[2:]: + sys.stderr.write("usage: python dis.py [-|file]\n") + sys.exit(2) + fn = sys.argv[1] + if not fn or fn == "-": + fn = None + else: + fn = None + if fn is None: + f = sys.stdin + else: + f = open(fn) + source = f.read() + if fn is not None: + f.close() + else: + fn = "" + code = compile(source, fn, "exec") + Mdis.dis(code) + +if __name__ == "__main__": + _test() diff --git a/uncompyle6/dparser.py b/uncompyle6/dparser.py index 9903a94e..aa6b1e43 100644 --- a/uncompyle6/dparser.py +++ b/uncompyle6/dparser.py @@ -17,7 +17,7 @@ except ImportError: import string, sys -if (sys.version_info > (3, 0)): +if (sys.version_info >= (3, 0)): intern = sys.intern from collections import UserList else: diff --git a/uncompyle6/magics.py b/uncompyle6/magics.py index 88696d95..0c369660 100755 --- a/uncompyle6/magics.py +++ b/uncompyle6/magics.py @@ -5,7 +5,7 @@ import struct, sys __all__ = ['magics', 'versions'] def __build_magic(magic): - if (sys.version_info > (3, 0)): + if (sys.version_info >= (3, 0)): return struct.pack('Hcc', magic, bytes('\r', 'utf-8'), bytes('\n', 'utf-8')) else: return struct.pack('Hcc', magic, '\r', '\n') diff --git a/uncompyle6/verify.py b/uncompyle6/verify.py index 6000b676..d2a72220 100755 --- a/uncompyle6/verify.py +++ b/uncompyle6/verify.py @@ -11,7 +11,7 @@ import dis, operator, sys, types import uncompyle6 import uncompyle6.scanner as scanner -if (sys.version_info > (3, 0)): +if (sys.version_info >= (3, 0)): truediv = operator.truediv else: truediv = operator.div diff --git a/uncompyle6/walker.py b/uncompyle6/walker.py index 856b6b30..625757e5 100644 --- a/uncompyle6/walker.py +++ b/uncompyle6/walker.py @@ -45,7 +45,7 @@ from __future__ import print_function import sys, re -if (sys.version_info > (3, 0)): +if (sys.version_info >= (3, 0)): from io import StringIO import uncompyle6 from .spark import GenericASTTraversal