diff --git a/.gitignore b/.gitignore index e4ea5e61..0e6284e3 100644 --- a/.gitignore +++ b/.gitignore @@ -1,6 +1,7 @@ -*_dis *.pyc +*_dis *~ +/.eggs /.python-version /dist /uncompyle6.egg-info diff --git a/__pkginfo__.py b/__pkginfo__.py index 149db19d..e5ad7e78 100644 --- a/__pkginfo__.py +++ b/__pkginfo__.py @@ -30,11 +30,9 @@ modname = 'uncompyle6' packages = ['uncompyle6', 'uncompyle6.opcodes'] py_modules = None short_desc = 'Python byte-code disassembler and source-code converter' -# scripts = ['bin/uncompyle6', 'bin/pydissassemble'] -scripts = ['bin/uncompyle6'] +scripts = ['bin/uncompyle6', 'bin/pydisassemble'] -import os -import os.path, sys +import os.path def get_srcdir(): diff --git a/bin/pydisassemble b/bin/pydisassemble new file mode 100755 index 00000000..2e93e7fb --- /dev/null +++ b/bin/pydisassemble @@ -0,0 +1,77 @@ +#!/usr/bin/env python +# Mode: -*- python -*- +# +# Copyright (c) 2015 by Rocky Bernstein +# +""" +Usage: pydisassemble [OPTIONS]... FILE + +Examples: + pydisassemble foo.pyc + pydisassemble foo.py + pydisassemble -o foo.pydis foo.pyc + pydisassemble -o /tmp foo.pyc + +Options: + -o output decompiled files to this path: + if multiple input files are decompiled, the common prefix + is stripped from these names and the remainder appended to + + --help show this message + +""" + +from __future__ import print_function + +Usage_short = \ +"pydissassemble [--help] [--verify] [--showasm] [--showast] [-o ] FILE|DIR..." + +import sys, os, getopt +import os.path + +from uncompyle6.disas import disassemble_files + +if sys.version[:3] != '2.7' and sys.version[:3] != '3.4': + print('Error: pydisassemble requires Python 2.7 or 3.4.', file=sys.stderr) + sys.exit(-1) + +outfile = '-' +out_base = None + + +try: + opts, files = getopt.getopt(sys.argv[1:], 'ho:', ['help']) +except getopt.GetoptError as e: + print('%s: %s' % (os.path.basename(sys.argv[0]), e), file=sys.stderr) + sys.exit(-1) + +for opt, val in opts: + if opt in ('-h', '--help'): + print(__doc__) + sys.exit(0) + elif opt == '-o': + outfile = val + else: + print(opt) + print(Usage_short) + sys.exit(1) + +# argl, commonprefix works on strings, not on path parts, +# thus we must handle the case with files in 'some/classes' +# and 'some/cmds' +src_base = os.path.commonprefix(files) +if src_base[-1:] != os.sep: + src_base = os.path.dirname(src_base) +if src_base: + sb_len = len( os.path.join(src_base, '') ) + files = [f[sb_len:] for f in files] + del sb_len + +if outfile == '-': + outfile = None # use stdout +elif outfile and os.path.isdir(outfile): + out_base = outfile; outfile = None +elif outfile and len(files) > 1: + out_base = outfile; outfile = None + +disassemble_files(src_base, out_base, files, outfile) diff --git a/bin/pydissassemble b/bin/pydissassemble deleted file mode 100755 index e8ca97a2..00000000 --- a/bin/pydissassemble +++ /dev/null @@ -1,201 +0,0 @@ -#!/usr/bin/env python -# Mode: -*- python -*- -# -# Copyright (c) 2015 by Rocky Bernstein -# -""" -Usage: pydisassemble [OPTIONS]... FILE - -Examples: - pydisassemble foo.pyc - pydisassemble foo.py - pydisassemble -o foo.pydis foo.pyc - pydisassemble -o /tmp foo.pyc - -Options: - -o output decompiled files to this path: - if multiple input files are decompiled, the common prefix - is stripped from these names and the remainder appended to - - --help show this message - -""" - -from __future__ import print_function - -Usage_short = \ -"pydissassemble [--help] [--verify] [--showasm] [--showast] [-o ] FILE|DIR..." - -import sys, os, getopt, time, types -import os.path -import uncompyle6 - -def disassemble_code(version, co, out=None): - """ - diassembles and deparses a given code block 'co' - """ - - assert isinstance(co, types.CodeType) - - # store final output stream for case of error - real_out = out or sys.stdout - print('# Python %s' % version, file=real_out) - if co.co_filename: - print('# Embedded file name: %s' % co.co_filename, - file=real_out) - - # Pick up appropriate scanner - if version == 2.7: - import uncompyle6.scanners.scanner27 as scan - scanner = scan.Scanner27() - elif version == 2.6: - import uncompyle6.scanners.scanner26 as scan - scanner = scan.Scanner26() - elif version == 2.5: - import uncompyle6.scanners.scanner25 as scan - scanner = scan.Scanner25() - elif version == 3.2: - import uncompyle6.scanners.scanner32 as scan - scanner = scan.Scanner32() - elif version == 3.4: - import uncompyle6.scanners.scanner34 as scan - scanner = scan.Scanner34() - scanner.setShowAsm(True, out) - tokens, customize = scanner.disassemble(co) - - for t in tokens: - print(t, file=real_out) - print(file=out) - - - -def disassemble_file(filename, outstream=None, showasm=False, showast=False): - """ - disassemble Python byte-code file (.pyc) - """ - version, co = uncompyle6.load_module(filename) - if type(co) == list: - for con in co: - disassemble_code(version, con, outstream) - else: - from trepan.api import debug; debug - disassemble_code(version, co, outstream) - co = None - -def disassemble_files(in_base, out_base, files, outfile=None): - """ - in_base base directory for input files - out_base base directory for output files (ignored when - files list of filenames to be uncompyled (relative to src_base) - outfile write output to this filename (overwrites out_base) - - For redirecting output to - - outfile= (out_base is ignored) - - files below out_base out_base=... - - stdout out_base=None, outfile=None - """ - def _get_outstream(outfile): - dir = os.path.dirname(outfile) - failed_file = outfile + '_failed' - if os.path.exists(failed_file): - os.remove(failed_file) - try: - os.makedirs(dir) - except OSError: - pass - return open(outfile, 'w') - - of = outfile - if outfile == '-': - outfile = None # use stdout - elif outfile and os.path.isdir(outfile): - out_base = outfile; outfile = None - elif outfile: - out_base = outfile; outfile = None - - - for filename in files: - infile = os.path.join(in_base, filename) - # print (infile, file=sys.stderr) - - if of: # outfile was given as parameter - outstream = _get_outstream(outfile) - elif out_base is None: - outstream = sys.stdout - else: - outfile = os.path.join(out_base, file) + '_dis' - outstream = _get_outstream(outfile) - # print(outfile, file=sys.stderr) - pass - - # try to decomyple the input file - try: - disassemble_file(infile, outstream, showasm=True, showast=False) - except KeyboardInterrupt: - if outfile: - outstream.close() - os.remove(outfile) - raise - except: - if outfile: - outstream.close() - os.rename(outfile, outfile + '_failed') - else: - sys.stderr.write("\n# Can't disassemble %s\n" % infile) - import traceback - traceback.print_exc() - else: # uncompyle successfull - if outfile: - outstream.close() - if not outfile: print('\n# okay decompyling', infile) - sys.stdout.flush() - - if outfile: - sys.stdout.write("\n") - sys.stdout.flush() - return - -if sys.version[:3] != '2.7' and sys.version[:3] != '3.4': - print('Error: pydisassemble requires Python 2.7 or 3.4.', file=sys.stderr) - sys.exit(-1) - -outfile = '-' -out_base = None - - -try: - opts, files = getopt.getopt(sys.argv[1:], 'ho:', ['help']) -except getopt.GetoptError as e: - print('%s: %s' % (os.path.basename(sys.argv[0]), e), file=sys.stderr) - sys.exit(-1) - -for opt, val in opts: - if opt in ('-h', '--help'): - print(__doc__) - sys.exit(0) - elif opt == '-o': - outfile = val - else: - print(opt) - print(Usage_short) - sys.exit(1) - -# argl, commonprefix works on strings, not on path parts, -# thus we must handle the case with files in 'some/classes' -# and 'some/cmds' -src_base = os.path.commonprefix(files) -if src_base[-1:] != os.sep: - src_base = os.path.dirname(src_base) -if src_base: - sb_len = len( os.path.join(src_base, '') ) - files = [f[sb_len:] for f in files] - del sb_len - -if outfile == '-': - outfile = None # use stdout -elif outfile and os.path.isdir(outfile): - out_base = outfile; outfile = None -elif outfile and len(files) > 1: - out_base = outfile; outfile = None - -disassemble_files(src_base, out_base, files, outfile) diff --git a/test/test_pythonlib.py b/test/test_pythonlib.py index 8f0b8df4..8e05e942 100755 --- a/test/test_pythonlib.py +++ b/test/test_pythonlib.py @@ -1,9 +1,7 @@ #!/usr/bin/env python # emacs-mode: -*-python-*- -from __future__ import print_function - -''' +""" test_pythonlib.py -- compile, uncompyle, and verify Python libraries Usage-Examples: @@ -27,11 +25,13 @@ Step 1) Edit this file and add a new entry to 'test_options', eg. Step 2: Run the test: test_pythonlib.py --mylib # decompile 'mylib' test_pythonlib.py --mylib --verify # decompile verify 'mylib' -''' +""" + +from __future__ import print_function import getopt, os, py_compile, sys, shutil, tempfile, time -from uncompyle6 import main, verify, PYTHON_VERSION +from uncompyle6 import main, PYTHON_VERSION from fnmatch import fnmatch def get_srcdir(): diff --git a/uncompyle6/__init__.py b/uncompyle6/__init__.py index 0fe86601..c3ee6a13 100644 --- a/uncompyle6/__init__.py +++ b/uncompyle6/__init__.py @@ -4,6 +4,7 @@ from __future__ import print_function ''' Copyright (c) 1999 John Aycock Copyright (c) 2000 by hartmut Goebel + Copyright (c) 2015 by Rocky Bernstein Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the @@ -33,8 +34,10 @@ from __future__ import print_function import os, marshal, sys, types PYTHON_VERSION = sys.version_info.major + (sys.version_info.minor / 10.0) +PYTHON3 = (sys.version_info >= (3, 0)) -from uncompyle6 import disas, walker, verify, magics +import uncompyle6 +from uncompyle6 import walker, verify, magics sys.setrecursionlimit(5000) __all__ = ['uncompyle_file', 'main'] @@ -95,7 +98,7 @@ def load_module(filename): bytecode = fp.read() co = marshal.loads(bytecode) else: - co = disas.load(fp, magic_int) + co = uncompyle6.marsh.load_code(fp, magic_int) pass return version, co @@ -284,7 +287,7 @@ def main(in_base, out_base, files, codes, outfile=None, verify_failed_files += 1 os.rename(outfile, outfile + '_unverified') if not outfile: - print("### Error Verifiying %s" % file, file=sys.stderr) + print("### Error Verifiying %s" % filename, file=sys.stderr) print(e, file=sys.stderr) else: okay_files += 1 diff --git a/uncompyle6/deparser.py b/uncompyle6/deparser.py index b37ff896..d21f944b 100644 --- a/uncompyle6/deparser.py +++ b/uncompyle6/deparser.py @@ -1,4 +1,6 @@ -''' +""" + Deparsing saving text fragment information indexed by offset + Copyright (c) 1999 John Aycock Copyright (c) 2000-2002 by hartmut Goebel Copyright (c) 2005 by Dan Pascu @@ -6,7 +8,6 @@ See main module for license. - Decompilation (walking AST) All table-driven. (rocky: well, mostly. I need to add more format @@ -44,7 +45,7 @@ The '%' may optionally be followed by a number (C) in square brackets, which makes the engine walk down to N[C] before evaluating the escape code. -''' +""" from __future__ import print_function diff --git a/uncompyle6/disas.py b/uncompyle6/disas.py index 74528b62..b033d25c 100644 --- a/uncompyle6/disas.py +++ b/uncompyle6/disas.py @@ -1,201 +1,153 @@ -from __future__ import print_function +""" +CPython magic- and version- independent disassembly routines -"""Disassembler of Python byte code into mnemonics. +Copyright (c) 1999 John Aycock +Copyright (c) 2000-2002 by hartmut Goebel +Copyright (c) 2005 by Dan Pascu +Copyright (c) 2015 by Rocky Bernstein This is needed when the bytecode extracted is from a different version than the currently-running Python. -When the two are the same, you can simply use marshal.loads() -to prodoce a code object +When the two are the same, you can simply use Python's built-in disassemble """ -import imp, marshal, pickle, sys, types -import dis as Mdis +from __future__ import print_function -from struct import unpack +import os, sys, types import uncompyle6 -from uncompyle6.magics import magic2int -internStrings = [] - -# XXX For backwards compatibility -disco = Mdis.disassemble - -PYTHON3 = (sys.version_info >= (3, 0)) -PYTHON_MAGIC_INT = magic2int(imp.get_magic()) - -if PYTHON3: - def long(n): return n - -def compat_str(s): - return s.decode('utf-8', errors='ignore') if PYTHON3 else str(s) - -def marshalLoad(fp): - global internStrings - internStrings = [] - return load(fp) - -def load(fp, magic_int): +def disco(version, co, out=None): """ - marshal.load() written in Python. When the Python bytecode magic loaded is the - same magic for the running Python interpreter, we can simply use the - Python-supplied mashal.load(). - - However we need to use this when versions are different since the internal - code structures are different. Sigh. + diassembles and deparses a given code block 'co' """ - global internStrings - marshalType = fp.read(1).decode('utf-8') - if marshalType == 'c': - Code = types.CodeType + assert isinstance(co, types.CodeType) - # FIXME If 'i' is deprecated, what would we use? - co_argcount = unpack('i', fp.read(4))[0] - co_nlocals = unpack('i', fp.read(4))[0] - co_stacksize = unpack('i', fp.read(4))[0] - co_flags = unpack('i', fp.read(4))[0] - # FIXME: somewhere between Python 2.7 and python 3.2 there's - # another 4 bytes before we get to the bytecode. What's going on? - # Again, because magic ints decreased between python 2.7 and 3.0 we need - # a range here. - if 3000 < magic_int < 20121: - fp.read(4) - co_code = load(fp, magic_int) - co_consts = load(fp, magic_int) - co_names = load(fp, magic_int) - co_varnames = load(fp, magic_int) - co_freevars = load(fp, magic_int) - co_cellvars = load(fp, magic_int) - co_filename = load(fp, magic_int) - co_name = load(fp, magic_int) - co_firstlineno = unpack('i', fp.read(4))[0] - co_lnotab = load(fp, magic_int) - # The Python3 code object is different than Python2's which - # we are reading if we get here. - # Also various parameters which were strings are now - # bytes (which is probably more logical). - if PYTHON3: - if PYTHON_MAGIC_INT > 3020: - # In later Python3 magic_ints, there is a - # kwonlyargcount parameter which we set to 0. - return Code(co_argcount, 0, co_nlocals, co_stacksize, co_flags, - bytes(co_code, encoding='utf-8'), - co_consts, co_names, co_varnames, co_filename, co_name, - co_firstlineno, bytes(co_lnotab, encoding='utf-8'), - co_freevars, co_cellvars) - else: - return Code(co_argcount, 0, co_nlocals, co_stacksize, co_flags, - bytes(co_code, encoding='utf-8'), - co_consts, co_names, co_varnames, co_filename, co_name, - co_firstlineno, bytes(co_lnotab, encoding='utf-8'), - co_freevars, co_cellvars) - else: - return Code(co_argcount, co_nlocals, co_stacksize, co_flags, co_code, - co_consts, co_names, co_varnames, co_filename, co_name, - co_firstlineno, co_lnotab, co_freevars, co_cellvars) + # store final output stream for case of error + real_out = out or sys.stdout + print('# Python %s' % version, file=real_out) + if co.co_filename: + print('# Embedded file name: %s' % co.co_filename, + file=real_out) - # const type - elif marshalType == '.': - return Ellipsis - elif marshalType == '0': - raise KeyError(marshalType) - return None - elif marshalType == 'N': - return None - elif marshalType == 'T': - return True - elif marshalType == 'F': - return False - elif marshalType == 'S': - return StopIteration - # number type - elif marshalType == 'f': - n = fp.read(1) - return float(unpack('d', fp.read(n))[0]) - elif marshalType == 'g': - return float(unpack('d', fp.read(8))[0]) - elif marshalType == 'i': - return int(unpack('i', fp.read(4))[0]) - elif marshalType == 'I': - return unpack('q', fp.read(8))[0] - elif marshalType == 'x': - raise KeyError(marshalType) - return None - elif marshalType == 'y': - raise KeyError(marshalType) - return None - elif marshalType == 'l': - n = unpack('i', fp.read(4))[0] - if n == 0: - return long(0) - size = abs(n) - d = long(0) - for j in range(0, size): - md = int(unpack('h', fp.read(2))[0]) - d += md << j*15 - if n < 0: - return long(d*-1) - return d - # strings type - elif marshalType == 'R': - refnum = unpack('i', fp.read(4))[0] - return internStrings[refnum] - elif marshalType == 's': - strsize = unpack('i', fp.read(4))[0] - return compat_str(fp.read(strsize)) - elif marshalType == 't': - strsize = unpack('i', fp.read(4))[0] - interned = compat_str(fp.read(strsize)) - internStrings.append(interned) - return interned - elif marshalType == 'u': - strsize = unpack('i', fp.read(4))[0] - unicodestring = fp.read(strsize) - return unicodestring.decode('utf-8') - # collection type - elif marshalType == '(': - tuplesize = unpack('i', fp.read(4))[0] - ret = tuple() - while tuplesize > 0: - ret += load(fp, magic_int), - tuplesize -= 1 - return ret - elif marshalType == '[': - raise KeyError(marshalType) - return None - elif marshalType == '{': - raise KeyError(marshalType) - return None - elif marshalType in ['<', '>']: - raise KeyError(marshalType) - return None + # Pick up appropriate scanner + if version == 2.7: + import uncompyle6.scanners.scanner27 as scan + scanner = scan.Scanner27() + elif version == 2.6: + import uncompyle6.scanners.scanner26 as scan + scanner = scan.Scanner26() + elif version == 2.5: + import uncompyle6.scanners.scanner25 as scan + scanner = scan.Scanner25() + elif version == 3.2: + import uncompyle6.scanners.scanner32 as scan + scanner = scan.Scanner32() + elif version == 3.4: + import uncompyle6.scanners.scanner34 as scan + scanner = scan.Scanner34() + scanner.setShowAsm(True, out) + tokens, customize = scanner.disassemble(co) + + for t in tokens: + print(t, file=real_out) + print(file=out) + + +def disassemble_file(filename, outstream=None): + """ + disassemble Python byte-code file (.pyc) + """ + version, co = uncompyle6.load_module(filename) + if type(co) == list: + for con in co: + disco(version, con, outstream) else: - sys.stderr.write("Unknown type %i (hex %x)\n" % (ord(marshalType), ord(marshalType))) + disco(version, co, outstream) + co = None + +def disassemble_files(in_base, out_base, files, outfile=None): + """ + in_base base directory for input files + out_base base directory for output files (ignored when + files list of filenames to be uncompyled (relative to src_base) + outfile write output to this filename (overwrites out_base) + + For redirecting output to + - outfile= (out_base is ignored) + - files below out_base out_base=... + - stdout out_base=None, outfile=None + """ + def _get_outstream(outfile): + dir = os.path.dirname(outfile) + failed_file = outfile + '_failed' + if os.path.exists(failed_file): + os.remove(failed_file) + try: + os.makedirs(dir) + except OSError: + pass + return open(outfile, 'w') + + of = outfile + if outfile == '-': + outfile = None # use stdout + elif outfile and os.path.isdir(outfile): + out_base = outfile; outfile = None + elif outfile: + out_base = outfile; outfile = None + + for filename in files: + infile = os.path.join(in_base, filename) + # print (infile, file=sys.stderr) + + if of: # outfile was given as parameter + outstream = _get_outstream(outfile) + elif out_base is None: + outstream = sys.stdout + else: + outfile = os.path.join(out_base, filename) + '_dis' + outstream = _get_outstream(outfile) + # print(outfile, file=sys.stderr) + pass + + # try to decomyple the input file + try: + disassemble_file(infile, outstream) + except KeyboardInterrupt: + if outfile: + outstream.close() + os.remove(outfile) + raise + except: + if outfile: + outstream.close() + os.rename(outfile, outfile + '_failed') + else: + sys.stderr.write("\n# Can't disassemble %s\n" % infile) + import traceback + traceback.print_exc() + else: # uncompyle successfull + if outfile: + outstream.close() + if not outfile: print('\n# okay decompyling', infile) + sys.stdout.flush() + + if outfile: + sys.stdout.write("\n") + sys.stdout.flush() + return def _test(): """Simple test program to disassemble a file.""" - if sys.argv[1:]: - if sys.argv[2:]: - sys.stderr.write("usage: python dis.py [-|file]\n") - sys.exit(2) - fn = sys.argv[1] - if not fn or fn == "-": - fn = None - else: - fn = None - if fn is None: - f = sys.stdin - else: - f = open(fn) - source = f.read() - if fn is not None: - f.close() - else: - fn = "" - code = compile(source, fn, "exec") - Mdis.dis(code) + argc = len(sys.argv) + if argc != 2: + sys.stderr.write("usage: %s [-|CPython compiled file]\n" % __file__) + sys.exit(2) + fn = sys.argv[1] + disassemble_file(fn) if __name__ == "__main__": _test() diff --git a/uncompyle6/magics.py b/uncompyle6/magics.py index 37aec100..74002af8 100755 --- a/uncompyle6/magics.py +++ b/uncompyle6/magics.py @@ -2,7 +2,7 @@ from __future__ import print_function import struct, sys -__all__ = ['magics', 'versions', 'magics2int'] +__all__ = ['magics', 'versions', 'magic2int'] def __build_magic(magic): if (sys.version_info >= (3, 0)): @@ -10,7 +10,7 @@ def __build_magic(magic): else: return struct.pack('Hcc', magic, '\r', '\n') -def magic2int (magic): +def magic2int(magic): return struct.unpack('Hcc', magic)[0] by_magic = {} @@ -73,20 +73,20 @@ versions = { __build_magic(3160): '3.2', # 3.2a0 (add SETUP_WITH) __build_magic(3170): '3.2', # 3.2a1 (add DUP_TOP_TWO, remove DUP_TOPX and ROT_FOUR) __build_magic(3180): '3.2', # 3.2a2 (add DELETE_DEREF) - __build_magic(3190): '3.3', # Python 3.3a0 3190 __class__ super closure changed - __build_magic(3200): '3.3', # Python 3.3a0 3200 (__qualname__ added) - __build_magic(3200): '3.3', # 3210 (added size modulo 2**32 to the pyc header) - __build_magic(3220): '3.3', # Python 3.3a1 3220 (changed PEP 380 implementation) - __build_magic(3230): '3.3', # Python 3.3a4 3230 (revert changes to implicit __class__ closure) - __build_magic(3250): '3.4', # Python 3.4a1 3250 (evaluate positional default arg - # keyword-only defaults) - __build_magic(3260): '3.4', # Python 3.4a1 3260 (add LOAD_CLASSDEREF; allow locals of class to override - # free vars) - __build_magic(3270): '3.4', # Python 3.4a1 3270 (various tweaks to the __class__ closure) - __build_magic(3280): '3.4', # Python 3.4a1 3280 (remove implicit class argument) - __build_magic(3290): '3.4', # Python 3.4a4 3290 (changes to __qualname__ computation) - __build_magic(3300): '3.4', # Python 3.4a4 3300 (more changes to __qualname__ computation) - __build_magic(3310): '3.4', # Python 3.4rc2 3310 (alter __qualname__ computation) + __build_magic(3190): '3.3', # 3.3a0 3190 __class__ super closure changed + __build_magic(3100): '3.3', # 3.3a0 3200 (__qualname__ added) + __build_magic(3210): '3.3', # 3210 (added size modulo 2**32 to the pyc header) + __build_magic(3220): '3.3', # 3.3a1 3220 (changed PEP 380 implementation) + __build_magic(3230): '3.3', # 3.3a4 3230 (revert changes to implicit __class__ closure) + __build_magic(3250): '3.4', # 3.4a1 3250 (evaluate positional default arg + # keyword-only defaults) + __build_magic(3260): '3.4', # 3.4a1 3260 (add LOAD_CLASSDEREF; + # allow locals of class to override free vars) + __build_magic(3270): '3.4', # 3.4a1 3270 (various tweaks to the __class__ closure) + __build_magic(3280): '3.4', # 3.4a1 3280 (remove implicit class argument) + __build_magic(3290): '3.4', # 3.4a4 3290 (changes to __qualname__ computation) + __build_magic(3300): '3.4', # 3.4a4 3300 (more changes to __qualname__ computation) + __build_magic(3310): '3.4', # 3.4rc2 3310 (alter __qualname__ computation) } diff --git a/uncompyle6/marsh.py b/uncompyle6/marsh.py new file mode 100644 index 00000000..0f6b1141 --- /dev/null +++ b/uncompyle6/marsh.py @@ -0,0 +1,176 @@ +""" +CPython magic- and version- independent marshal routines + + Copyright (c) 1999 John Aycock + Copyright (c) 2000-2002 by hartmut Goebel + Copyright (c) 2005 by Dan Pascu + Copyright (c) 2015 by Rocky Bernstein + +This is needed when the bytecode extracted is from +a different version than the currently-running Python. + +When the two are the same, you can simply use Python's built-in marshal.loads() +to produce a code object +""" + +from __future__ import print_function + +import imp, sys, types +from struct import unpack + +from uncompyle6.magics import magic2int + +internStrings = [] + +PYTHON_MAGIC_INT = magic2int(imp.get_magic()) +PYTHON3 = (sys.version_info >= (3, 0)) + +if PYTHON3: + def long(n): return n + +def compat_str(s): + return s.decode('utf-8', errors='ignore') if PYTHON3 else str(s) + +def load_code(fp, magic_int): + """ + marshal.load() written in Python. When the Python bytecode magic loaded is the + same magic for the running Python interpreter, we can simply use the + Python-supplied mashal.load(). + + However we need to use this when versions are different since the internal + code structures are different. Sigh. + """ + global internStrings + internStrings = [] + return load_code_internal(fp, magic_int) + +def load_code_internal(fp, magic_int): + global internStrings + + marshalType = fp.read(1).decode('utf-8') + if marshalType == 'c': + Code = types.CodeType + + # FIXME If 'i' is deprecated, what would we use? + co_argcount = unpack('i', fp.read(4))[0] + co_nlocals = unpack('i', fp.read(4))[0] + co_stacksize = unpack('i', fp.read(4))[0] + co_flags = unpack('i', fp.read(4))[0] + # FIXME: somewhere between Python 2.7 and python 3.2 there's + # another 4 bytes before we get to the bytecode. What's going on? + # Again, because magic ints decreased between python 2.7 and 3.0 we need + # a range here. + if 3000 < magic_int < 20121: + fp.read(4) + co_code = load_code(fp, magic_int) + co_consts = load_code(fp, magic_int) + co_names = load_code(fp, magic_int) + co_varnames = load_code(fp, magic_int) + co_freevars = load_code(fp, magic_int) + co_cellvars = load_code(fp, magic_int) + co_filename = load_code(fp, magic_int) + co_name = load_code(fp, magic_int) + co_firstlineno = unpack('i', fp.read(4))[0] + co_lnotab = load_code(fp, magic_int) + # The Python3 code object is different than Python2's which + # we are reading if we get here. + # Also various parameters which were strings are now + # bytes (which is probably more logical). + if PYTHON3: + if PYTHON_MAGIC_INT > 3020: + # In later Python3 magic_ints, there is a + # kwonlyargcount parameter which we set to 0. + return Code(co_argcount, 0, co_nlocals, co_stacksize, co_flags, + bytes(co_code, encoding='utf-8'), + co_consts, co_names, co_varnames, co_filename, co_name, + co_firstlineno, bytes(co_lnotab, encoding='utf-8'), + co_freevars, co_cellvars) + else: + return Code(co_argcount, 0, co_nlocals, co_stacksize, co_flags, + bytes(co_code, encoding='utf-8'), + co_consts, co_names, co_varnames, co_filename, co_name, + co_firstlineno, bytes(co_lnotab, encoding='utf-8'), + co_freevars, co_cellvars) + else: + return Code(co_argcount, co_nlocals, co_stacksize, co_flags, co_code, + co_consts, co_names, co_varnames, co_filename, co_name, + co_firstlineno, co_lnotab, co_freevars, co_cellvars) + + # const type + elif marshalType == '.': + return Ellipsis + elif marshalType == '0': + raise KeyError(marshalType) + return None + elif marshalType == 'N': + return None + elif marshalType == 'T': + return True + elif marshalType == 'F': + return False + elif marshalType == 'S': + return StopIteration + # number type + elif marshalType == 'f': + n = fp.read(1) + return float(unpack('d', fp.read(n))[0]) + elif marshalType == 'g': + return float(unpack('d', fp.read(8))[0]) + elif marshalType == 'i': + return int(unpack('i', fp.read(4))[0]) + elif marshalType == 'I': + return unpack('q', fp.read(8))[0] + elif marshalType == 'x': + raise KeyError(marshalType) + return None + elif marshalType == 'y': + raise KeyError(marshalType) + return None + elif marshalType == 'l': + n = unpack('i', fp.read(4))[0] + if n == 0: + return long(0) + size = abs(n) + d = long(0) + for j in range(0, size): + md = int(unpack('h', fp.read(2))[0]) + d += md << j*15 + if n < 0: + return long(d*-1) + return d + # strings type + elif marshalType == 'R': + refnum = unpack('i', fp.read(4))[0] + return internStrings[refnum] + elif marshalType == 's': + strsize = unpack('i', fp.read(4))[0] + return compat_str(fp.read(strsize)) + elif marshalType == 't': + strsize = unpack('i', fp.read(4))[0] + interned = compat_str(fp.read(strsize)) + internStrings.append(interned) + return interned + elif marshalType == 'u': + strsize = unpack('i', fp.read(4))[0] + unicodestring = fp.read(strsize) + return unicodestring.decode('utf-8') + # collection type + elif marshalType == '(': + tuplesize = unpack('i', fp.read(4))[0] + ret = tuple() + while tuplesize > 0: + ret += load_code(fp, magic_int), + tuplesize -= 1 + return ret + elif marshalType == '[': + raise KeyError(marshalType) + return None + elif marshalType == '{': + raise KeyError(marshalType) + return None + elif marshalType in ['<', '>']: + raise KeyError(marshalType) + return None + else: + sys.stderr.write("Unknown type %i (hex %x)\n" % (ord(marshalType), ord(marshalType))) + return diff --git a/uncompyle6/scanners/scanner25.py b/uncompyle6/scanners/scanner25.py index d26e4b20..3bf218f1 100644 --- a/uncompyle6/scanners/scanner25.py +++ b/uncompyle6/scanners/scanner25.py @@ -1,16 +1,21 @@ -''' +""" + Python 2.5 bytecode scanner/deparser + Copyright (c) 1999 John Aycock Copyright (c) 2000-2002 by hartmut Goebel Copyright (c) 2005 by Dan Pascu + Copyright (c) 2015 by Rocky Bernstein See main module for license. -''' + +This overlaps Python's 2.5's dis module, but it can be run from +Python3 and other versions of Python. Also, we save token information +for later use in deparsing. +""" import types from collections import namedtuple from array import array -from operator import itemgetter -from struct import * import dis from uncompyle6.opcodes.opcode_25 import * diff --git a/uncompyle6/scanners/scanner26.py b/uncompyle6/scanners/scanner26.py index ada5d0e9..a40e6f4e 100644 --- a/uncompyle6/scanners/scanner26.py +++ b/uncompyle6/scanners/scanner26.py @@ -1,15 +1,21 @@ -''' +""" + Python 2.6 bytecode scanner + Copyright (c) 1999 John Aycock Copyright (c) 2000-2002 by hartmut Goebel Copyright (c) 2005 by Dan Pascu + Copyright (c) 2015 by Rocky Bernstein See main module for license. -''' + +This overlaps Python's 2.6's dis module, but it can be run from Python3 and +other versions of Python. Also, we save token information for later +use in deparsing. +""" import types from collections import namedtuple from array import array -from operator import itemgetter from uncompyle6.opcodes.opcode_26 import * import dis diff --git a/uncompyle6/scanners/scanner27.py b/uncompyle6/scanners/scanner27.py index a6aff588..d2eee055 100644 --- a/uncompyle6/scanners/scanner27.py +++ b/uncompyle6/scanners/scanner27.py @@ -1,17 +1,23 @@ -from __future__ import print_function +""" + Python 2.5 bytecode scanner/deparser -''' Copyright (c) 1999 John Aycock Copyright (c) 2000-2002 by hartmut Goebel Copyright (c) 2005 by Dan Pascu + Copyright (c) 2015 by Rocky Bernstein See main module for license. -''' + +This overlaps Python's 2.7's dis module, but it can be run from +Python3 and other versions of Python. Also, we save token information +for later use in deparsing. +""" + +from __future__ import print_function import dis, types from collections import namedtuple from array import array -from operator import itemgetter from uncompyle6.opcodes.opcode_27 import * import uncompyle6.scanner as scan diff --git a/uncompyle6/scanners/scanner32.py b/uncompyle6/scanners/scanner32.py index d6108848..bf1c4210 100644 --- a/uncompyle6/scanners/scanner32.py +++ b/uncompyle6/scanners/scanner32.py @@ -1,12 +1,19 @@ -from __future__ import print_function +""" + Python 3.2 bytecode scanner/deparser -''' Copyright (c) 1999 John Aycock Copyright (c) 2000-2002 by hartmut Goebel Copyright (c) 2005 by Dan Pascu + Copyright (c) 2015 by Rocky Bernstein See main module for license. -''' + +This overlaps Python's 3.2's dis module, but it can be run from +Python3 and other versions of Python. Also, we save token information +for later use in deparsing. +""" + +from __future__ import print_function import dis, marshal from collections import namedtuple diff --git a/uncompyle6/scanners/scanner34.py b/uncompyle6/scanners/scanner34.py index 70dfe627..f5c7fd3c 100644 --- a/uncompyle6/scanners/scanner34.py +++ b/uncompyle6/scanners/scanner34.py @@ -1,12 +1,15 @@ -from __future__ import print_function +""" + Python 2.5 bytecode scanner/deparser -''' Copyright (c) 1999 John Aycock Copyright (c) 2000-2002 by hartmut Goebel Copyright (c) 2005 by Dan Pascu + Copyright (c) 2015 by Rocky Bernstein See main module for license. -''' +""" + +from __future__ import print_function import dis, marshal from collections import namedtuple