diff --git a/__pkginfo__.py b/__pkginfo__.py index 345b05f0..d5e2a1c1 100644 --- a/__pkginfo__.py +++ b/__pkginfo__.py @@ -37,7 +37,7 @@ entry_points={ ]} ftp_url = None install_requires = ['spark-parser >= 1.4.0', - 'xdis >= 1.1.7'] + 'xdis >= 1.2.0'] license = 'MIT' mailing_list = 'python-debugger@googlegroups.com' modname = 'uncompyle6' diff --git a/test/bytecode_pypy_2.6.1/00_assign.pyc b/test/bytecode_pypy_2.6.1/00_assign.pyc new file mode 100644 index 00000000..99378bca Binary files /dev/null and b/test/bytecode_pypy_2.6.1/00_assign.pyc differ diff --git a/test/bytecode_pypy_2.6.1/00_import.pyc b/test/bytecode_pypy_2.6.1/00_import.pyc new file mode 100644 index 00000000..98a1e207 Binary files /dev/null and b/test/bytecode_pypy_2.6.1/00_import.pyc differ diff --git a/test/bytecode_pypy_2.6.1/00_pass.pyc b/test/bytecode_pypy_2.6.1/00_pass.pyc new file mode 100644 index 00000000..4331f639 Binary files /dev/null and b/test/bytecode_pypy_2.6.1/00_pass.pyc differ diff --git a/test/bytecode_pypy_2.6.1/01_boolean.pyc b/test/bytecode_pypy_2.6.1/01_boolean.pyc new file mode 100644 index 00000000..5779abcd Binary files /dev/null and b/test/bytecode_pypy_2.6.1/01_boolean.pyc differ diff --git a/test/bytecode_pypy_2.6.1/01_class.pyc b/test/bytecode_pypy_2.6.1/01_class.pyc new file mode 100644 index 00000000..63b72486 Binary files /dev/null and b/test/bytecode_pypy_2.6.1/01_class.pyc differ diff --git a/test/bytecode_pypy_2.6.1/01_list_comprehension.pyc b/test/bytecode_pypy_2.6.1/01_list_comprehension.pyc new file mode 100644 index 00000000..6aceb4d6 Binary files /dev/null and b/test/bytecode_pypy_2.6.1/01_list_comprehension.pyc differ diff --git a/test/bytecode_pypy_2.6.1/01_positional.pyc b/test/bytecode_pypy_2.6.1/01_positional.pyc new file mode 100644 index 00000000..717cf4d3 Binary files /dev/null and b/test/bytecode_pypy_2.6.1/01_positional.pyc differ diff --git a/test/bytecode_pypy_2.6.1/01_try_except.pyc b/test/bytecode_pypy_2.6.1/01_try_except.pyc new file mode 100644 index 00000000..67bfd4c2 Binary files /dev/null and b/test/bytecode_pypy_2.6.1/01_try_except.pyc differ diff --git a/test/test_pyenvlib.py b/test/test_pyenvlib.py index c9c247dd..72ea73ad 100755 --- a/test/test_pyenvlib.py +++ b/test/test_pyenvlib.py @@ -27,7 +27,8 @@ from fnmatch import fnmatch #----- configure this for your needs -TEST_VERSIONS=('2.3.7', '2.4.6', '2.5.6', '2.6.9', '2.7.10', '2.7.11', +TEST_VERSIONS=('2.3.7', '2.4.6', '2.5.6', '2.6.9', 'pypy-2.6.1', + '2.7.10', '2.7.11', '3.2.6', '3.3.5', '3.4.2', '3.5.1') target_base = '/tmp/py-dis/' @@ -45,9 +46,14 @@ test_options = { } for vers in TEST_VERSIONS: - short_vers = vers[:3] - test_options[vers] = (os.path.join(lib_prefix, vers, 'lib', 'python'+short_vers), - PYC, 'python-lib'+short_vers) + if vers.startswith('pypy-'): + short_vers = vers[0:-2] + test_options[vers] = (os.path.join(lib_prefix, vers, 'lib_pypy'), + PYC, 'python-lib'+short_vers) + else: + short_vers = vers[:3] + test_options[vers] = (os.path.join(lib_prefix, vers, 'lib', 'python'+short_vers), + PYC, 'python-lib'+short_vers) def do_tests(src_dir, patterns, target_dir, start_with=None, do_verify=False): diff --git a/uncompyle6/disas.py b/uncompyle6/disas.py index 549567fb..97bffbeb 100644 --- a/uncompyle6/disas.py +++ b/uncompyle6/disas.py @@ -77,7 +77,7 @@ def disassemble_file(filename, outstream=None, native=False): return filename = check_object_path(filename) - version, timestamp, magic_int, co = load_module(filename) + version, timestamp, magic_int, co, is_pypy = load_module(filename) if type(co) == list: for con in co: disco(version, con, outstream) diff --git a/uncompyle6/main.py b/uncompyle6/main.py index 4fe41a4b..0cffb34e 100644 --- a/uncompyle6/main.py +++ b/uncompyle6/main.py @@ -1,7 +1,7 @@ from __future__ import print_function import datetime, os, sys -from uncompyle6 import verify, PYTHON_VERSION +from uncompyle6 import verify, PYTHON_VERSION, IS_PYPY from xdis.code import iscode from uncompyle6.disas import check_object_path from uncompyle6.semantics import pysource @@ -9,8 +9,10 @@ from uncompyle6.parser import ParserError from xdis.load import load_module -def uncompyle(version, co, out=None, showasm=False, showast=False, - timestamp=None, showgrammar=False, code_objects={}): +def uncompyle( + version, co, out=None, showasm=False, showast=False, + timestamp=None, showgrammar=False, code_objects={}, + is_pypy=False): """ disassembles and deparses a given code block 'co' """ @@ -19,7 +21,10 @@ def uncompyle(version, co, out=None, showasm=False, showast=False, # store final output stream for case of error real_out = out or sys.stdout - print('# Python bytecode %s (decompiled from Python %s)' % (version, PYTHON_VERSION), + co_pypy_str = 'PyPy ' if is_pypy else '' + run_pypy_str = 'PyPy ' if IS_PYPY else '' + print('# %sPython bytecode %s (disassembled from %sPython %s)\n' % + (co_pypy_str, version, run_pypy_str, PYTHON_VERSION), file=real_out) if co.co_filename: print('# Embedded file name: %s' % co.co_filename, @@ -30,7 +35,7 @@ def uncompyle(version, co, out=None, showasm=False, showast=False, try: pysource.deparse_code(version, co, out, showasm, showast, showgrammar, - code_objects=code_objects) + code_objects=code_objects, is_pypy=is_pypy) except pysource.SourceWalkerError as e: # deparsing failed print("\n") @@ -49,16 +54,18 @@ def uncompyle_file(filename, outstream=None, showasm=False, showast=False, filename = check_object_path(filename) code_objects = {} - version, timestamp, magic_int, co = load_module(filename, code_objects) + version, timestamp, magic_int, co, is_pypy = load_module(filename, code_objects) if type(co) == list: for con in co: uncompyle(version, con, outstream, showasm, showast, - timestamp, showgrammar, code_objects=code_objects) + timestamp, showgrammar, code_objects=code_objects, + is_pypy=is_pypy) else: uncompyle(version, co, outstream, showasm, showast, - timestamp, showgrammar, code_objects=code_objects) + timestamp, showgrammar, code_objects=code_objects, + is_pypy=is_pypy) co = None # FIXME: combine into an options parameter diff --git a/uncompyle6/parser.py b/uncompyle6/parser.py index 0c6ea93e..bafc121c 100644 --- a/uncompyle6/parser.py +++ b/uncompyle6/parser.py @@ -570,7 +570,9 @@ def parse(p, tokens, customize): return ast -def get_python_parser(version, debug_parser, compile_mode='exec'): +def get_python_parser( + version, debug_parser, compile_mode='exec', + is_pypy = False): """Returns parser object for Python version 2 or 3, 3.2, 3.5on, etc., depending on the parameters passed. *compile_mode* is either 'exec', 'eval', or 'single'. See @@ -662,7 +664,7 @@ class PythonParserSingle(PythonParser): def python_parser(version, co, out=sys.stdout, showasm=False, - parser_debug=PARSER_DEFAULT_DEBUG): + parser_debug=PARSER_DEFAULT_DEBUG, is_pypy=False): """ Parse a code object to an abstract syntax tree representation. @@ -681,7 +683,7 @@ def python_parser(version, co, out=sys.stdout, showasm=False, assert iscode(co) from uncompyle6.scanner import get_scanner - scanner = get_scanner(version) + scanner = get_scanner(version, is_pypy) tokens, customize = scanner.disassemble(co) maybe_show_asm(showasm, tokens) @@ -693,8 +695,8 @@ def python_parser(version, co, out=sys.stdout, showasm=False, if __name__ == '__main__': def parse_test(co): - from uncompyle6 import PYTHON_VERSION - ast = python_parser(PYTHON_VERSION, co, showasm=True) + from uncompyle6 import PYTHON_VERSION, IS_PYPY + ast = python_parser(PYTHON_VERSION, co, showasm=True, is_pypy=IS_PYPY) print(ast) return parse_test(parse_test.__code__) diff --git a/uncompyle6/parsers/parse27.py b/uncompyle6/parsers/parse27.py index 49c4edb2..aeede81e 100644 --- a/uncompyle6/parsers/parse27.py +++ b/uncompyle6/parsers/parse27.py @@ -15,6 +15,8 @@ class Python27Parser(Python2Parser): def p_list_comprehension27(self, args): """ list_for ::= expr _for designator list_iter JUMP_BACK + + list_compr ::= expr BUILD_LIST_FROM_ARG _for designator list_iter JUMP_BACK """ def p_try27(self, args): diff --git a/uncompyle6/scanner.py b/uncompyle6/scanner.py index f69afc1b..e936abf3 100755 --- a/uncompyle6/scanner.py +++ b/uncompyle6/scanner.py @@ -48,12 +48,15 @@ class Code(object): class Scanner(object): - def __init__(self, version, show_asm=None): + def __init__(self, version, show_asm=None, is_pypy=False): self.version = version self.show_asm = show_asm if version in PYTHON_VERSIONS: - v_str = "opcode_%s" % (int(version * 10)) + if is_pypy: + v_str = "opcode_pypy%s" % (int(version * 10)) + else: + v_str = "opcode_%s" % (int(version * 10)) exec("from xdis.opcodes import %s" % v_str) exec("self.opc = %s" % v_str) else: @@ -251,18 +254,26 @@ class Scanner(object): def parse_fn_counts(argc): return ((argc & 0xFF), (argc >> 8) & 0xFF, (argc >> 16) & 0x7FFF) -def get_scanner(version, show_asm=None): +def get_scanner(version, show_asm=None, is_pypy=False): # Pick up appropriate scanner if version in PYTHON_VERSIONS: v_str = "%s" % (int(version * 10)) - exec("import uncompyle6.scanners.scanner%s as scan" % v_str) if PYTHON3: import importlib - scan = importlib.import_module("uncompyle6.scanners.scanner%s" % v_str) + if is_pypy: + scan = importlib.import_module("uncompyle6.scanners.pypy%s" % v_str) + else: + scan = importlib.import_module("uncompyle6.scanners.scanner%s" % v_str) if False: print(scan) # Avoid unused scan else: - exec("import uncompyle6.scanners.scanner%s as scan" % v_str) - scanner = eval("scan.Scanner%s(show_asm=show_asm)" % v_str) + if is_pypy: + exec("import uncompyle6.scanners.pypy%s as scan" % v_str) + else: + exec("import uncompyle6.scanners.scanner%s as scan" % v_str) + if is_pypy: + scanner = eval("scan.ScannerPyPy%s(show_asm=show_asm)" % v_str) + else: + scanner = eval("scan.Scanner%s(show_asm=show_asm)" % v_str) else: raise RuntimeError("Unsupported Python version %s" % version) return scanner diff --git a/uncompyle6/scanners/scanner2.py b/uncompyle6/scanners/scanner2.py index 6b49dba2..c1610c88 100755 --- a/uncompyle6/scanners/scanner2.py +++ b/uncompyle6/scanners/scanner2.py @@ -32,8 +32,8 @@ from xdis.bytecode import findlinestarts import uncompyle6.scanner as scan class Scanner2(scan.Scanner): - def __init__(self, version, show_asm=None): - scan.Scanner.__init__(self, version, show_asm) + def __init__(self, version, show_asm=None, is_pypy=False): + scan.Scanner.__init__(self, version, show_asm, is_pypy) self.pop_jump_if = frozenset([self.opc.PJIF, self.opc.PJIT]) self.jump_forward = frozenset([self.opc.JUMP_ABSOLUTE, self.opc.JUMP_FORWARD]) # This is the 2.5+ default diff --git a/uncompyle6/scanners/scanner27.py b/uncompyle6/scanners/scanner27.py index 2c529f49..1f1d943e 100755 --- a/uncompyle6/scanners/scanner27.py +++ b/uncompyle6/scanners/scanner27.py @@ -17,8 +17,8 @@ from xdis.opcodes import opcode_27 JUMP_OPs = opcode_27.JUMP_OPs class Scanner27(Scanner2): - def __init__(self, show_asm=False): - super(Scanner27, self).__init__(2.7, show_asm) + def __init__(self, show_asm=False, is_pypy=False): + super(Scanner27, self).__init__(2.7, show_asm, is_pypy) # opcodes that start statements self.stmt_opcodes = frozenset([ diff --git a/uncompyle6/semantics/fragments.py b/uncompyle6/semantics/fragments.py index 8fb2e441..3a6042b5 100644 --- a/uncompyle6/semantics/fragments.py +++ b/uncompyle6/semantics/fragments.py @@ -118,7 +118,8 @@ class FragmentsWalker(pysource.SourceWalker, object): stacked_params = ('f', 'indent', 'isLambda', '_globals') def __init__(self, version, scanner, showast=False, - debug_parser=PARSER_DEFAULT_DEBUG): + debug_parser=PARSER_DEFAULT_DEBUG, + compile_mode='exec', is_pypy=False): GenericASTTraversal.__init__(self, ast=None) self.scanner = scanner params = { @@ -126,7 +127,10 @@ class FragmentsWalker(pysource.SourceWalker, object): 'indent': '', } self.version = version - self.p = get_python_parser(version, dict(debug_parser)) + self.p = get_python_parser( + version, dict(debug_parser), + compile_mode=compile_mode, is_pypy=is_pypy + ) self.showast = showast self.params = params self.param_stack = [] diff --git a/uncompyle6/semantics/pysource.py b/uncompyle6/semantics/pysource.py index 38f7a2d0..46f37db1 100644 --- a/uncompyle6/semantics/pysource.py +++ b/uncompyle6/semantics/pysource.py @@ -526,7 +526,7 @@ class SourceWalker(GenericASTTraversal, object): def __init__(self, version, out, scanner, showast=False, debug_parser=PARSER_DEFAULT_DEBUG, - compile_mode='exec'): + compile_mode='exec', is_pypy=False): GenericASTTraversal.__init__(self, ast=None) self.scanner = scanner params = { @@ -555,6 +555,7 @@ class SourceWalker(GenericASTTraversal, object): self.hide_internal = True self.name = None self.version = version + self.is_pypy = is_pypy if 2.0 <= version <= 2.3: TABLE_DIRECT['tryfinallystmt'] = ( @@ -1022,6 +1023,9 @@ class SourceWalker(GenericASTTraversal, object): p = self.prec self.prec = 27 if self.version >= 2.7: + if self.is_pypy: + self.n_list_compr_pypy27(node) + return n = node[-1] elif node[-1] == 'del_stmt': n = node[-3] if node[-2] == 'JUMP_BACK' else node[-2] @@ -1053,6 +1057,42 @@ class SourceWalker(GenericASTTraversal, object): self.prec = p self.prune() # stop recursing + def n_list_compr_pypy27(self, node): + """List comprehensions the way they are done in PYPY Python 2.7. + """ + p = self.prec + self.prec = 27 + n = node[-2] if self.is_pypy and node[-1] == 'JUMP_BACK' else node[-1] + list_expr = node[0] + designator = node[3] + + assert n == 'list_iter' + assert designator == 'designator' + + # find innermost node + while n == 'list_iter': + n = n[0] # recurse one step + if n == 'list_for': n = n[3] + elif n == 'list_if': n = n[2] + elif n == 'list_if_not': n= n[2] + assert n == 'lc_body' + self.write( '[ ') + + expr = n[0] + list_iter = node[-2] if self.is_pypy and node[-1] == 'JUMP_BACK' else node[-1] + + assert expr == 'expr' + assert list_iter == 'list_iter' + + self.preorder(expr) + self.write( ' for ') + self.preorder(designator) + self.write( ' in ') + self.preorder(list_expr) + self.write( ' ]') + self.prec = p + self.prune() # stop recursing + def comprehension_walk(self, node, iter_index, code_index=-5): p = self.prec self.prec = 27 @@ -2127,14 +2167,14 @@ class SourceWalker(GenericASTTraversal, object): def deparse_code(version, co, out=sys.stdout, showasm=False, showast=False, - showgrammar=False, code_objects={}, compile_mode='exec'): + showgrammar=False, code_objects={}, compile_mode='exec', is_pypy=False): """ disassembles and deparses a given code block 'co' """ assert iscode(co) # store final output stream for case of error - scanner = get_scanner(version) + scanner = get_scanner(version, is_pypy=is_pypy) tokens, customize = scanner.disassemble(co, code_objects=code_objects) maybe_show_asm(showasm, tokens) @@ -2146,7 +2186,8 @@ def deparse_code(version, co, out=sys.stdout, showasm=False, showast=False, # Build AST from disassembly. deparsed = SourceWalker(version, out, scanner, showast=showast, - debug_parser=debug_parser, compile_mode=compile_mode) + debug_parser=debug_parser, compile_mode=compile_mode, + is_pypy = is_pypy) isTopLevel = co.co_name == '' deparsed.ast = deparsed.build_ast(tokens, customize, isTopLevel=isTopLevel) diff --git a/uncompyle6/verify.py b/uncompyle6/verify.py index 21e90f01..726a0a05 100755 --- a/uncompyle6/verify.py +++ b/uncompyle6/verify.py @@ -364,7 +364,7 @@ class Token(scanner.Token): def compare_code_with_srcfile(pyc_filename, src_filename): """Compare a .pyc with a source code file.""" - version, timestamp, magic_int, code_obj1 = load_module(pyc_filename) + version, timestamp, magic_int, code_obj1, is_pypy = load_module(pyc_filename) if magic_int != PYTHON_MAGIC_INT: msg = ("Can't compare code - Python is running with magic %s, but code is magic %s " % (PYTHON_MAGIC_INT, magic_int)) @@ -375,8 +375,8 @@ def compare_code_with_srcfile(pyc_filename, src_filename): def compare_files(pyc_filename1, pyc_filename2): """Compare two .pyc files.""" - version, timestamp, magic_int1, code_obj1 = uncompyle6.load_module(pyc_filename1) - version, timestamp, magic_int2, code_obj2 = uncompyle6.load_module(pyc_filename2) + version, timestamp, magic_int1, code_obj1, is_pypy = uncompyle6.load_module(pyc_filename1) + version, timestamp, magic_int2, code_obj2, is_pypy = uncompyle6.load_module(pyc_filename2) cmp_code_objects(version, code_obj1, code_obj2) if __name__ == '__main__':