diff --git a/NEWS b/NEWS index 605ab19d..a661249b 100644 --- a/NEWS +++ b/NEWS @@ -1,3 +1,8 @@ +uncompyle6 3.2.3 2018-06-04 Michael Cohen flips and Fleetwood Redux + +- Python 1.3 support 3.0 bug and +- fix botched parameter ordering of 3.x in last release + uncompyle6 3.2.2 2018-06-04 When I'm 64 - Python 3.0 support and bug fixes diff --git a/README.rst b/README.rst index f852bfb9..51d0b544 100644 --- a/README.rst +++ b/README.rst @@ -11,8 +11,9 @@ Introduction ------------ *uncompyle6* translates Python bytecode back into equivalent Python -source code. It accepts bytecodes from Python version 1.5, and 2.1 to -3.7 or so, including PyPy bytecode and Dropbox's Python 2.5 bytecode. +source code. It accepts bytecodes from Python version 1.3 to version +3.7, spanning over 22 years of Python releases. We include Dropbox's +Python 2.5 bytecode and some PyPy bytecode. Why this? --------- @@ -75,7 +76,7 @@ Requirements The code here can be run on Python versions 2.6 or later, PyPy 3-2.4, or PyPy-5.0.1. Python versions 2.4-2.7 are supported in the python-2.4 branch. The bytecode files it can read have been tested on -Python bytecodes from versions 1.5, 2.1-2.7, and 3.0-3.6 and the +Python bytecodes from versions 1.4, 2.1-2.7, and 3.0-3.6 and the above-mentioned PyPy versions. Installation @@ -217,7 +218,7 @@ See Also * https://github.com/zrax/pycdc : purports to support all versions of Python. It is written in C++ and is most accurate for Python versions around 2.7 and 3.3 when the code was more actively developed. Accuracy for more recent versions of Python 3 and early versions of Python are especially lacking. See its `issue tracker `_ for details. Currently lightly maintained. * https://code.google.com/archive/p/unpyc3/ : supports Python 3.2 only. The above projects use a different decompiling technique than what is used here. Currently unmaintained. * https://github.com/figment/unpyc3/ : fork of above, but supports Python 3.3 only. Includes some fixes like supporting function annotations. Currently unmaintained. -* https://github.com/wibiti/uncompyle2 : supports Python 2.7 only, but does that fairly well. Because of it specificity it can sometimes to better than uncompyle6 which we can't do withouth breaking other 2.7 cases. Currently lightly maintained. See its issue `tracker `_ for more details +* https://github.com/wibiti/uncompyle2 : supports Python 2.7 only, but does that fairly well. There situtations where `uncompyle6` results are incorrect while `uncompyle2` results are not, but more often uncompyle6 is correct when uncompyle2 is not. Because `uncompyle6` adheres to accuracy over idiomatic Python, `uncompyle2` can produce more natural-looking code when it is correct. Currently `uncompyle2` is lightly maintained. See its issue `tracker `_ for more details * `How to report a bug `_ * The HISTORY_ file. * https://github.com/rocky/python-xdis : Cross Python version disassembler diff --git a/test/Makefile b/test/Makefile index a75cb15b..facf76ad 100644 --- a/test/Makefile +++ b/test/Makefile @@ -22,7 +22,7 @@ COVER_DIR=../tmp/grammar-cover # Run short tests check-short: @$(PYTHON) -V && PYTHON_VERSION=`$(PYTHON) -V 2>&1 | cut -d ' ' -f 2 | cut -d'.' -f1,2`; \ - $(MAKE) check-bytecode + $(MAKE) check-bytecode-short # Run all tests check: @@ -91,16 +91,31 @@ check-bytecode-3: --bytecode-3.1 --bytecode-3.2 --bytecode-3.3 \ --bytecode-3.4 --bytecode-3.5 --bytecode-3.6 --bytecode-pypy3.2 -#: Check deparsing bytecode that works running Python 2 and Python 3 +#: Check deparsing on selected bytecode 3.x +check-bytecode-3-short: + $(PYTHON) test_pythonlib.py \ + --bytecode-3.4 --bytecode-3.5 --bytecode-3.6 + +#: Check deparsing bytecode on all Python 2 and Python 3 versions check-bytecode: check-bytecode-3 $(PYTHON) test_pythonlib.py \ - --bytecode-1.4 --bytecode-1.5 \ + --bytecode-1.3 --bytecode-1.4 --bytecode-1.5 \ --bytecode-2.2 --bytecode-2.3 --bytecode-2.4 \ --bytecode-2.1 --bytecode-2.2 --bytecode-2.3 --bytecode-2.4 \ --bytecode-2.5 --bytecode-2.6 --bytecode-2.7 \ --bytecode-pypy2.7 +#: Check deparsing bytecode on selected Python 2 and Python 3 versions +check-bytecode-short: check-bytecode-3-short + $(PYTHON) test_pythonlib.py \ + --bytecode-2.6 --bytecode-2.7 --bytecode-pypy2.7 + + +#: Check deparsing bytecode 1.3 only +check-bytecode-1.3: + $(PYTHON) test_pythonlib.py --bytecode-1.3 + #: Check deparsing bytecode 1.4 only check-bytecode-1.4: $(PYTHON) test_pythonlib.py --bytecode-1.4 @@ -210,6 +225,11 @@ check-bytecode-2.7: check-bytecode-3.0: $(PYTHON) test_pythonlib.py --bytecode-3.0 --weak-verify +#: Check deparsing Python 3.0 +check-bytecode-3.0: + $(PYTHON) test_pythonlib.py --bytecode-3.0 --weak-verify + # $(PYTHON) test_pythonlib.py --bytecode-3.0-run --verify-run + #: Check deparsing Python 3.1 check-bytecode-3.1: $(PYTHON) test_pythonlib.py --bytecode-3.1 --weak-verify diff --git a/test/bytecode_1.3/test_builtin.pyc b/test/bytecode_1.3/test_builtin.pyc new file mode 100644 index 00000000..9b0d4f0e Binary files /dev/null and b/test/bytecode_1.3/test_builtin.pyc differ diff --git a/test/bytecode_1.3/test_exceptions.pyc b/test/bytecode_1.3/test_exceptions.pyc new file mode 100644 index 00000000..2a808316 Binary files /dev/null and b/test/bytecode_1.3/test_exceptions.pyc differ diff --git a/test/bytecode_1.3/test_grammar.pyc b/test/bytecode_1.3/test_grammar.pyc new file mode 100644 index 00000000..bb1e6eae Binary files /dev/null and b/test/bytecode_1.3/test_grammar.pyc differ diff --git a/test/bytecode_1.3/test_operations.pyc b/test/bytecode_1.3/test_operations.pyc new file mode 100644 index 00000000..2d7c63a5 Binary files /dev/null and b/test/bytecode_1.3/test_operations.pyc differ diff --git a/test/bytecode_1.3/testall.pyc b/test/bytecode_1.3/testall.pyc new file mode 100644 index 00000000..0cce0629 Binary files /dev/null and b/test/bytecode_1.3/testall.pyc differ diff --git a/test/test_pythonlib.py b/test/test_pythonlib.py index b5a03912..e891afb0 100755 --- a/test/test_pythonlib.py +++ b/test/test_pythonlib.py @@ -76,7 +76,7 @@ for vers in (2.7, 3.4, 3.5, 3.6): test_options[key] = (os.path.join(src_dir, pythonlib), PYOC, key, vers) pass -for vers in (1.4, 1.5, +for vers in (1.3, 1.4, 1.5, 2.1, 2.2, 2.3, 2.4, 2.5, 2.6, 2.7, 3.0, 3.1, 3.2, 3.3, 3.4, 3.5, 3.6, 3.7, 'pypy3.2', 'pypy2.7'): diff --git a/uncompyle6/parser.py b/uncompyle6/parser.py index 16422d35..baa179b7 100644 --- a/uncompyle6/parser.py +++ b/uncompyle6/parser.py @@ -619,7 +619,13 @@ def get_python_parser( if version < 3.0: if version < 2.2: - if version == 1.4: + if version == 1.3: + import uncompyle6.parsers.parse13 as parse13 + if compile_mode == 'exec': + p = parse13.Python14Parser(debug_parser) + else: + p = parse13.Python14ParserSingle(debug_parser) + elif version == 1.4: import uncompyle6.parsers.parse14 as parse14 if compile_mode == 'exec': p = parse14.Python14Parser(debug_parser) diff --git a/uncompyle6/parsers/parse13.py b/uncompyle6/parsers/parse13.py new file mode 100644 index 00000000..593805ee --- /dev/null +++ b/uncompyle6/parsers/parse13.py @@ -0,0 +1,49 @@ +# Copyright (c) 2018 Rocky Bernstein + +from spark_parser import DEFAULT_DEBUG as PARSER_DEFAULT_DEBUG +from uncompyle6.parser import PythonParserSingle +from uncompyle6.parsers.parse14 import Python14Parser + +class Python13Parser(Python14Parser): + + def p_misc13(self, args): + """ + # Nothing here yet, but will need to add LOAD_GLOBALS + """ + + def __init__(self, debug_parser=PARSER_DEFAULT_DEBUG): + super(Python13Parser, self).__init__(debug_parser) + self.customized = {} + + # def customize_grammar_rules(self, tokens, customize): + # super(Python13Parser, self).customize_grammar_rules(tokens, customize) + # self.remove_rules(""" + # whileelsestmt ::= SETUP_LOOP testexpr l_stmts_opt + # jb_pop + # POP_BLOCK else_suitel COME_FROM + # """) + # self.check_reduce['doc_junk'] = 'tokens' + + + # def reduce_is_invalid(self, rule, ast, tokens, first, last): + # invalid = super(Python14Parser, + # self).reduce_is_invalid(rule, ast, + # tokens, first, last) + # if invalid or tokens is None: + # return invalid + # if rule[0] == 'doc_junk': + # return not isinstance(tokens[first].pattr, str) + + + +class Python13ParserSingle(Python13Parser, PythonParserSingle): + pass + +if __name__ == '__main__': + # Check grammar + p = Python13Parser() + p.check_grammar() + p.dump_grammar() + +# local variables: +# tab-width: 4 diff --git a/uncompyle6/parsers/parse14.py b/uncompyle6/parsers/parse14.py index 55c856f3..250f37f4 100644 --- a/uncompyle6/parsers/parse14.py +++ b/uncompyle6/parsers/parse14.py @@ -8,12 +8,11 @@ class Python14Parser(Python15Parser): def p_misc14(self, args): """ - # Nothing here yet, but will need to add UNARY_CALL, BINARY_CALL, + # Not much here yet, but will probably need to add UNARY_CALL, BINARY_CALL, # RAISE_EXCEPTION, BUILD_FUNCTION, UNPACK_ARG, UNPACK_VARARG, LOAD_LOCAL, # SET_FUNC_ARGS, and RESERVE_FAST - # FIXME: should check that this indeed around __doc__ - # Possibly not strictly needed + # Not strictly needed, but tidies up output stmt ::= doc_junk doc_junk ::= LOAD_CONST POP_TOP diff --git a/uncompyle6/parsers/parse30.py b/uncompyle6/parsers/parse30.py index 18adca08..60fa0a9f 100644 --- a/uncompyle6/parsers/parse30.py +++ b/uncompyle6/parsers/parse30.py @@ -73,6 +73,8 @@ class Python30Parser(Python31Parser): for_block ::= l_stmts_opt _come_froms POP_TOP JUMP_BACK except_handler ::= JUMP_FORWARD COME_FROM_EXCEPT except_stmts POP_TOP END_FINALLY come_froms + except_handler ::= jmp_abs COME_FROM_EXCEPT except_stmts + POP_TOP END_FINALLY return_if_stmt ::= ret_expr RETURN_END_IF POP_TOP and ::= expr JUMP_IF_FALSE POP_TOP expr COME_FROM whilestmt ::= SETUP_LOOP testexpr l_stmts_opt @@ -93,6 +95,8 @@ class Python30Parser(Python31Parser): assert ::= assert_expr jmp_true LOAD_ASSERT RAISE_VARARGS_1 return_if_lambda ::= RETURN_END_IF_LAMBDA compare_chained1 ::= expr DUP_TOP ROT_THREE COMPARE_OP JUMP_IF_FALSE_OR_POP compare_chained2 COME_FROM + except_handler ::= jmp_abs COME_FROM_EXCEPT except_stmts END_FINALLY + """) return diff --git a/uncompyle6/scanner.py b/uncompyle6/scanner.py index 38c919c8..860816fd 100755 --- a/uncompyle6/scanner.py +++ b/uncompyle6/scanner.py @@ -39,7 +39,7 @@ else: # The byte code versions we support. # Note: these all have to be floats -PYTHON_VERSIONS = frozenset((1.4, 1.5, +PYTHON_VERSIONS = frozenset((1.3, 1.4, 1.5, 2.1, 2.2, 2.3, 2.4, 2.5, 2.6, 2.7, 3.0, 3.1, 3.2, 3.3, 3.4, 3.5, 3.6, 3.7)) diff --git a/uncompyle6/scanners/scanner13.py b/uncompyle6/scanners/scanner13.py new file mode 100644 index 00000000..611199dd --- /dev/null +++ b/uncompyle6/scanners/scanner13.py @@ -0,0 +1,35 @@ +# Copyright (c) 2018 by Rocky Bernstein +""" +Python 1.3 bytecode decompiler massaging. + +This massages tokenized 1.3 bytecode to make it more amenable for +grammar parsing. +""" + +import uncompyle6.scanners.scanner14 as scan +# from uncompyle6.scanners.scanner26 import ingest as ingest26 + +# bytecode verification, verify(), uses JUMP_OPs from here +from xdis.opcodes import opcode_13 +JUMP_OPS = opcode_13.JUMP_OPS + +# We base this off of 1.4 instead of the other way around +# because we cleaned things up this way. +# The history is that 2.7 support is the cleanest, +# then from that we got 2.6 and so on. +class Scanner13(scan.Scanner14): + def __init__(self, show_asm=False): + scan.Scanner14.__init__(self, show_asm) + self.opc = opcode_13 + self.opname = opcode_13.opname + self.version = 1.3 + return + + # def ingest22(self, co, classname=None, code_objects={}, show_asm=None): + # tokens, customize = self.parent_ingest(co, classname, code_objects, show_asm) + # tokens = [t for t in tokens if t.kind != 'SET_LINENO'] + + # # for t in tokens: + # # print(t) + # + # return tokens, customize diff --git a/uncompyle6/semantics/make_function.py b/uncompyle6/semantics/make_function.py index 8ad84981..f85bceed 100644 --- a/uncompyle6/semantics/make_function.py +++ b/uncompyle6/semantics/make_function.py @@ -595,9 +595,8 @@ def make_function3(self, node, is_lambda, nested=1, code_node=None): paramnames = list(scanner_code.co_varnames[:argc]) # defaults are for last n parameters, thus reverse - if self.version < 3.6: - paramnames.reverse(); - defparams.reverse() + paramnames.reverse(); + defparams.reverse() try: ast = self.build_ast(scanner_code._tokens, @@ -626,8 +625,7 @@ def make_function3(self, node, is_lambda, nested=1, code_node=None): else: params = paramnames - if not 3.1 <= self.version < 3.6: - params.reverse() # back to correct order + params.reverse() # back to correct order if code_has_star_arg(code): if self.version > 3.0: diff --git a/uncompyle6/version.py b/uncompyle6/version.py index 244078b5..a4c67266 100644 --- a/uncompyle6/version.py +++ b/uncompyle6/version.py @@ -12,4 +12,4 @@ # along with this program. If not, see . # This file is suitable for sourcing inside bash as # well as importing into Python -VERSION='3.2.2' +VERSION='3.2.3'