diff --git a/.gitignore b/.gitignore index 6507827e..ab75fa8e 100644 --- a/.gitignore +++ b/.gitignore @@ -6,6 +6,7 @@ /.eggs /.hypothesis /.idea +/.mypy_cache /.pytest_cache /.python-version /.tox diff --git a/NEWS.md b/NEWS.md index c7b9af50..b54a4df3 100644 --- a/NEWS.md +++ b/NEWS.md @@ -1,3 +1,12 @@ +4.1.0 2019-10-12 Stony Brook Ride +================================= + +- Fix fragment bugs + * missing RETURN_LAST introduced when adding transformation layer + * more parent entries on tokens +- Preliminary support for decompiling Python 1.0, 1.1. 1.2 and 1.6 + * Newer xdis version needed + 3.4.1 2019-10-02 ================ diff --git a/__pkginfo__.py b/__pkginfo__.py index 58b46e4d..3a36b559 100644 --- a/__pkginfo__.py +++ b/__pkginfo__.py @@ -58,7 +58,7 @@ entry_points = { ]} ftp_url = None install_requires = ["spark-parser >= 1.8.9, < 1.9.0", - "xdis >= 4.0.4, < 4.1.0"] + "xdis >= 4.1.0, < 4.2.0"] license = "GPL3" mailing_list = "python-debugger@googlegroups.com" diff --git a/test/Makefile b/test/Makefile index 21f0aae5..afaf1489 100644 --- a/test/Makefile +++ b/test/Makefile @@ -1,5 +1,6 @@ PHONY=check clean dist distclean test test-unit test-functional rmChangeLog clean_pyc nosetests \ - check-bytecode-1 check-bytecode-1.3 check-bytecode-1.4 check-bytecode-1.5 \ + check-bytecode-1.0 check-bytecode-1.1 check-bytecode-1.2 check-bytecode-1.3 \ + check-bytecode-1 check-bytecode-1.4 check-bytecode-1.5 check-bytecode-1.6 \ check-bytecode-2 check-bytecode-3 check-bytecode-3-short \ check-bytecode-2.2 check-byteocde-2.3 check-bytecode-2.4 \ check-short check-2.6 check-2.7 check-3.0 check-3.1 check-3.2 check-3.3 \ @@ -85,7 +86,7 @@ check-disasm: $(PYTHON) dis-compare.py #: Check deparsing bytecode 1.x only -check-bytecode-1: check-bytecode-1.4 check-bytecode-1.5 +check-bytecode-1: check-bytecode-1.0 check-bytecode-1.1 check-bytecode-1.2 check-bytecode-1.3 check-bytecode-1.4 check-bytecode-1.5 check-byecode-1.4 #: Check deparsing bytecode 2.x only check-bytecode-2: @@ -109,6 +110,7 @@ check-bytecode-3-short: #: Check deparsing bytecode on all Python 2 and Python 3 versions check-bytecode: check-bytecode-3 $(PYTHON) test_pythonlib.py \ + --bytecode-1.0 --bytecode-1.1 --bytecode-1.2 \ --bytecode-1.3 --bytecode-1.4 --bytecode-1.5 \ --bytecode-2.2 --bytecode-2.3 --bytecode-2.4 \ --bytecode-2.1 --bytecode-2.2 --bytecode-2.3 --bytecode-2.4 \ @@ -122,6 +124,18 @@ check-bytecode-short: check-bytecode-3-short --bytecode-2.6 --bytecode-2.7 --bytecode-pypy2.7 +#: Check deparsing bytecode 1.0 only +check-bytecode-1.0: + $(PYTHON) test_pythonlib.py --bytecode-1.0 + +#: Check deparsing bytecode 1.1 only +check-bytecode-1.1: + $(PYTHON) test_pythonlib.py --bytecode-1.1 + +#: Check deparsing bytecode 1.2 only +check-bytecode-1.2: + $(PYTHON) test_pythonlib.py --bytecode-1.2 + #: Check deparsing bytecode 1.3 only check-bytecode-1.3: $(PYTHON) test_pythonlib.py --bytecode-1.3 diff --git a/test/bytecode_1.0/simple_const.pyc b/test/bytecode_1.0/simple_const.pyc new file mode 100644 index 00000000..b6997b60 Binary files /dev/null and b/test/bytecode_1.0/simple_const.pyc differ diff --git a/test/bytecode_1.0/unpack_assign.pyc b/test/bytecode_1.0/unpack_assign.pyc new file mode 100644 index 00000000..79410b51 Binary files /dev/null and b/test/bytecode_1.0/unpack_assign.pyc differ diff --git a/test/bytecode_1.1/simple_const.pyc b/test/bytecode_1.1/simple_const.pyc new file mode 100644 index 00000000..fa938033 Binary files /dev/null and b/test/bytecode_1.1/simple_const.pyc differ diff --git a/test/bytecode_1.2/simple_const.pyc b/test/bytecode_1.2/simple_const.pyc new file mode 100644 index 00000000..1d91a84f Binary files /dev/null and b/test/bytecode_1.2/simple_const.pyc differ diff --git a/test/bytecode_1.6/simple_const.pyc b/test/bytecode_1.6/simple_const.pyc new file mode 100644 index 00000000..c7717dd7 Binary files /dev/null and b/test/bytecode_1.6/simple_const.pyc differ diff --git a/test/test_pythonlib.py b/test/test_pythonlib.py index b9efa7ed..715b6e92 100755 --- a/test/test_pythonlib.py +++ b/test/test_pythonlib.py @@ -77,9 +77,13 @@ for vers in (2.7, 3.4, 3.5, 3.6): pass for vers in ( + 1.0, + 1.1, + 1.2, 1.3, 1.4, 1.5, + 1.6, 2.1, 2.2, 2.3, diff --git a/uncompyle6/parser.py b/uncompyle6/parser.py index 926975dc..9637a56b 100644 --- a/uncompyle6/parser.py +++ b/uncompyle6/parser.py @@ -628,12 +628,30 @@ def get_python_parser( if version < 3.0: if version < 2.2: + if version == 1.0: + import uncompyle6.parsers.parse10 as parse10 + if compile_mode == 'exec': + p = parse10.Python10Parser(debug_parser) + else: + p = parse10.Python01ParserSingle(debug_parser) + elif version == 1.1: + import uncompyle6.parsers.parse11 as parse11 + if compile_mode == 'exec': + p = parse11.Python11Parser(debug_parser) + else: + p = parse11.Python11ParserSingle(debug_parser) + if version == 1.2: + import uncompyle6.parsers.parse12 as parse12 + if compile_mode == 'exec': + p = parse12.Python12Parser(debug_parser) + else: + p = parse12.Python12ParserSingle(debug_parser) if version == 1.3: import uncompyle6.parsers.parse13 as parse13 if compile_mode == 'exec': - p = parse13.Python14Parser(debug_parser) + p = parse13.Python13Parser(debug_parser) else: - p = parse13.Python14ParserSingle(debug_parser) + p = parse13.Python13ParserSingle(debug_parser) elif version == 1.4: import uncompyle6.parsers.parse14 as parse14 if compile_mode == 'exec': @@ -646,6 +664,12 @@ def get_python_parser( p = parse15.Python15Parser(debug_parser) else: p = parse15.Python15ParserSingle(debug_parser) + elif version == 1.6: + import uncompyle6.parsers.parse16 as parse16 + if compile_mode == 'exec': + p = parse16.Python16Parser(debug_parser) + else: + p = parse16.Python16ParserSingle(debug_parser) elif version == 2.1: import uncompyle6.parsers.parse21 as parse21 if compile_mode == 'exec': diff --git a/uncompyle6/parsers/parse10.py b/uncompyle6/parsers/parse10.py new file mode 100644 index 00000000..d862cdf1 --- /dev/null +++ b/uncompyle6/parsers/parse10.py @@ -0,0 +1,25 @@ +# Copyright (c) 2019 Rocky Bernstein + +from spark_parser import DEFAULT_DEBUG as PARSER_DEFAULT_DEBUG +from uncompyle6.parser import PythonParserSingle +from uncompyle6.parsers.parse11 import Python11Parser + + +class Python10Parser(Python11Parser): + def __init__(self, debug_parser=PARSER_DEFAULT_DEBUG): + super(Python11Parser, self).__init__(debug_parser) + self.customized = {} + + +class Python10ParserSingle(Python10Parser, PythonParserSingle): + pass + + +if __name__ == "__main__": + # Check grammar + p = Python10Parser() + p.check_grammar() + p.dump_grammar() + +# local variables: +# tab-width: 4 diff --git a/uncompyle6/parsers/parse11.py b/uncompyle6/parsers/parse11.py new file mode 100644 index 00000000..e5f6bec0 --- /dev/null +++ b/uncompyle6/parsers/parse11.py @@ -0,0 +1,25 @@ +# Copyright (c) 2019 Rocky Bernstein + +from spark_parser import DEFAULT_DEBUG as PARSER_DEFAULT_DEBUG +from uncompyle6.parser import PythonParserSingle +from uncompyle6.parsers.parse12 import Python12Parser + + +class Python11Parser(Python12Parser): + def __init__(self, debug_parser=PARSER_DEFAULT_DEBUG): + super(Python12Parser, self).__init__(debug_parser) + self.customized = {} + + +class Python11ParserSingle(Python11Parser, PythonParserSingle): + pass + + +if __name__ == "__main__": + # Check grammar + p = Python12Parser() + p.check_grammar() + p.dump_grammar() + +# local variables: +# tab-width: 4 diff --git a/uncompyle6/parsers/parse12.py b/uncompyle6/parsers/parse12.py new file mode 100644 index 00000000..b6d4750a --- /dev/null +++ b/uncompyle6/parsers/parse12.py @@ -0,0 +1,25 @@ +# Copyright (c) 2019 Rocky Bernstein + +from spark_parser import DEFAULT_DEBUG as PARSER_DEFAULT_DEBUG +from uncompyle6.parser import PythonParserSingle +from uncompyle6.parsers.parse13 import Python13Parser + + +class Python12Parser(Python13Parser): + def __init__(self, debug_parser=PARSER_DEFAULT_DEBUG): + super(Python12Parser, self).__init__(debug_parser) + self.customized = {} + + +class Python12ParserSingle(Python12Parser, PythonParserSingle): + pass + + +if __name__ == "__main__": + # Check grammar + p = Python12Parser() + p.check_grammar() + p.dump_grammar() + +# local variables: +# tab-width: 4 diff --git a/uncompyle6/parsers/parse16.py b/uncompyle6/parsers/parse16.py new file mode 100644 index 00000000..d5718009 --- /dev/null +++ b/uncompyle6/parsers/parse16.py @@ -0,0 +1,46 @@ +# Copyright (c) 2019 Rocky Bernstein + +from spark_parser import DEFAULT_DEBUG as PARSER_DEFAULT_DEBUG +from uncompyle6.parser import PythonParserSingle, nop_func +from uncompyle6.parsers.parse21 import Python21Parser + +class Python16Parser(Python21Parser): + + def __init__(self, debug_parser=PARSER_DEFAULT_DEBUG): + super(Python16Parser, self).__init__(debug_parser) + self.customized = {} + + def p_import16(self, args): + """ + import ::= filler IMPORT_NAME STORE_FAST + import ::= filler IMPORT_NAME STORE_NAME + + import_from ::= filler IMPORT_NAME importlist + import_from ::= filler filler IMPORT_NAME importlist POP_TOP + + importlist ::= importlist IMPORT_FROM + importlist ::= IMPORT_FROM + """ + + def customize_grammar_rules(self, tokens, customize): + super(Python16Parser, self).customize_grammar_rules(tokens, customize) + for i, token in enumerate(tokens): + opname = token.kind + opname_base = opname[:opname.rfind('_')] + + if opname_base == 'UNPACK_LIST': + self.addRule("store ::= unpack_list", nop_func) + + + +class Python16ParserSingle(Python16Parser, PythonParserSingle): + pass + +if __name__ == '__main__': + # Check grammar + p = Python15Parser() + p.check_grammar() + p.dump_grammar() + +# local variables: +# tab-width: 4 diff --git a/uncompyle6/scanner.py b/uncompyle6/scanner.py index 60c5c443..af25afb9 100755 --- a/uncompyle6/scanner.py +++ b/uncompyle6/scanner.py @@ -37,7 +37,7 @@ from xdis.util import code2num # The byte code versions we support. # Note: these all have to be floats -PYTHON_VERSIONS = frozenset((1.3, 1.4, 1.5, +PYTHON_VERSIONS = frozenset((1.0, 1.1, 1.3, 1.4, 1.5, 1.6, 2.1, 2.2, 2.3, 2.4, 2.5, 2.6, 2.7, 3.0, 3.1, 3.2, 3.3, 3.4, 3.5, 3.6, 3.7, 3.8)) diff --git a/uncompyle6/scanners/scanner10.py b/uncompyle6/scanners/scanner10.py new file mode 100644 index 00000000..afb3c2e3 --- /dev/null +++ b/uncompyle6/scanners/scanner10.py @@ -0,0 +1,35 @@ +# Copyright (c) 2019 by Rocky Bernstein +""" +Python 1.0 bytecode decompiler massaging. + +This massages tokenized 1.0 bytecode to make it more amenable for +grammar parsing. +""" + +import uncompyle6.scanners.scanner11 as scan + +# bytecode verification, verify(), uses JUMP_OPs from here +from xdis.opcodes import opcode_10 + +JUMP_OPS = opcode_10.JUMP_OPS + +# We base this off of 1.1 instead of the other way around +# because we cleaned things up this way. +# The history is that 2.7 support is the cleanest, +# then from that we got 2.6 and so on. +class Scanner10(scan.Scanner11): + def __init__(self, show_asm=False): + scan.Scanner11.__init__(self, show_asm) + self.opc = opcode_10 + self.opname = opcode_10.opname + self.version = 1.0 + return + + # def ingest(self, co, classname=None, code_objects={}, show_asm=None): + # tokens, customize = self.parent_ingest(co, classname, code_objects, show_asm) + # tokens = [t for t in tokens if t.kind != 'SET_LINENO'] + + # # for t in tokens: + # # print(t) + # + # return tokens, customize diff --git a/uncompyle6/scanners/scanner11.py b/uncompyle6/scanners/scanner11.py new file mode 100644 index 00000000..f81bc91b --- /dev/null +++ b/uncompyle6/scanners/scanner11.py @@ -0,0 +1,35 @@ +# Copyright (c) 2019 by Rocky Bernstein +""" +Python 1.1 bytecode decompiler massaging. + +This massages tokenized 1.1 bytecode to make it more amenable for +grammar parsing. +""" + +import uncompyle6.scanners.scanner13 as scan + +# bytecode verification, verify(), uses JUMP_OPs from here +from xdis.opcodes import opcode_11 + +JUMP_OPS = opcode_11.JUMP_OPS + +# We base this off of 1.2 instead of the other way around +# because we cleaned things up this way. +# The history is that 2.7 support is the cleanest, +# then from that we got 2.6 and so on. +class Scanner11(scan.Scanner13): # no scanner 1.2 + def __init__(self, show_asm=False): + scan.Scanner13.__init__(self, show_asm) + self.opc = opcode_11 + self.opname = opcode_11.opname + self.version = 1.1 + return + + # def ingest(self, co, classname=None, code_objects={}, show_asm=None): + # tokens, customize = self.parent_ingest(co, classname, code_objects, show_asm) + # tokens = [t for t in tokens if t.kind != 'SET_LINENO'] + + # # for t in tokens: + # # print(t) + # + # return tokens, customize diff --git a/uncompyle6/scanners/scanner12.py b/uncompyle6/scanners/scanner12.py new file mode 100644 index 00000000..8390ee0d --- /dev/null +++ b/uncompyle6/scanners/scanner12.py @@ -0,0 +1,36 @@ +# Copyright (c) 2019 by Rocky Bernstein +""" +Python 1.2 bytecode decompiler massaging. + +This massages tokenized 1.2 bytecode to make it more amenable for +grammar parsing. + +""" + +import uncompyle6.scanners.scanner13 as scan + +# bytecode verification, verify(), uses JUMP_OPs from here +from xdis.opcodes import opcode_11 + +JUMP_OPS = opcode_11.JUMP_OPS + +# We base this off of 1.3 instead of the other way around +# because we cleaned things up this way. +# The history is that 2.7 support is the cleanest, +# then from that we got 2.6 and so on. +class Scanner12(scan.Scanner13): + def __init__(self, show_asm=False): + scan.Scanner14.__init__(self, show_asm) + self.opc = opcode_11 + self.opname = opcode_11.opname + self.version = 1.2 # Note: is the same as 1.1 bytecode + return + + # def ingest(self, co, classname=None, code_objects={}, show_asm=None): + # tokens, customize = self.parent_ingest(co, classname, code_objects, show_asm) + # tokens = [t for t in tokens if t.kind != 'SET_LINENO'] + + # # for t in tokens: + # # print(t) + # + # return tokens, customize diff --git a/uncompyle6/scanners/scanner13.py b/uncompyle6/scanners/scanner13.py index f605fcf9..2d3b1bf7 100644 --- a/uncompyle6/scanners/scanner13.py +++ b/uncompyle6/scanners/scanner13.py @@ -7,10 +7,12 @@ grammar parsing. """ import uncompyle6.scanners.scanner14 as scan + # from uncompyle6.scanners.scanner26 import ingest as ingest26 # bytecode verification, verify(), uses JUMP_OPs from here from xdis.opcodes import opcode_13 + JUMP_OPS = opcode_13.JUMP_OPS # We base this off of 1.4 instead of the other way around diff --git a/uncompyle6/scanners/scanner16.py b/uncompyle6/scanners/scanner16.py new file mode 100644 index 00000000..d312a540 --- /dev/null +++ b/uncompyle6/scanners/scanner16.py @@ -0,0 +1,41 @@ +# Copyright (c) 2019 by Rocky Bernstein +""" +Python 1.6 bytecode decompiler massaging. + +This massages tokenized 1.6 bytecode to make it more amenable for +grammar parsing. +""" + +import uncompyle6.scanners.scanner21 as scan +# from uncompyle6.scanners.scanner26 import ingest as ingest26 + +# bytecode verification, verify(), uses JUMP_OPs from here +from xdis.opcodes import opcode_16 +JUMP_OPS = opcode_16.JUMP_OPS + +# We base this off of 2.2 instead of the other way around +# because we cleaned things up this way. +# The history is that 2.7 support is the cleanest, +# then from that we got 2.6 and so on. +class Scanner16(scan.Scanner21): + def __init__(self, show_asm=False): + scan.Scanner21.__init__(self, show_asm) + self.opc = opcode_16 + self.opname = opcode_16.opname + self.version = 1.6 + self.genexpr_name = '' + return + + def ingest(self, co, classname=None, code_objects={}, show_asm=None): + """ + Pick out tokens from an uncompyle6 code object, and transform them, + returning a list of uncompyle6 Token's. + + The transformations are made to assist the deparsing grammar. + """ + tokens, customize = scan.Scanner21.ingest(self, co, classname, code_objects, show_asm) + for t in tokens: + if t.op == self.opc.UNPACK_LIST: + t.kind = 'UNPACK_LIST_%d' % t.attr + pass + return tokens, customize diff --git a/uncompyle6/version.py b/uncompyle6/version.py index e63f6b2c..aaf8a48e 100644 --- a/uncompyle6/version.py +++ b/uncompyle6/version.py @@ -12,4 +12,4 @@ # along with this program. If not, see . # This file is suitable for sourcing inside bash as # well as importing into Python -VERSION="3.4.1" # noqa +VERSION="3.5.0" # noqa