diff --git a/.gitignore b/.gitignore index 88363be6..76e99b32 100644 --- a/.gitignore +++ b/.gitignore @@ -1,7 +1,7 @@ +*.pyo *.pyc *_dis *~ -*.pyc /.cache /.eggs /.python-version @@ -13,5 +13,6 @@ /nose-*.egg /tmp /uncompyle6.egg-info +/unpyc __pycache__ build diff --git a/ChangeLog b/ChangeLog index 57d20a1e..e5b12139 100644 --- a/ChangeLog +++ b/ChangeLog @@ -1,6 +1,173 @@ +2016-11-02 rocky + + * __pkginfo__.py, uncompyle6/version.py: Get ready for release 2.9.4 + +2016-11-02 rocky + + * README.rst: Update unpyc3 info. + +2016-11-01 rocky + + * pytest/test_grammar.py, uncompyle6/parsers/parse3.py, + uncompyle6/parsers/parse31.py, uncompyle6/parsers/parse32.py, + uncompyle6/semantics/make_function.py: Clean up annotation grammar a + little + +2016-11-01 rocky + + * test/simple_source/bug31/04_def_annotate.py, + uncompyle6/semantics/make_function.py: Full Python 3 annotations + +2016-10-30 rocky + + * .gitignore, README.rst, test/simple_source/def/03_class_method.py: + Note github unpyc3 and.. - Add source to bytecode_2.2/03_class_method.pyc - more ignore + +2016-10-30 rocky + + * uncompyle6/semantics/make_function.py: More source-code line + indention in make_function.. and remove Python 3 situations from make_function2() + +2016-10-29 rocky + + * uncompyle6/semantics/make_function.py, + uncompyle6/semantics/pysource.py: More annotation processing in to + make_function Move return-value annotation determination from n_mkfunc_annotate to + make_function_annotate which is where other kinds of annotation + handling will also need to be done. + +2016-10-29 rocky + + * uncompyle6/semantics/fragments.py, + uncompyle6/semantics/make_function.py, + uncompyle6/semantics/parser_error.py, + uncompyle6/semantics/pysource.py: Break out make_function() into its + own file. It is already too complex and will get worse in Python 3.6. Note: make_function in fragments.py is still inside and probably + needs fixup. + +2016-10-28 rocky + + * pytest/test_grammar.py, uncompyle6/parsers/parse3.py, + uncompyle6/parsers/parse31.py, uncompyle6/parsers/parse32.py, + uncompyle6/parsers/parse35.py, uncompyle6/semantics/pysource.py: + More complete annotate handling Still have a bit of work to do though. + +2016-10-28 rocky + + * pytest/test_grammar.py, uncompyle6/parsers/parse3.py, + uncompyle6/parsers/parse32.py, uncompyle6/parsers/parse33.py, + uncompyle6/parsers/parse34.py, uncompyle6/semantics/pysource.py: + Expand annotate return to Python 3.4 + +2016-10-28 rocky + + * pytest/test_grammar.py, uncompyle6/parsers/parse31.py, + uncompyle6/parsers/parse32.py, uncompyle6/semantics/pysource.py: + Expand annotate handling to 3.3 (and possibly 3.2) - DRY Python 3.1-3.3 grammar a little + +2016-10-28 rocky + + * uncompyle6/parser.py, uncompyle6/parsers/parse3.py, + uncompyle6/parsers/parse31.py, uncompyle6/parsers/parse32.py, + uncompyle6/parsers/parse33.py, uncompyle6/parsers/parse35.py: Split + out 3.1-3.3 parsers from parser3.py This is anticipation of extending annotation to Python 3.2+ + +2016-10-27 rocky + + * test/simple_source/bug31/04_def_annotate.py, + test/simple_source/bug31/04_def_attr.py, + uncompyle6/parsers/parse31.py, uncompyle6/semantics/pysource.py: + Clean and fix Python 3 annotate arg return + +2016-10-26 rocky + + * __pkginfo__.py: Dependencies stay within 2nd semantic level + +2016-10-26 rocky + + * ChangeLog, NEWS, uncompyle6/version.py: Get ready for release + 2.9.3 + +2016-10-26 rocky + + * test/simple_source/bug31/04_def_attr.py, + uncompyle6/parsers/parse31.py, uncompyle6/scanner.py, + uncompyle6/semantics/pysource.py: Start to attack Python 3.1 def() + -> xx construct Start to localize make_function routines by Python version + +2016-10-25 rocky + + * __pkginfo__.py, uncompyle6/parser.py, + uncompyle6/parsers/parse3.py, uncompyle6/parsers/parse31.py: Split + out Python 3.1 parser from rest. __pkginfo__.py: use Python 3.1 bytecode fixes + +2016-10-25 rocky + + * uncompyle6/parsers/parse3.py: Handle Python 3.1 "with ... as" + statement + +2016-10-24 rocky + + * test/Makefile: Add python 3.1 bytecode testing + +2016-10-24 rocky + + * test/simple_source/stmts/07_withstmt_fn.py, + uncompyle6/parsers/parse3.py: Python 3.1 "with" statement bug + +2016-10-24 rocky + + * uncompyle6/parsers/parse3.py, uncompyle6/parsers/parse34.py, + uncompyle6/parsers/parse35.py: Python 3.1 compile bug. DRY Python + 3.x rules ... via inheritance + +2016-10-24 rocky + + * uncompyle6/parsers/parse3.py, uncompyle6/scanners/scanner3.py: Fix + some Python 3.1 bugs + +2016-10-22 Daniel Bradburn + + * : Merge pull request #60 from rocky/buildstring Buildstring + +2016-10-22 rocky + + * pytest/test_fstring.py, test/simple_source/bug36/01_fstring.py, + uncompyle6/semantics/pysource.py: Move fstring rules inside a 3.6+ + check + +2016-10-22 rocky + + * : commit d6f7ef4e178e04d9a612d3a6c0b77a008732357f Author: rocky + Date: Fri Oct 21 07:40:35 2016 -0400 + +2016-10-20 moagstar + + * pytest/test_fstring.py, uncompyle6/parsers/parse3.py, + uncompyle6/parsers/parse36.py, uncompyle6/semantics/pysource.py: + further work on supporting single and multiple fstring decompilation + +2016-10-20 rocky + + * uncompyle6/main.py, uncompyle6/scanners/scanner2.py, + uncompyle6/scanners/scanner26.py: DRY Python 2.x unmangle_classname main.py: small typo: Disassembled -> Decompiled + +2016-10-19 moagstar + + * pytest/test_fstring.py, uncompyle6/parsers/parse3.py, + uncompyle6/parsers/parse36.py, uncompyle6/semantics/pysource.py: + urther work on fstrings for python 3.6 - there is a new opcode + build_string which is used to improve fstring performance, but broke + the fstring support in uncompyle + 2016-10-15 rocky - * uncompyle6/version.py: Get ready for release 2.9.2 + * uncompyle6/main.py: Change meta data info in uncompyle6: * Show file size of source when possible, i.e. in Python 3.x * Show full information about python interpreter used to decompile + +2016-10-15 rocky + + * ChangeLog, NEWS, __pkginfo__.py, requirements.txt, + uncompyle6/version.py: Get ready for release 2.9.2 2016-10-14 rocky diff --git a/NEWS b/NEWS index 630f278b..b65d7177 100644 --- a/NEWS +++ b/NEWS @@ -1,3 +1,24 @@ +uncompyle6 2.9.4 2016-11-02 + +- Handle Python 3.x function annotations +- track def keywoard-parameter line-splitting in source code better +- bump min xdis version to mask previous xdis bug + +uncompyle6 2.9.3 2016-10-26 + +Release forced by incompatiblity change in xdis 3.2.0. + +- Python 3.1 bugs: + * handle "with ... as" + * handle "with" + * Start handling def (...) -> yy (has bugs still) + +- DRY Python 3.x via inheritance +- Python 3.6 work (from Daniel Bradburn) + * Handle 3.6 buildstring + * Handle 3.6 handle single and multiple fstring better + + uncompyle6 2.9.2 2016-10-15 - use source-code line breaks to assist in where to break diff --git a/README.rst b/README.rst index 46c1eb08..a9cb5e1d 100644 --- a/README.rst +++ b/README.rst @@ -20,9 +20,9 @@ Why this? There were a number of decompyle, uncompile, uncompyle2, uncompyle3 forks around. All of them came basically from the same code base, and almost all of them no were no longer actively maintained. Only one -handled Python 3, and even there, only 3.2. This code pulls these -together and moves forward. It also addresses a number of open issues -in the previous forks. +handled Python 3, and even there, only 3.2 or 3.3 depending on which +code is used. This code pulls these together and moves forward. It +also addresses a number of open issues in the previous forks. What makes this different from other CPython bytecode decompilers?: its ability to deparse just fragments and give source-code information @@ -132,6 +132,7 @@ See Also * https://github.com/zrax/pycdc : supports all versions of Python and is written in C++ * https://code.google.com/archive/p/unpyc3/ : supports Python 3.2 only. The above projects use a different decompiling technique what is used here. +* https://github.com/figment/unpyc3/ : fork of above, but supports Python 3.3 only. Include some fixes like supporting function annotations * The HISTORY_ file. .. |downloads| image:: https://img.shields.io/pypi/dd/uncompyle6.svg diff --git a/__pkginfo__.py b/__pkginfo__.py index 2c211f66..13a669a1 100644 --- a/__pkginfo__.py +++ b/__pkginfo__.py @@ -37,8 +37,8 @@ entry_points={ 'pydisassemble=uncompyle6.bin.pydisassemble:main', ]} ftp_url = None -install_requires = ['spark-parser >= 1.4.0', - 'xdis >= 3.1.0'] +install_requires = ['spark-parser >= 1.4.0, < 1.5.0', + 'xdis >= 3.2.2, < 3.3.0'] license = 'MIT' mailing_list = 'python-debugger@googlegroups.com' modname = 'uncompyle6' diff --git a/pytest/test_grammar.py b/pytest/test_grammar.py index c091bd6c..c6cc8e02 100644 --- a/pytest/test_grammar.py +++ b/pytest/test_grammar.py @@ -1,4 +1,4 @@ -import pytest, re +import re from uncompyle6 import PYTHON_VERSION, PYTHON3, IS_PYPY # , PYTHON_VERSION from uncompyle6.parser import get_python_parser from uncompyle6.scanner import get_scanner @@ -16,14 +16,21 @@ def test_grammar(): p = get_python_parser(PYTHON_VERSION, is_pypy=IS_PYPY) lhs, rhs, tokens, right_recursive = p.checkSets() expect_lhs = set(['expr1024', 'pos_arg']) - unused_rhs = set(['build_list', 'call_function', 'mkfunc', 'mklambda', + unused_rhs = set(['build_list', 'call_function', 'mkfunc', + 'mklambda', 'unpack', 'unpack_list']) expect_right_recursive = [['designList', ('designator', 'DUP_TOP', 'designList')]] if PYTHON3: expect_lhs.add('load_genexpr') + unused_rhs = unused_rhs.union(set(""" except_pop_except genexpr classdefdeco2 listcomp """.split())) + if 3.0 <= PYTHON_VERSION: + expect_lhs.add("annotate_arg") + expect_lhs.add("annotate_tuple") + unused_rhs.add("mkfunc_annotate") + pass else: expect_lhs.add('kwarg') assert expect_lhs == set(lhs) @@ -43,5 +50,6 @@ def test_grammar(): check_tokens(tokens, opcode_set) elif PYTHON_VERSION == 3.4: ignore_set.add('LOAD_CLASSNAME') + ignore_set.add('STORE_LOCALS') opcode_set = set(s.opc.opname).union(ignore_set) check_tokens(tokens, opcode_set) diff --git a/test/Makefile b/test/Makefile index 8985d0b0..e36ff3c0 100644 --- a/test/Makefile +++ b/test/Makefile @@ -66,7 +66,7 @@ check-bytecode-2: #: Check deparsing bytecode 3.x only check-bytecode-3: - $(PYTHON) test_pythonlib.py --bytecode-3.2 --bytecode-3.3 \ + $(PYTHON) test_pythonlib.py --bytecode-3.1 --bytecode-3.2 --bytecode-3.3 \ --bytecode-3.4 --bytecode-3.5 --bytecode-pypy3.2 #: Check deparsing bytecode that works running Python 2 and Python 3 diff --git a/test/bytecode_2.2/03_class_method.pyc b/test/bytecode_2.2/03_class_method.pyc index 03044567..08f9ed77 100644 Binary files a/test/bytecode_2.2/03_class_method.pyc and b/test/bytecode_2.2/03_class_method.pyc differ diff --git a/test/bytecode_3.1/04_def_annotate.pyc b/test/bytecode_3.1/04_def_annotate.pyc new file mode 100644 index 00000000..c8a5c062 Binary files /dev/null and b/test/bytecode_3.1/04_def_annotate.pyc differ diff --git a/test/bytecode_3.1/04_def_attr.pyc-notyet b/test/bytecode_3.1/04_def_attr.pyc-notyet new file mode 100644 index 00000000..cae0c27f Binary files /dev/null and b/test/bytecode_3.1/04_def_attr.pyc-notyet differ diff --git a/test/bytecode_3.1/04_withas.pyc b/test/bytecode_3.1/04_withas.pyc new file mode 100644 index 00000000..d9c2af12 Binary files /dev/null and b/test/bytecode_3.1/04_withas.pyc differ diff --git a/test/bytecode_3.1/07_withstmt_fn.pyc b/test/bytecode_3.1/07_withstmt_fn.pyc new file mode 100644 index 00000000..f8e08863 Binary files /dev/null and b/test/bytecode_3.1/07_withstmt_fn.pyc differ diff --git a/test/bytecode_3.3/04_raise.pyc b/test/bytecode_3.3/04_raise.pyc new file mode 100644 index 00000000..a2c350e1 Binary files /dev/null and b/test/bytecode_3.3/04_raise.pyc differ diff --git a/test/bytecode_3.3/04_withas.pyc b/test/bytecode_3.3/04_withas.pyc new file mode 100644 index 00000000..e1b54e5a Binary files /dev/null and b/test/bytecode_3.3/04_withas.pyc differ diff --git a/test/bytecode_3.4/04_def_annotate.pyc b/test/bytecode_3.4/04_def_annotate.pyc new file mode 100644 index 00000000..881b347f Binary files /dev/null and b/test/bytecode_3.4/04_def_annotate.pyc differ diff --git a/test/simple_source/bug31/04_def_annotate.py b/test/simple_source/bug31/04_def_annotate.py new file mode 100644 index 00000000..023c6775 --- /dev/null +++ b/test/simple_source/bug31/04_def_annotate.py @@ -0,0 +1,10 @@ +# Python 3 annotations + +def foo(a, b: 'annotating b', c: int) -> float: + print(a + b + c) + +# Python 3.1 _pyio.py uses the -> "IOBase" annotation +def open(file, mode = "r", buffering = None, + encoding = None, errors = None, + newline = None, closefd = True) -> "IOBase": + return text diff --git a/test/simple_source/def/03_class_method.py b/test/simple_source/def/03_class_method.py new file mode 100644 index 00000000..47d635d9 --- /dev/null +++ b/test/simple_source/def/03_class_method.py @@ -0,0 +1,13 @@ +# From Decompyle++ +# File: 22_class_method.pyc (Python 2.2) +# An old-style Python class. + +class MyClass: + + def method(self, i): + if i is 5: + print 'five' + elif not (i is 2): + print 'not two' + else: + print '2' diff --git a/test/simple_source/stmts/07_withstmt_fn.py b/test/simple_source/stmts/07_withstmt_fn.py index 03a26247..dee5a7a0 100644 --- a/test/simple_source/stmts/07_withstmt_fn.py +++ b/test/simple_source/stmts/07_withstmt_fn.py @@ -1,4 +1,4 @@ -# Python 2.6 has a truly weird way of handling with here. +# Python 2.6 has a truly weird way of handling "with" here. # added rule for 2.6 # setupwith ::= DUP_TOP LOAD_ATTR ROT_TWO LOAD_ATTR CALL_FUNCTION_0 POP_TOP diff --git a/uncompyle6/bin/uncompile.py b/uncompyle6/bin/uncompile.py index 18c2651a..7aa68910 100755 --- a/uncompyle6/bin/uncompile.py +++ b/uncompyle6/bin/uncompile.py @@ -98,7 +98,7 @@ def main_bin(): elif opt == '--verify': options['do_verify'] = True elif opt in ('--asm', '-a'): - options['showasm'] = True + options['showasm'] = 'after' options['do_verify'] = False elif opt in ('--tree', '-t'): options['showast'] = True diff --git a/uncompyle6/main.py b/uncompyle6/main.py index de82d794..e81eff26 100644 --- a/uncompyle6/main.py +++ b/uncompyle6/main.py @@ -11,7 +11,7 @@ from uncompyle6.version import VERSION from xdis.load import load_module def uncompyle( - bytecode_version, co, out=None, showasm=False, showast=False, + bytecode_version, co, out=None, showasm=None, showast=False, timestamp=None, showgrammar=False, code_objects={}, source_size=None, is_pypy=False, magic_int=None): """ @@ -53,7 +53,7 @@ def uncompyle( -def uncompyle_file(filename, outstream=None, showasm=False, showast=False, +def uncompyle_file(filename, outstream=None, showasm=None, showast=False, showgrammar=False): """ decompile Python byte-code file (.pyc) @@ -79,7 +79,7 @@ def uncompyle_file(filename, outstream=None, showasm=False, showast=False, # FIXME: combine into an options parameter def main(in_base, out_base, files, codes, outfile=None, - showasm=False, showast=False, do_verify=False, + showasm=None, showast=False, do_verify=False, showgrammar=False, raise_on_error=False): """ in_base base directory for input files diff --git a/uncompyle6/parser.py b/uncompyle6/parser.py index c459b4eb..d096e409 100644 --- a/uncompyle6/parser.py +++ b/uncompyle6/parser.py @@ -627,20 +627,23 @@ def get_python_parser( else: p = parse3.Python30ParserSingle(debug_parser) elif version == 3.1: + import uncompyle6.parsers.parse31 as parse31 if compile_mode == 'exec': - p = parse3.Python31Parser(debug_parser) + p = parse31.Python31Parser(debug_parser) else: - p = parse3.Python31ParserSingle(debug_parser) + p = parse31.Python31ParserSingle(debug_parser) elif version == 3.2: + import uncompyle6.parsers.parse32 as parse32 if compile_mode == 'exec': - p = parse3.Python32Parser(debug_parser) + p = parse32.Python32Parser(debug_parser) else: - p = parse3.Python32ParserSingle(debug_parser) + p = parse32.Python32ParserSingle(debug_parser) elif version == 3.3: + import uncompyle6.parsers.parse33 as parse33 if compile_mode == 'exec': - p = parse3.Python33Parser(debug_parser) + p = parse33.Python33Parser(debug_parser) else: - p = parse3.Python33ParserSingle(debug_parser) + p = parse33.Python33ParserSingle(debug_parser) elif version == 3.4: import uncompyle6.parsers.parse34 as parse34 if compile_mode == 'exec': diff --git a/uncompyle6/parsers/astnode.py b/uncompyle6/parsers/astnode.py index 7c66502d..69d06388 100644 --- a/uncompyle6/parsers/astnode.py +++ b/uncompyle6/parsers/astnode.py @@ -33,7 +33,7 @@ class AST(spark_AST): else: child = node.__repr1__(indent, None) else: - inst = str(node) + inst = node.format(line_prefix='L.') if inst.startswith("\n"): # Nuke leading \n inst = inst[1:] diff --git a/uncompyle6/parsers/parse3.py b/uncompyle6/parsers/parse3.py index 9bf3314a..e55d0a1b 100644 --- a/uncompyle6/parsers/parse3.py +++ b/uncompyle6/parsers/parse3.py @@ -246,6 +246,25 @@ class Python3Parser(PythonParser): c_stmts_opt34 ::= JUMP_BACK JUMP_ABSOLUTE c_stmts_opt """ + + def p_def_annotations3(self, args): + """ + # Annotated functions + stmt ::= funcdef_annotate + funcdef_annotate ::= mkfunc_annotate designator + + # This has the annotation value. + # LOAD_NAME is used in an annotation type like + # int, float, str + annotate_arg ::= LOAD_NAME + # LOAD_CONST is used in an annotation string + annotate_arg ::= LOAD_CONST + + # This stores the tuple of parameter names + # that have been annotated + annotate_tuple ::= LOAD_CONST + """ + def p_come_from3(self, args): """ opt_come_from_except ::= COME_FROM_EXCEPT @@ -360,10 +379,9 @@ class Python3Parser(PythonParser): # Python 3.4+ expr ::= LOAD_CLASSDEREF + binary_subscr2 ::= expr expr DUP_TOP_TWO BINARY_SUBSCR # Python3 drops slice0..slice3 - # In Python 2, DUP_TOP_TWO is DUP_TOPX_2 - binary_subscr2 ::= expr expr DUP_TOP_TWO BINARY_SUBSCR ''' @staticmethod @@ -671,7 +689,6 @@ class Python3Parser(PythonParser): self.add_unique_rule(rule, opname, token.attr, customize) return - class Python30Parser(Python3Parser): def p_30(self, args): @@ -679,55 +696,14 @@ class Python30Parser(Python3Parser): # Store locals is only in Python 3.0 to 3.3 stmt ::= store_locals store_locals ::= LOAD_FAST STORE_LOCALS - """ -class Python31Parser(Python3Parser): - - def p_31(self, args): - """ - # Store locals is only in Python 3.0 to 3.3 - stmt ::= store_locals - store_locals ::= LOAD_FAST STORE_LOCALS - """ - -class Python32Parser(Python3Parser): - - def p_32(self, args): - """ - # Store locals is only in Python 3.0 to 3.3 - stmt ::= store_locals - store_locals ::= LOAD_FAST STORE_LOCALS - """ - -class Python33Parser(Python3Parser): - def p_33(self, args): - """ - # Store locals is only in Python 3.0 to 3.3 - stmt ::= store_locals - store_locals ::= LOAD_FAST STORE_LOCALS - - # Python 3.3 adds yield from. - expr ::= yield_from - yield_from ::= expr expr YIELD_FROM + jmp_true ::= JUMP_IF_TRUE_OR_POP POP_TOP + _ifstmts_jump ::= c_stmts_opt JUMP_FORWARD POP_TOP COME_FROM """ class Python3ParserSingle(Python3Parser, PythonParserSingle): pass - -class Python30ParserSingle(Python30Parser, PythonParserSingle): - pass - -class Python31ParserSingle(Python31Parser, PythonParserSingle): - pass - -class Python32ParserSingle(Python32Parser, PythonParserSingle): - pass - - -class Python33ParserSingle(Python33Parser, PythonParserSingle): - pass - def info(args): # Check grammar p = Python3Parser() @@ -737,8 +713,10 @@ def info(args): from uncompyle6.parser.parse35 import Python35Parser p = Python35Parser() elif arg == '3.3': + from uncompyle6.parser.parse33 import Python33Parser p = Python33Parser() elif arg == '3.2': + from uncompyle6.parser.parse32 import Python32Parser p = Python32Parser() elif arg == '3.0': p = Python30Parser() diff --git a/uncompyle6/parsers/parse31.py b/uncompyle6/parsers/parse31.py new file mode 100644 index 00000000..647ed421 --- /dev/null +++ b/uncompyle6/parsers/parse31.py @@ -0,0 +1,51 @@ +# Copyright (c) 2016 Rocky Bernstein +""" +spark grammar differences over Python 3.2 for Python 3.1. +""" +from __future__ import print_function + +from uncompyle6.parser import PythonParserSingle +from uncompyle6.parsers.parse32 import Python32Parser + +class Python31Parser(Python32Parser): + + def p_31(self, args): + """ + binary_subscr2 ::= expr expr DUP_TOPX BINARY_SUBSCR + + setupwith ::= DUP_TOP LOAD_ATTR store LOAD_ATTR CALL_FUNCTION_0 POP_TOP + setupwithas ::= DUP_TOP LOAD_ATTR store LOAD_ATTR CALL_FUNCTION_0 store + withstmt ::= expr setupwith SETUP_FINALLY + suite_stmts_opt + POP_BLOCK LOAD_CONST COME_FROM_FINALLY + load del_stmt WITH_CLEANUP END_FINALLY + + # Keeps Python 3.1 withas desigator in the same position as it is in other version + setupwithas31 ::= setupwithas SETUP_FINALLY load del_stmt + + withasstmt ::= expr setupwithas31 designator + suite_stmts_opt + POP_BLOCK LOAD_CONST COME_FROM_FINALLY + load del_stmt WITH_CLEANUP END_FINALLY + + store ::= STORE_FAST + store ::= STORE_NAME + load ::= LOAD_FAST + load ::= LOAD_NAME + """ + + def add_custom_rules(self, tokens, customize): + super(Python31Parser, self).add_custom_rules(tokens, customize) + for i, token in enumerate(tokens): + opname = token.type + if opname.startswith('MAKE_FUNCTION_A'): + args_pos, args_kw, annotate_args = token.attr + # Check that there are 2 annotated params? + # rule = ('mkfunc2 ::= %s%sEXTENDED_ARG %s' % + # ('pos_arg ' * (args_pos), 'kwargs ' * (annotate_args-1), opname)) + rule = ('mkfunc_annotate ::= %s%sannotate_tuple LOAD_CONST EXTENDED_ARG %s' % + (('pos_arg ' * (args_pos)), + ('annotate_arg ' * (annotate_args-1)), opname)) + self.add_unique_rule(rule, opname, token.attr, customize) +class Python31ParserSingle(Python31Parser, PythonParserSingle): + pass diff --git a/uncompyle6/parsers/parse32.py b/uncompyle6/parsers/parse32.py new file mode 100644 index 00000000..d88ee91d --- /dev/null +++ b/uncompyle6/parsers/parse32.py @@ -0,0 +1,35 @@ +# Copyright (c) 2016 Rocky Bernstein +""" +spark grammar differences over Python 3 for Python 3.2. +""" +from __future__ import print_function + +from uncompyle6.parser import PythonParserSingle +from uncompyle6.parsers.parse3 import Python3Parser + +class Python32Parser(Python3Parser): + def p_32on(self, args): + """ + # In Python 3.2+, DUP_TOPX is DUP_TOP_TWO + binary_subscr2 ::= expr expr DUP_TOP_TWO BINARY_SUBSCR + stmt ::= store_locals + store_locals ::= LOAD_FAST STORE_LOCALS + """ + pass + + def add_custom_rules(self, tokens, customize): + super(Python32Parser, self).add_custom_rules(tokens, customize) + for i, token in enumerate(tokens): + opname = token.type + if opname.startswith('MAKE_FUNCTION_A'): + args_pos, args_kw, annotate_args = token.attr + # Check that there are 2 annotated params? + rule = (('mkfunc_annotate ::= %s%sannotate_tuple ' + 'LOAD_CONST LOAD_CONST EXTENDED_ARG %s') % + (('pos_arg ' * (args_pos)), + ('annotate_arg ' * (annotate_args-1)), opname)) + self.add_unique_rule(rule, opname, token.attr, customize) + + +class Python32ParserSingle(Python32Parser, PythonParserSingle): + pass diff --git a/uncompyle6/parsers/parse33.py b/uncompyle6/parsers/parse33.py new file mode 100644 index 00000000..dd43c12e --- /dev/null +++ b/uncompyle6/parsers/parse33.py @@ -0,0 +1,20 @@ +# Copyright (c) 2016 Rocky Bernstein +""" +spark grammar differences over Python 3.2 for Python 3.3. +""" +from __future__ import print_function + +from uncompyle6.parser import PythonParserSingle +from uncompyle6.parsers.parse32 import Python32Parser + +class Python33Parser(Python32Parser): + + def p_33on(self, args): + """ + # Python 3.3+ adds yield from. + expr ::= yield_from + yield_from ::= expr expr YIELD_FROM + """ + +class Python33ParserSingle(Python33Parser, PythonParserSingle): + pass diff --git a/uncompyle6/parsers/parse34.py b/uncompyle6/parsers/parse34.py index 1ef1f7b8..bce24488 100644 --- a/uncompyle6/parsers/parse34.py +++ b/uncompyle6/parsers/parse34.py @@ -1,13 +1,13 @@ # Copyright (c) 2016 Rocky Bernstein """ -spark grammar differences over Python3 for Python 3.4.2. +spark grammar differences over Python 3.3 for Python 3.4 """ from uncompyle6.parser import PythonParserSingle from spark_parser import DEFAULT_DEBUG as PARSER_DEFAULT_DEBUG -from uncompyle6.parsers.parse3 import Python3Parser +from uncompyle6.parsers.parse33 import Python33Parser -class Python34Parser(Python3Parser): +class Python34Parser(Python33Parser): def __init__(self, debug_parser=PARSER_DEFAULT_DEBUG): super(Python34Parser, self).__init__(debug_parser) @@ -28,12 +28,6 @@ class Python34Parser(Python3Parser): iflaststmt ::= testexpr c_stmts_opt34 c_stmts_opt34 ::= JUMP_BACK JUMP_ABSOLUTE c_stmts_opt - # Python 3.3 added "yield from." Do it the same way as in - # 3.3 - - expr ::= yield_from - yield_from ::= expr expr YIELD_FROM - # Is this 3.4 only? yield_from ::= expr GET_ITER LOAD_CONST YIELD_FROM diff --git a/uncompyle6/parsers/parse35.py b/uncompyle6/parsers/parse35.py index 8d9bb1f8..996eec72 100644 --- a/uncompyle6/parsers/parse35.py +++ b/uncompyle6/parsers/parse35.py @@ -1,14 +1,14 @@ # Copyright (c) 2016 Rocky Bernstein """ -spark grammar differences over Python3 for Python 3.5. +spark grammar differences over Python 3.4 for Python 3.5. """ from __future__ import print_function from uncompyle6.parser import PythonParserSingle from spark_parser import DEFAULT_DEBUG as PARSER_DEFAULT_DEBUG -from uncompyle6.parsers.parse3 import Python3Parser +from uncompyle6.parsers.parse34 import Python34Parser -class Python35Parser(Python3Parser): +class Python35Parser(Python34Parser): def __init__(self, debug_parser=PARSER_DEFAULT_DEBUG): super(Python35Parser, self).__init__(debug_parser) diff --git a/uncompyle6/scanners/scanner2.py b/uncompyle6/scanners/scanner2.py index 839887a3..0625de03 100644 --- a/uncompyle6/scanners/scanner2.py +++ b/uncompyle6/scanners/scanner2.py @@ -291,7 +291,7 @@ class Scanner2(scan.Scanner): if show_asm in ('both', 'after'): for t in tokens: - print(t) + print(t.format(line_prefix='L.')) print() return tokens, customize diff --git a/uncompyle6/scanners/scanner30.py b/uncompyle6/scanners/scanner30.py new file mode 100644 index 00000000..84b0edd9 --- /dev/null +++ b/uncompyle6/scanners/scanner30.py @@ -0,0 +1,34 @@ +# Copyright (c) 2016 by Rocky Bernstein +""" +Python 3.0 bytecode scanner/deparser + +This sets up opcodes Python's 3.0 and calls a generalized +scanner routine for Python 3. +""" + +from __future__ import print_function + +# bytecode verification, verify(), uses JUMP_OPs from here +from xdis.opcodes import opcode_30 as opc +JUMP_OPs = map(lambda op: opc.opname[op], opc.hasjrel + opc.hasjabs) + +from uncompyle6.scanners.scanner3 import Scanner3 +class Scanner30(Scanner3): + + def __init__(self, show_asm=None, is_pypy=False): + Scanner3.__init__(self, 3.1, show_asm, is_pypy) + return + pass + +if __name__ == "__main__": + from uncompyle6 import PYTHON_VERSION + if PYTHON_VERSION == 3.0: + import inspect + co = inspect.currentframe().f_code + tokens, customize = Scanner30().ingest(co) + for t in tokens: + print(t) + pass + else: + print("Need to be Python 3.0 to demo; I am %s." % + PYTHON_VERSION) diff --git a/uncompyle6/scanners/tok.py b/uncompyle6/scanners/tok.py index f9879334..a260a24c 100644 --- a/uncompyle6/scanners/tok.py +++ b/uncompyle6/scanners/tok.py @@ -53,7 +53,11 @@ class Token: # ('%9s %-18s %r' % (self.offset, self.type, pattr))) def __str__(self): - prefix = '\n%4d ' % self.linestart if self.linestart else (' ' * 6) + return self.format(line_prefix='') + + def format(self, line_prefix=''): + prefix = ('\n%s%4d ' % (line_prefix, self.linestart) + if self.linestart else (' ' * (6 + len(line_prefix)))) offset_opname = '%6s %-17s' % (self.offset, self.type) if not self.has_arg: return "%s%s" % (prefix, offset_opname) diff --git a/uncompyle6/semantics/fragments.py b/uncompyle6/semantics/fragments.py index a1324f73..9a4c0e0a 100644 --- a/uncompyle6/semantics/fragments.py +++ b/uncompyle6/semantics/fragments.py @@ -67,7 +67,9 @@ from uncompyle6.show import ( ) from uncompyle6.semantics.pysource import AST, INDENT_PER_LEVEL, NONE, PRECEDENCE, \ - ParserError, TABLE_DIRECT, escape, find_all_globals, find_globals, find_none, minint, MAP + ParserError, TABLE_DIRECT, escape, find_globals, minint, MAP + +from uncompyle6.semantics.make_function import find_all_globals, find_none if PYTHON3: from itertools import zip_longest @@ -77,8 +79,7 @@ else: from StringIO import StringIO -from spark_parser import GenericASTTraversalPruningException, \ - DEFAULT_DEBUG as PARSER_DEFAULT_DEBUG +from spark_parser import DEFAULT_DEBUG as PARSER_DEFAULT_DEBUG from collections import namedtuple NodeInfo = namedtuple("NodeInfo", "node start finish") diff --git a/uncompyle6/semantics/make_function.py b/uncompyle6/semantics/make_function.py new file mode 100644 index 00000000..5d207ca9 --- /dev/null +++ b/uncompyle6/semantics/make_function.py @@ -0,0 +1,558 @@ +# Copyright (c) 2015, 2016 by Rocky Bernstein +# Copyright (c) 2000-2002 by hartmut Goebel +""" +All the crazy things we have to do to handle Python functions +""" +from xdis.code import iscode +from uncompyle6.scanner import Code +from uncompyle6.parsers.astnode import AST +from uncompyle6 import PYTHON3 +from uncompyle6.semantics.parser_error import ParserError + +if PYTHON3: + from itertools import zip_longest +else: + from itertools import izip_longest as zip_longest + +from uncompyle6.show import maybe_show_ast_param_default + +def find_all_globals(node, globs): + """Find globals in this statement.""" + for n in node: + if isinstance(n, AST): + globs = find_all_globals(n, globs) + elif n.type in ('STORE_GLOBAL', 'DELETE_GLOBAL', 'LOAD_GLOBAL'): + globs.add(n.pattr) + return globs + +def find_globals(node, globs): + """Find globals in this statement.""" + for n in node: + if isinstance(n, AST): + globs = find_globals(n, globs) + elif n.type in ('STORE_GLOBAL', 'DELETE_GLOBAL'): + globs.add(n.pattr) + return globs + +def find_none(node): + for n in node: + if isinstance(n, AST): + if not n in ('return_stmt', 'return_if_stmt'): + if find_none(n): + return True + elif n.type == 'LOAD_CONST' and n.pattr is None: + return True + return False + +# FIXME: DRY the below code... + +def make_function3_annotate(self, node, isLambda, nested=1, + codeNode=None, annotate_last=-1): + """ + Dump function defintion, doc string, and function + body. This code is specialized for Python 3""" + + def build_param(ast, name, default): + """build parameters: + - handle defaults + - handle format tuple parameters + """ + if default: + value = self.traverse(default, indent='') + maybe_show_ast_param_default(self.showast, name, value) + result = '%s=%s' % (name, value) + if result[-2:] == '= ': # default was 'LOAD_CONST None' + result += 'None' + return result + else: + return name + + # MAKE_FUNCTION_... or MAKE_CLOSURE_... + assert node[-1].type.startswith('MAKE_') + + annotate_tuple = node[annotate_last] + annotate_args = {} + + if (annotate_tuple == 'annotate_tuple' + and annotate_tuple[0] in ('LOAD_CONST', 'LOAD_NAME') + and isinstance(annotate_tuple[0].attr, tuple)): + annotate_tup = annotate_tuple[0].attr + i = -1 + j = annotate_last-1 + l = -len(node) + while j >= l and node[j].type in ('annotate_arg' 'annotate_tuple'): + annotate_args[annotate_tup[i]] = (node[j][0].attr, + node[j][0] == 'LOAD_CONST') + i -= 1 + j -= 1 + + args_node = node[-1] + if isinstance(args_node.attr, tuple): + # positional args are before kwargs + defparams = node[:args_node.attr[0]] + pos_args, kw_args, annotate_argc = args_node.attr + else: + defparams = node[:args_node.attr] + kw_args = 0 + pass + + if 3.0 <= self.version <= 3.2: + lambda_index = -2 + elif 3.03 <= self.version: + lambda_index = -3 + else: + lambda_index = None + + if lambda_index and isLambda and iscode(node[lambda_index].attr): + assert node[lambda_index].type == 'LOAD_LAMBDA' + code = node[lambda_index].attr + else: + code = codeNode.attr + + assert iscode(code) + code = Code(code, self.scanner, self.currentclass) + + # add defaults values to parameter names + argc = code.co_argcount + paramnames = list(code.co_varnames[:argc]) + + try: + ast = self.build_ast(code._tokens, + code._customize, + isLambda = isLambda, + noneInNames = ('None' in code.co_names)) + except ParserError as p: + self.write(str(p)) + self.ERROR = p + return + + kw_pairs = args_node.attr[1] + indent = self.indent + + if isLambda: + self.write("lambda ") + else: + self.write("(") + + last_line = self.f.getvalue().split("\n")[-1] + l = len(last_line) + indent = ' ' * l + line_number = self.line_number + + if 4 & code.co_flags: # flag 2 -> variable number of args + self.write('*%s' % code.co_varnames[argc + kw_pairs]) + argc += 1 + + i = len(paramnames) - len(defparams) + suffix = '' + for param in paramnames[:i]: + self.write(suffix, param) + if param in annotate_args: + value, string = annotate_args[param] + if string: + self.write(': "%s"' % value) + else: + self.write(': %s' % value) + suffix = ', ' + + suffix = ', ' if i > 0 else '' + for n in node: + if n == 'pos_arg': + self.write(suffix) + param = paramnames[i] + self.write(param) + if param in annotate_args: + self.write(':"%s' % annotate_args[param]) + self.write('=') + i += 1 + self.preorder(n) + if (line_number != self.line_number): + suffix = ",\n" + indent + line_number = self.line_number + else: + suffix = ', ' + + # self.println(indent, '#flags:\t', int(code.co_flags)) + if kw_args > 0: + if not (4 & code.co_flags): + if argc > 0: + self.write(", *, ") + else: + self.write("*, ") + pass + else: + self.write(", ") + + kwargs = node[0] + last = len(kwargs)-1 + i = 0 + for n in node[0]: + if n == 'kwarg': + self.write('%s=' % n[0].pattr) + self.preorder(n[1]) + if i < last: + self.write(', ') + i += 1 + pass + pass + pass + + if 8 & code.co_flags: # flag 3 -> keyword args + if argc > 0: + self.write(', ') + self.write('**%s' % code.co_varnames[argc + kw_pairs]) + + if isLambda: + self.write(": ") + else: + self.write(')') + if 'return' in annotate_args: + value, string = annotate_args['return'] + if string: + self.write(' -> "%s"' % value) + else: + self.write(' -> %s' % value) + + self.println(":") + + if (len(code.co_consts) > 0 and + code.co_consts[0] is not None and not isLambda): # ugly + # docstring exists, dump it + self.print_docstring(indent, code.co_consts[0]) + + code._tokens = None # save memory + assert ast == 'stmts' + + all_globals = find_all_globals(ast, set()) + for g in ((all_globals & self.mod_globs) | find_globals(ast, set())): + self.println(self.indent, 'global ', g) + self.mod_globs -= all_globals + has_none = 'None' in code.co_names + rn = has_none and not find_none(ast) + self.gen_source(ast, code.co_name, code._customize, isLambda=isLambda, + returnNone=rn) + code._tokens = code._customize = None # save memory + +def make_function2(self, node, isLambda, nested=1, codeNode=None): + """ + Dump function defintion, doc string, and function body. + This code is specialied for Python 2. + """ + + # FIXME: call make_function3 if we are self.version >= 3.0 + # and then simplify the below. + + def build_param(ast, name, default): + """build parameters: + - handle defaults + - handle format tuple parameters + """ + # if formal parameter is a tuple, the paramater name + # starts with a dot (eg. '.1', '.2') + if name.startswith('.'): + # replace the name with the tuple-string + name = self.get_tuple_parameter(ast, name) + pass + + if default: + value = self.traverse(default, indent='') + maybe_show_ast_param_default(self.showast, name, value) + result = '%s=%s' % (name, value) + if result[-2:] == '= ': # default was 'LOAD_CONST None' + result += 'None' + return result + else: + return name + + # MAKE_FUNCTION_... or MAKE_CLOSURE_... + assert node[-1].type.startswith('MAKE_') + + args_node = node[-1] + if isinstance(args_node.attr, tuple): + # positional args are after kwargs + defparams = node[1:args_node.attr[0]+1] + pos_args, kw_args, annotate_argc = args_node.attr + else: + defparams = node[:args_node.attr] + kw_args = 0 + pass + + lambda_index = None + + if lambda_index and isLambda and iscode(node[lambda_index].attr): + assert node[lambda_index].type == 'LOAD_LAMBDA' + code = node[lambda_index].attr + else: + code = codeNode.attr + + assert iscode(code) + code = Code(code, self.scanner, self.currentclass) + + # add defaults values to parameter names + argc = code.co_argcount + paramnames = list(code.co_varnames[:argc]) + + # defaults are for last n parameters, thus reverse + paramnames.reverse(); defparams.reverse() + + try: + ast = self.build_ast(code._tokens, + code._customize, + isLambda = isLambda, + noneInNames = ('None' in code.co_names)) + except ParserError as p: + self.write(str(p)) + self.ERROR = p + return + + kw_pairs = args_node.attr[1] if self.version >= 3.0 else 0 + indent = self.indent + + # build parameters + params = [build_param(ast, name, default) for + name, default in zip_longest(paramnames, defparams, fillvalue=None)] + params.reverse() # back to correct order + + if 4 & code.co_flags: # flag 2 -> variable number of args + params.append('*%s' % code.co_varnames[argc]) + argc += 1 + + # dump parameter list (with default values) + if isLambda: + self.write("lambda ", ", ".join(params)) + else: + self.write("(", ", ".join(params)) + + if kw_args > 0: + if not (4 & code.co_flags): + if argc > 0: + self.write(", *, ") + else: + self.write("*, ") + pass + else: + self.write(", ") + + for n in node: + if n == 'pos_arg': + continue + else: + self.preorder(n) + break + pass + + if 8 & code.co_flags: # flag 3 -> keyword args + if argc > 0: + self.write(', ') + self.write('**%s' % code.co_varnames[argc + kw_pairs]) + + if isLambda: + self.write(": ") + else: + self.println("):") + + if len(code.co_consts) > 0 and code.co_consts[0] is not None and not isLambda: # ugly + # docstring exists, dump it + self.print_docstring(indent, code.co_consts[0]) + + code._tokens = None # save memory + assert ast == 'stmts' + + all_globals = find_all_globals(ast, set()) + for g in ((all_globals & self.mod_globs) | find_globals(ast, set())): + self.println(self.indent, 'global ', g) + self.mod_globs -= all_globals + has_none = 'None' in code.co_names + rn = has_none and not find_none(ast) + self.gen_source(ast, code.co_name, code._customize, isLambda=isLambda, + returnNone=rn) + code._tokens = None; code._customize = None # save memory + + +def make_function3(self, node, isLambda, nested=1, codeNode=None): + """Dump function definition, doc string, and function body.""" + + # FIXME: call make_function3 if we are self.version >= 3.0 + # and then simplify the below. + + def build_param(ast, name, default): + """build parameters: + - handle defaults + - handle format tuple parameters + """ + if default: + value = self.traverse(default, indent='') + maybe_show_ast_param_default(self.showast, name, value) + result = '%s=%s' % (name, value) + if result[-2:] == '= ': # default was 'LOAD_CONST None' + result += 'None' + return result + else: + return name + + # MAKE_FUNCTION_... or MAKE_CLOSURE_... + assert node[-1].type.startswith('MAKE_') + + args_node = node[-1] + if isinstance(args_node.attr, tuple): + if self.version <= 3.3: + # positional args are after kwargs + defparams = node[1:args_node.attr[0]+1] + else: + # positional args are before kwargs + defparams = node[:args_node.attr[0]] + pos_args, kw_args, annotate_argc = args_node.attr + else: + defparams = node[:args_node.attr] + kw_args = 0 + pass + + if 3.0 <= self.version <= 3.2: + lambda_index = -2 + elif 3.03 <= self.version: + lambda_index = -3 + else: + lambda_index = None + + if lambda_index and isLambda and iscode(node[lambda_index].attr): + assert node[lambda_index].type == 'LOAD_LAMBDA' + code = node[lambda_index].attr + else: + code = codeNode.attr + + assert iscode(code) + code = Code(code, self.scanner, self.currentclass) + + # add defaults values to parameter names + argc = code.co_argcount + paramnames = list(code.co_varnames[:argc]) + + # defaults are for last n parameters, thus reverse + if not 3.0 <= self.version <= 3.2: + paramnames.reverse(); defparams.reverse() + + try: + ast = self.build_ast(code._tokens, + code._customize, + isLambda = isLambda, + noneInNames = ('None' in code.co_names)) + except ParserError as p: + self.write(str(p)) + self.ERROR = p + return + + kw_pairs = args_node.attr[1] if self.version >= 3.0 else 0 + indent = self.indent + + # build parameters + if self.version != 3.2: + params = [build_param(ast, name, default) for + name, default in zip_longest(paramnames, defparams, fillvalue=None)] + params.reverse() # back to correct order + + if 4 & code.co_flags: # flag 2 -> variable number of args + if self.version > 3.0: + params.append('*%s' % code.co_varnames[argc + kw_pairs]) + else: + params.append('*%s' % code.co_varnames[argc]) + argc += 1 + + # dump parameter list (with default values) + if isLambda: + self.write("lambda ", ", ".join(params)) + else: + self.write("(", ", ".join(params)) + # self.println(indent, '#flags:\t', int(code.co_flags)) + + else: + if isLambda: + self.write("lambda ") + else: + self.write("(") + pass + + last_line = self.f.getvalue().split("\n")[-1] + l = len(last_line) + indent = ' ' * l + line_number = self.line_number + + if 4 & code.co_flags: # flag 2 -> variable number of args + self.write('*%s' % code.co_varnames[argc + kw_pairs]) + argc += 1 + + i = len(paramnames) - len(defparams) + self.write(", ".join(paramnames[:i])) + suffix = ', ' if i > 0 else '' + for n in node: + if n == 'pos_arg': + self.write(suffix) + self.write(paramnames[i] + '=') + i += 1 + self.preorder(n) + if (line_number != self.line_number): + suffix = ",\n" + indent + line_number = self.line_number + else: + suffix = ', ' + + if kw_args > 0: + if not (4 & code.co_flags): + if argc > 0: + self.write(", *, ") + else: + self.write("*, ") + pass + else: + self.write(", ") + + if not 3.0 <= self.version <= 3.2: + for n in node: + if n == 'pos_arg': + continue + elif self.version >= 3.4 and n.type != 'kwargs': + continue + else: + self.preorder(n) + break + else: + kwargs = node[0] + last = len(kwargs)-1 + i = 0 + for n in node[0]: + if n == 'kwarg': + self.write('%s=' % n[0].pattr) + self.preorder(n[1]) + if i < last: + self.write(', ') + i += 1 + pass + pass + pass + pass + + if 8 & code.co_flags: # flag 3 -> keyword args + if argc > 0: + self.write(', ') + self.write('**%s' % code.co_varnames[argc + kw_pairs]) + + if isLambda: + self.write(": ") + else: + self.println("):") + + if len(code.co_consts) > 0 and code.co_consts[0] is not None and not isLambda: # ugly + # docstring exists, dump it + self.print_docstring(indent, code.co_consts[0]) + + code._tokens = None # save memory + assert ast == 'stmts' + + all_globals = find_all_globals(ast, set()) + for g in ((all_globals & self.mod_globs) | find_globals(ast, set())): + self.println(self.indent, 'global ', g) + self.mod_globs -= all_globals + has_none = 'None' in code.co_names + rn = has_none and not find_none(ast) + self.gen_source(ast, code.co_name, code._customize, isLambda=isLambda, + returnNone=rn) + code._tokens = None; code._customize = None # save memory diff --git a/uncompyle6/semantics/parser_error.py b/uncompyle6/semantics/parser_error.py new file mode 100644 index 00000000..a0a24ecd --- /dev/null +++ b/uncompyle6/semantics/parser_error.py @@ -0,0 +1,11 @@ +import uncompyle6.parser as python_parser +class ParserError(python_parser.ParserError): + def __init__(self, error, tokens): + self.error = error # previous exception + self.tokens = tokens + + def __str__(self): + lines = ['--- This code section failed: ---'] + lines.extend([str(i) for i in self.tokens]) + lines.extend( ['', str(self.error)] ) + return '\n'.join(lines) diff --git a/uncompyle6/semantics/pysource.py b/uncompyle6/semantics/pysource.py index 511dae89..8b0eef72 100644 --- a/uncompyle6/semantics/pysource.py +++ b/uncompyle6/semantics/pysource.py @@ -79,10 +79,13 @@ from spark_parser import GenericASTTraversal, DEFAULT_DEBUG as PARSER_DEFAULT_DE from uncompyle6.scanner import Code, get_scanner from uncompyle6.scanners.tok import Token, NoneToken import uncompyle6.parser as python_parser +from uncompyle6.semantics.make_function import ( + make_function2, make_function3, make_function3_annotate, find_globals) +from uncompyle6.semantics.parser_error import ParserError + from uncompyle6.show import ( maybe_show_asm, maybe_show_ast, - maybe_show_ast_param_default, ) if PYTHON3: @@ -430,45 +433,6 @@ def is_docstring(node): except: return False -class ParserError(python_parser.ParserError): - def __init__(self, error, tokens): - self.error = error # previous exception - self.tokens = tokens - - def __str__(self): - lines = ['--- This code section failed: ---'] - lines.extend([str(i) for i in self.tokens]) - lines.extend( ['', str(self.error)] ) - return '\n'.join(lines) - -def find_globals(node, globs): - """Find globals in this statement.""" - for n in node: - if isinstance(n, AST): - globs = find_globals(n, globs) - elif n.type in ('STORE_GLOBAL', 'DELETE_GLOBAL'): - globs.add(n.pattr) - return globs - -def find_all_globals(node, globs): - """Find globals in this statement.""" - for n in node: - if isinstance(n, AST): - globs = find_all_globals(n, globs) - elif n.type in ('STORE_GLOBAL', 'DELETE_GLOBAL', 'LOAD_GLOBAL'): - globs.add(n.pattr) - return globs - -def find_none(node): - for n in node: - if isinstance(n, AST): - if not n in ('return_stmt', 'return_if_stmt'): - if find_none(n): - return True - elif n.type == 'LOAD_CONST' and n.pattr is None: - return True - return False - class SourceWalkerError(Exception): def __init__(self, errmsg): self.errmsg = errmsg @@ -611,45 +575,74 @@ class SourceWalker(GenericASTTraversal, object): 'comp_for': ( ' for %c in %c%c', 2, 0, 3 ), }) - - ########################## - # Python 3.2 and 3.3 only - ########################## - if 3.2 <= version <= 3.3: + if version >= 3.0: TABLE_DIRECT.update({ + 'funcdef_annotate': ( '\n\n%|def %c%c\n', -1, 0), 'store_locals': ( '%|# inspect.currentframe().f_locals = __locals__\n', ), }) - elif version >= 3.4: - ######################## - # Python 3.4+ Additions - ####################### - TABLE_DIRECT.update({ - 'LOAD_CLASSDEREF': ( '%{pattr}', ), - }) - if version >= 3.6: + + def n_mkfunc_annotate(node): + + if self.version >= 3.3 or node[-2] == 'kwargs': + # LOAD_CONST code object .. + # LOAD_CONST 'x0' if >= 3.3 + # EXTENDED_ARG + # MAKE_FUNCTION .. + code = node[-4] + elif node[-3] == 'expr': + code = node[-3][0] + else: + # LOAD_CONST code object .. + # MAKE_FUNCTION .. + code = node[-3] + + self.indentMore() + annotate_last = -4 if self.version == 3.1 else -5 + + # FIXME: handle and pass full annotate args + make_function3_annotate(self, node, isLambda=False, + codeNode=code, annotate_last=annotate_last) + + if len(self.param_stack) > 1: + self.write('\n\n') + else: + self.write('\n\n\n') + self.indentLess() + self.prune() # stop recursing + self.n_mkfunc_annotate = n_mkfunc_annotate + + + if version >= 3.4: ######################## - # Python 3.6+ Additions + # Python 3.4+ Additions ####################### TABLE_DIRECT.update({ - 'fstring_expr': ( "{%c%{conversion}}", 0), - 'fstring_single': ( "f'{%c%{conversion}}'", 0), - 'fstring_multi': ( "f'%c'", 0), - }) + 'LOAD_CLASSDEREF': ( '%{pattr}', ), + }) + if version >= 3.6: + ######################## + # Python 3.6+ Additions + ####################### + TABLE_DIRECT.update({ + 'fstring_expr': ( "{%c%{conversion}}", 0), + 'fstring_single': ( "f'{%c%{conversion}}'", 0), + 'fstring_multi': ( "f'%c'", 0), + }) - FSTRING_CONVERSION_MAP = {1: '!s', 2: '!r', 3: '!a'} - def f_conversion(node): - node.conversion = FSTRING_CONVERSION_MAP.get(node.data[1].attr, '') + FSTRING_CONVERSION_MAP = {1: '!s', 2: '!r', 3: '!a'} + def f_conversion(node): + node.conversion = FSTRING_CONVERSION_MAP.get(node.data[1].attr, '') - def n_fstring_expr(node): - f_conversion(node) - self.default(node) - self.n_fstring_expr = n_fstring_expr + def n_fstring_expr(node): + f_conversion(node) + self.default(node) + self.n_fstring_expr = n_fstring_expr - def n_fstring_single(node): - f_conversion(node) - self.default(node) + def n_fstring_single(node): + f_conversion(node) + self.default(node) - self.n_fstring_single = n_fstring_single + self.n_fstring_single = n_fstring_single return @@ -1157,6 +1150,13 @@ class SourceWalker(GenericASTTraversal, object): self.indentLess() self.prune() # stop recursing + def make_function(self, node, isLambda, nested=1, + codeNode=None, annotate=None): + if self.version >= 3.0: + make_function3(self, node, isLambda, nested, codeNode) + else: + make_function2(self, node, isLambda, nested, codeNode) + def n_mklambda(self, node): self.make_function(node, isLambda=True, codeNode=node[-2]) self.prune() # stop recursing @@ -2103,190 +2103,6 @@ class SourceWalker(GenericASTTraversal, object): # return self.traverse(node[1]) raise Exception("Can't find tuple parameter " + name) - def make_function(self, node, isLambda, nested=1, codeNode=None): - """Dump function defintion, doc string, and function body.""" - - def build_param(ast, name, default): - """build parameters: - - handle defaults - - handle format tuple parameters - """ - if self.version < 3.0: - # if formal parameter is a tuple, the paramater name - # starts with a dot (eg. '.1', '.2') - if name.startswith('.'): - # replace the name with the tuple-string - name = self.get_tuple_parameter(ast, name) - pass - pass - - if default: - value = self.traverse(default, indent='') - maybe_show_ast_param_default(self.showast, name, value) - result = '%s=%s' % (name, value) - if result[-2:] == '= ': # default was 'LOAD_CONST None' - result += 'None' - return result - else: - return name - - # MAKE_FUNCTION_... or MAKE_CLOSURE_... - assert node[-1].type.startswith('MAKE_') - - args_node = node[-1] - if isinstance(args_node.attr, tuple): - if self.version <= 3.3: - # positional args are after kwargs - defparams = node[1:args_node.attr[0]+1] - else: - # positional args are before kwargs - defparams = node[:args_node.attr[0]] - pos_args, kw_args, annotate_args = args_node.attr - else: - defparams = node[:args_node.attr] - kw_args = 0 - pass - - if 3.0 <= self.version <= 3.2: - lambda_index = -2 - elif 3.03 <= self.version: - lambda_index = -3 - else: - lambda_index = None - - if lambda_index and isLambda and iscode(node[lambda_index].attr): - assert node[lambda_index].type == 'LOAD_LAMBDA' - code = node[lambda_index].attr - else: - code = codeNode.attr - - assert iscode(code) - code = Code(code, self.scanner, self.currentclass) - - # add defaults values to parameter names - argc = code.co_argcount - paramnames = list(code.co_varnames[:argc]) - - # defaults are for last n parameters, thus reverse - if not 3.0 <= self.version <= 3.2: - paramnames.reverse(); defparams.reverse() - - try: - ast = self.build_ast(code._tokens, - code._customize, - isLambda = isLambda, - noneInNames = ('None' in code.co_names)) - except ParserError as p: - self.write(str(p)) - self.ERROR = p - return - - kw_pairs = args_node.attr[1] if self.version >= 3.0 else 0 - indent = self.indent - - # build parameters - if not 3.0 <= self.version <= 3.2: - params = [build_param(ast, name, default) for - name, default in zip_longest(paramnames, defparams, fillvalue=None)] - params.reverse() # back to correct order - - if 4 & code.co_flags: # flag 2 -> variable number of args - if self.version > 3.0: - params.append('*%s' % code.co_varnames[argc + kw_pairs]) - else: - params.append('*%s' % code.co_varnames[argc]) - argc += 1 - - # dump parameter list (with default values) - if isLambda: - self.write("lambda ", ", ".join(params)) - else: - self.write("(", ", ".join(params)) - # self.println(indent, '#flags:\t', int(code.co_flags)) - - else: - if isLambda: - self.write("lambda ") - else: - self.write("(") - - if 4 & code.co_flags: # flag 2 -> variable number of args - self.write('*%s' % code.co_varnames[argc + kw_pairs]) - argc += 1 - - i = len(paramnames) - len(defparams) - self.write(",".join(paramnames[:i])) - suffix = ', ' if i > 0 else '' - for n in node: - if n == 'pos_arg': - self.write(suffix) - self.write(paramnames[i] + '=') - i += 1 - self.preorder(n) - suffix = ', ' - - if kw_args > 0: - if not (4 & code.co_flags): - if argc > 0: - self.write(", *, ") - else: - self.write("*, ") - pass - else: - self.write(", ") - - if not 3.0 <= self.version <= 3.2: - for n in node: - if n == 'pos_arg': - continue - elif self.version >= 3.4 and n.type != 'kwargs': - continue - else: - self.preorder(n) - break - else: - kwargs = node[0] - last = len(kwargs)-1 - i = 0 - for n in node[0]: - if n == 'kwarg': - self.write('%s=' % n[0].pattr) - self.preorder(n[1]) - if i < last: - self.write(', ') - i += 1 - pass - pass - pass - pass - - if 8 & code.co_flags: # flag 3 -> keyword args - if argc > 0: - self.write(', ') - self.write('**%s' % code.co_varnames[argc + kw_pairs]) - - if isLambda: - self.write(": ") - else: - self.println("):") - - if len(code.co_consts)>0 and code.co_consts[0] is not None and not isLambda: # ugly - # docstring exists, dump it - self.print_docstring(indent, code.co_consts[0]) - - code._tokens = None # save memory - assert ast == 'stmts' - - all_globals = find_all_globals(ast, set()) - for g in ((all_globals & self.mod_globs) | find_globals(ast, set())): - self.println(self.indent, 'global ', g) - self.mod_globs -= all_globals - has_none = 'None' in code.co_names - rn = has_none and not find_none(ast) - self.gen_source(ast, code.co_name, code._customize, isLambda=isLambda, - returnNone=rn) - code._tokens = None; code._customize = None # save memory - def build_class(self, code): """Dump class definition, doc string and class body.""" @@ -2448,7 +2264,7 @@ class SourceWalker(GenericASTTraversal, object): return MAP.get(node, MAP_DIRECT) -def deparse_code(version, co, out=sys.stdout, showasm=False, showast=False, +def deparse_code(version, co, out=sys.stdout, showasm=None, showast=False, showgrammar=False, code_objects={}, compile_mode='exec', is_pypy=False): """ ingests and deparses a given code block 'co' @@ -2458,8 +2274,7 @@ def deparse_code(version, co, out=sys.stdout, showasm=False, showast=False, # store final output stream for case of error scanner = get_scanner(version, is_pypy=is_pypy) - tokens, customize = scanner.ingest(co, code_objects=code_objects) - maybe_show_asm(showasm, tokens) + tokens, customize = scanner.ingest(co, code_objects=code_objects, show_asm=showasm) debug_parser = dict(PARSER_DEFAULT_DEBUG) if showgrammar: @@ -2507,8 +2322,8 @@ if __name__ == '__main__': def deparse_test(co): "This is a docstring" sys_version = sys.version_info.major + (sys.version_info.minor / 10.0) - deparsed = deparse_code(sys_version, co, showasm=True, showast=True) - # deparsed = deparse_code(sys_version, co, showasm=False, showast=False, + deparsed = deparse_code(sys_version, co, showasm='after', showast=True) + # deparsed = deparse_code(sys_version, co, showasm=None, showast=False, # showgrammar=True) print(deparsed.text) return diff --git a/uncompyle6/version.py b/uncompyle6/version.py index cdeaaed2..b539e846 100644 --- a/uncompyle6/version.py +++ b/uncompyle6/version.py @@ -1,3 +1,3 @@ # This file is suitable for sourcing inside bash as # well as importing into Python -VERSION='2.9.2' +VERSION='2.9.4'