From abb61a4d7d21b12a86ecc2270275ef99a018e6be Mon Sep 17 00:00:00 2001 From: rocky Date: Mon, 24 Oct 2016 02:16:23 -0400 Subject: [PATCH 01/23] Fix some Python 3.1 bugs --- uncompyle6/parsers/parse3.py | 2 +- uncompyle6/scanners/scanner3.py | 3 ++- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/uncompyle6/parsers/parse3.py b/uncompyle6/parsers/parse3.py index 7154c68f..1eeb9ae1 100644 --- a/uncompyle6/parsers/parse3.py +++ b/uncompyle6/parsers/parse3.py @@ -577,7 +577,7 @@ class Python3Parser(PythonParser): self.add_unique_rule(rule, opname, token.attr, customize) rule = "mapexpr ::= %s %s" % (opname, kvlist_n) self.add_unique_rule(rule, opname, token.attr, customize) - elif opname_base in ('UNPACK_EX'): + elif opname_base in ('UNPACK_EX',): before_count, after_count = token.attr rule = 'unpack ::= ' + opname + ' designator' * (before_count + after_count + 1) self.add_unique_rule(rule, opname, token.attr, customize) diff --git a/uncompyle6/scanners/scanner3.py b/uncompyle6/scanners/scanner3.py index c8d0a310..beff574e 100644 --- a/uncompyle6/scanners/scanner3.py +++ b/uncompyle6/scanners/scanner3.py @@ -249,7 +249,7 @@ class Scanner3(Scanner): opname = '%s_N%d' % (opname, name_pair_args) pass if annotate_args > 0: - opname = '%s_A_%d' % [opname, annotate_args] + opname = '%s_A_%d' % (opname, annotate_args) pass opname = '%s_%d' % (opname, pos_args) pattr = ("%d positional, %d keyword pair, %d annotated" % @@ -286,6 +286,7 @@ class Scanner3(Scanner): pattr = "%d before vararg, %d after" % (before_args, after_args) argval = (before_args, after_args) opname = '%s_%d+%d' % (opname, before_args, after_args) + elif op == self.opc.JUMP_ABSOLUTE: # Further classify JUMP_ABSOLUTE into backward jumps # which are used in loops, and "CONTINUE" jumps which From 6319d33fa02aec2c314c38cf58adb7ae3db679c2 Mon Sep 17 00:00:00 2001 From: rocky Date: Mon, 24 Oct 2016 08:23:30 -0400 Subject: [PATCH 02/23] Python 3.1 compile bug. DRY Python 3.x rules ... via inheritance --- uncompyle6/parsers/parse3.py | 38 +++++++++++++++++------------------ uncompyle6/parsers/parse34.py | 2 +- uncompyle6/parsers/parse35.py | 6 +++--- 3 files changed, 22 insertions(+), 24 deletions(-) diff --git a/uncompyle6/parsers/parse3.py b/uncompyle6/parsers/parse3.py index 1eeb9ae1..0356a99e 100644 --- a/uncompyle6/parsers/parse3.py +++ b/uncompyle6/parsers/parse3.py @@ -360,10 +360,9 @@ class Python3Parser(PythonParser): # Python 3.4+ expr ::= LOAD_CLASSDEREF + binary_subscr2 ::= expr expr DUP_TOP_TWO BINARY_SUBSCR # Python3 drops slice0..slice3 - # In Python 2, DUP_TOP_TWO is DUP_TOPX_2 - binary_subscr2 ::= expr expr DUP_TOP_TWO BINARY_SUBSCR ''' @staticmethod @@ -672,24 +671,6 @@ class Python3Parser(PythonParser): return -class Python31Parser(Python3Parser): - - def p_31(self, args): - """ - # Store locals is only in Python 3.0 to 3.3 - stmt ::= store_locals - store_locals ::= LOAD_FAST STORE_LOCALS - """ - -class Python32Parser(Python3Parser): - - def p_32(self, args): - """ - # Store locals is only in Python 3.0 to 3.3 - stmt ::= store_locals - store_locals ::= LOAD_FAST STORE_LOCALS - """ - class Python33Parser(Python3Parser): def p_33(self, args): """ @@ -702,6 +683,23 @@ class Python33Parser(Python3Parser): yield_from ::= expr expr YIELD_FROM """ +class Python32Parser(Python3Parser): + def p_32on(self, args): + """ + # In Python 3.2+, DUP_TOPX is DUP_TOP_TWO + binary_subscr2 ::= expr expr DUP_TOP_TWO BINARY_SUBSCR + stmt ::= store_locals + store_locals ::= LOAD_FAST STORE_LOCALS + """ + pass + +class Python31Parser(Python32Parser): + + def p_31(self, args): + """ + binary_subscr2 ::= expr expr DUP_TOPX BINARY_SUBSCR + """ + class Python3ParserSingle(Python3Parser, PythonParserSingle): pass diff --git a/uncompyle6/parsers/parse34.py b/uncompyle6/parsers/parse34.py index 1ef1f7b8..cf094af6 100644 --- a/uncompyle6/parsers/parse34.py +++ b/uncompyle6/parsers/parse34.py @@ -1,6 +1,6 @@ # Copyright (c) 2016 Rocky Bernstein """ -spark grammar differences over Python3 for Python 3.4.2. +spark grammar differences over Python 3.2 for Python 3.4 """ from uncompyle6.parser import PythonParserSingle diff --git a/uncompyle6/parsers/parse35.py b/uncompyle6/parsers/parse35.py index 8d9bb1f8..39895eac 100644 --- a/uncompyle6/parsers/parse35.py +++ b/uncompyle6/parsers/parse35.py @@ -1,14 +1,14 @@ # Copyright (c) 2016 Rocky Bernstein """ -spark grammar differences over Python3 for Python 3.5. +spark grammar differences over Python 3.2 for Python 3.5. """ from __future__ import print_function from uncompyle6.parser import PythonParserSingle from spark_parser import DEFAULT_DEBUG as PARSER_DEFAULT_DEBUG -from uncompyle6.parsers.parse3 import Python3Parser +from uncompyle6.parsers.parse3 import Python32Parser -class Python35Parser(Python3Parser): +class Python35Parser(Python32Parser): def __init__(self, debug_parser=PARSER_DEFAULT_DEBUG): super(Python35Parser, self).__init__(debug_parser) From aff920d87b087a80ae737bd8c4bb3a2b8268ab94 Mon Sep 17 00:00:00 2001 From: rocky Date: Mon, 24 Oct 2016 20:47:12 -0400 Subject: [PATCH 03/23] Python 3.1 "with" statement bug --- test/bytecode_3.1/07_withstmt_fn.pyc | Bin 0 -> 467 bytes test/simple_source/stmts/07_withstmt_fn.py | 2 +- uncompyle6/parsers/parse3.py | 8 ++++++++ 3 files changed, 9 insertions(+), 1 deletion(-) create mode 100644 test/bytecode_3.1/07_withstmt_fn.pyc diff --git a/test/bytecode_3.1/07_withstmt_fn.pyc b/test/bytecode_3.1/07_withstmt_fn.pyc new file mode 100644 index 0000000000000000000000000000000000000000..f8e088639e21199eae8cfc1f7ba5a710d1a18e82 GIT binary patch literal 467 zcmZvZF;BxV5QWc4TAC6n#KOYR1u0@^K@2PiF))LGp;anmmE_vG#CBxIl(P2M@EcjU zBvoM`ieKJ4Tc6MILv+yn{_2fW`13^MZ)sRXlF`c`2S^(4LC!#Wka)n*gAG8i^dSjg z*{TBj90CF?+hCq*vj>?ESpe99jSq~FTVe=F2qR)+0FwY7$7G-57HuhLznh`4{ Date: Mon, 24 Oct 2016 20:49:05 -0400 Subject: [PATCH 04/23] Add python 3.1 bytecode testing --- test/Makefile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/Makefile b/test/Makefile index 6e74fc3e..b10baebd 100644 --- a/test/Makefile +++ b/test/Makefile @@ -62,7 +62,7 @@ check-bytecode-2: #: Check deparsing bytecode 3.x only check-bytecode-3: - $(PYTHON) test_pythonlib.py --bytecode-3.2 --bytecode-3.3 \ + $(PYTHON) test_pythonlib.py --bytecode-3.1 --bytecode-3.2 --bytecode-3.3 \ --bytecode-3.4 --bytecode-3.5 --bytecode-pypy3.2 #: Check deparsing bytecode that works running Python 2 and Python 3 From 7a10917857eb927e13ab18453f70128ef25b277d Mon Sep 17 00:00:00 2001 From: rocky Date: Tue, 25 Oct 2016 02:03:50 -0400 Subject: [PATCH 05/23] Handle Python 3.1 "with ... as" statement --- test/bytecode_3.1/04_withas.pyc | Bin 0 -> 388 bytes uncompyle6/parsers/parse3.py | 20 ++++++++++++++++---- 2 files changed, 16 insertions(+), 4 deletions(-) create mode 100644 test/bytecode_3.1/04_withas.pyc diff --git a/test/bytecode_3.1/04_withas.pyc b/test/bytecode_3.1/04_withas.pyc new file mode 100644 index 0000000000000000000000000000000000000000..d9c2af120e05711a065ee2458ef7820091deb409 GIT binary patch literal 388 zcmb79O-sW-5Ph3^sKp-o3-p+i_3J1ic&|tf6{QRznQCf2N_LkTNqf;B?#wDEcyi!9 z-pd=a72Fp>rgvxiDHfNN|wuI5cGJq z_!g&&u*!ATWM`hQS<~S$N^!p`Vn0L=DCBvTAZ)@>NLDkWbxMO&S0I literal 0 HcmV?d00001 diff --git a/uncompyle6/parsers/parse3.py b/uncompyle6/parsers/parse3.py index f868d2ca..e9d55199 100644 --- a/uncompyle6/parsers/parse3.py +++ b/uncompyle6/parsers/parse3.py @@ -698,10 +698,22 @@ class Python31Parser(Python32Parser): def p_31(self, args): """ binary_subscr2 ::= expr expr DUP_TOPX BINARY_SUBSCR - setupwith ::= DUP_TOP LOAD_ATTR store LOAD_ATTR CALL_FUNCTION_0 POP_TOP - withstmt ::= expr setupwith SETUP_FINALLY suite_stmts_opt - POP_BLOCK LOAD_CONST COME_FROM_FINALLY - load del_stmt WITH_CLEANUP END_FINALLY + + setupwith ::= DUP_TOP LOAD_ATTR store LOAD_ATTR CALL_FUNCTION_0 POP_TOP + setupwithas ::= DUP_TOP LOAD_ATTR store LOAD_ATTR CALL_FUNCTION_0 store + withstmt ::= expr setupwith SETUP_FINALLY + suite_stmts_opt + POP_BLOCK LOAD_CONST COME_FROM_FINALLY + load del_stmt WITH_CLEANUP END_FINALLY + + # Keeps Python 3.1 withas desigator in the same position as it is in other version + setupwithas31 ::= setupwithas SETUP_FINALLY load del_stmt + + withasstmt ::= expr setupwithas31 designator + suite_stmts_opt + POP_BLOCK LOAD_CONST COME_FROM_FINALLY + load del_stmt WITH_CLEANUP END_FINALLY + store ::= STORE_FAST store ::= STORE_NAME load ::= LOAD_FAST From 8feb472d510976f9e36854373bea1bac2e6ff86b Mon Sep 17 00:00:00 2001 From: rocky Date: Tue, 25 Oct 2016 21:46:46 -0400 Subject: [PATCH 06/23] Split out Python 3.1 parser from rest. __pkginfo__.py: use Python 3.1 bytecode fixes --- __pkginfo__.py | 2 +- uncompyle6/parser.py | 3 ++- uncompyle6/parsers/parse3.py | 30 -------------------------- uncompyle6/parsers/parse31.py | 40 +++++++++++++++++++++++++++++++++++ 4 files changed, 43 insertions(+), 32 deletions(-) create mode 100644 uncompyle6/parsers/parse31.py diff --git a/__pkginfo__.py b/__pkginfo__.py index 2c211f66..72659671 100644 --- a/__pkginfo__.py +++ b/__pkginfo__.py @@ -38,7 +38,7 @@ entry_points={ ]} ftp_url = None install_requires = ['spark-parser >= 1.4.0', - 'xdis >= 3.1.0'] + 'xdis >= 3.2.0'] license = 'MIT' mailing_list = 'python-debugger@googlegroups.com' modname = 'uncompyle6' diff --git a/uncompyle6/parser.py b/uncompyle6/parser.py index 1ab82008..5e6a0329 100644 --- a/uncompyle6/parser.py +++ b/uncompyle6/parser.py @@ -623,7 +623,8 @@ def get_python_parser( import uncompyle6.parsers.parse3 as parse3 if version == 3.1: if compile_mode == 'exec': - p = parse3.Python31Parser(debug_parser) + import uncompyle6.parsers.parse31 as parse31 + p = parse31.Python31Parser(debug_parser) else: p = parse3.Python31ParserSingle(debug_parser) elif version == 3.2: diff --git a/uncompyle6/parsers/parse3.py b/uncompyle6/parsers/parse3.py index e9d55199..da218ca4 100644 --- a/uncompyle6/parsers/parse3.py +++ b/uncompyle6/parsers/parse3.py @@ -693,40 +693,10 @@ class Python32Parser(Python3Parser): """ pass -class Python31Parser(Python32Parser): - - def p_31(self, args): - """ - binary_subscr2 ::= expr expr DUP_TOPX BINARY_SUBSCR - - setupwith ::= DUP_TOP LOAD_ATTR store LOAD_ATTR CALL_FUNCTION_0 POP_TOP - setupwithas ::= DUP_TOP LOAD_ATTR store LOAD_ATTR CALL_FUNCTION_0 store - withstmt ::= expr setupwith SETUP_FINALLY - suite_stmts_opt - POP_BLOCK LOAD_CONST COME_FROM_FINALLY - load del_stmt WITH_CLEANUP END_FINALLY - - # Keeps Python 3.1 withas desigator in the same position as it is in other version - setupwithas31 ::= setupwithas SETUP_FINALLY load del_stmt - - withasstmt ::= expr setupwithas31 designator - suite_stmts_opt - POP_BLOCK LOAD_CONST COME_FROM_FINALLY - load del_stmt WITH_CLEANUP END_FINALLY - - store ::= STORE_FAST - store ::= STORE_NAME - load ::= LOAD_FAST - load ::= LOAD_NAME - """ - class Python3ParserSingle(Python3Parser, PythonParserSingle): pass -class Python31ParserSingle(Python31Parser, PythonParserSingle): - pass - class Python32ParserSingle(Python32Parser, PythonParserSingle): pass diff --git a/uncompyle6/parsers/parse31.py b/uncompyle6/parsers/parse31.py new file mode 100644 index 00000000..1e615f9d --- /dev/null +++ b/uncompyle6/parsers/parse31.py @@ -0,0 +1,40 @@ +# Copyright (c) 2016 Rocky Bernstein +""" +spark grammar differences over Python 3.2 for Python 3.1. +""" +from __future__ import print_function + +from uncompyle6.parser import PythonParserSingle +from uncompyle6.parsers.parse3 import Python32Parser + +class Python31Parser(Python32Parser): + + def p_31(self, args): + """ + binary_subscr2 ::= expr expr DUP_TOPX BINARY_SUBSCR + + setupwith ::= DUP_TOP LOAD_ATTR store LOAD_ATTR CALL_FUNCTION_0 POP_TOP + setupwithas ::= DUP_TOP LOAD_ATTR store LOAD_ATTR CALL_FUNCTION_0 store + withstmt ::= expr setupwith SETUP_FINALLY + suite_stmts_opt + POP_BLOCK LOAD_CONST COME_FROM_FINALLY + load del_stmt WITH_CLEANUP END_FINALLY + + # Keeps Python 3.1 withas desigator in the same position as it is in other version + setupwithas31 ::= setupwithas SETUP_FINALLY load del_stmt + + withasstmt ::= expr setupwithas31 designator + suite_stmts_opt + POP_BLOCK LOAD_CONST COME_FROM_FINALLY + load del_stmt WITH_CLEANUP END_FINALLY + + store ::= STORE_FAST + store ::= STORE_NAME + load ::= LOAD_FAST + load ::= LOAD_NAME + + funcdef ::= mkfunc designator + """ + +class Python31ParserSingle(Python31Parser, PythonParserSingle): + pass From 7daec3352c4804a8b116431b78a98af5cdc26e90 Mon Sep 17 00:00:00 2001 From: rocky Date: Wed, 26 Oct 2016 08:20:19 -0400 Subject: [PATCH 07/23] Start to attack Python 3.1 def() -> xx construct Start to localize make_function routines by Python version --- test/bytecode_3.1/04_def_attr.pyc-notyet | Bin 0 -> 430 bytes test/simple_source/bug31/04_def_attr.py | 6 + uncompyle6/parsers/parse31.py | 16 ++- uncompyle6/scanner.py | 2 +- uncompyle6/semantics/pysource.py | 169 ++++++++++++++++++++++- 5 files changed, 187 insertions(+), 6 deletions(-) create mode 100644 test/bytecode_3.1/04_def_attr.pyc-notyet create mode 100644 test/simple_source/bug31/04_def_attr.py diff --git a/test/bytecode_3.1/04_def_attr.pyc-notyet b/test/bytecode_3.1/04_def_attr.pyc-notyet new file mode 100644 index 0000000000000000000000000000000000000000..cae0c27f4d4fa8fe153098f56c8798d4b0d5aa74 GIT binary patch literal 430 zcmb79Jx>EM4E23*6a*5#fhiLO!H5uoiRC)b0kNFg#Hmh7lT#84*xC69@K>?Zfv#AN z^Paz6vN$0#k!&;H!qu4qh3z@T@v+riw!(flrHv zRYE0+!XsviULP_SlUr(!d{WCH1MiSbzndvJe9uZcNthiS7O)KlNglZ-YcY;~Lo;UR zbudCi9%BqKkqmo$bEC(CwF?QYk&i(Mbj~P^+sWaO0!5AUHEpi1=9f2vK|8E+ju+EM v(G3&&M%rr`Z~CPYR!V(f0eR#y(hf-TpUl6sX1wQ?KS>Vnik>X)s;>G4ctuno literal 0 HcmV?d00001 diff --git a/test/simple_source/bug31/04_def_attr.py b/test/simple_source/bug31/04_def_attr.py new file mode 100644 index 00000000..d970aa44 --- /dev/null +++ b/test/simple_source/bug31/04_def_attr.py @@ -0,0 +1,6 @@ +# Bug in 3.1 _pyio.py. The -> "IOBase" is problematic + +def open(file, mode = "r", buffering = None, + encoding = None, errors = None, + newline = None, closefd = True) -> "IOBase": + return text diff --git a/uncompyle6/parsers/parse31.py b/uncompyle6/parsers/parse31.py index 1e615f9d..0356a224 100644 --- a/uncompyle6/parsers/parse31.py +++ b/uncompyle6/parsers/parse31.py @@ -33,8 +33,22 @@ class Python31Parser(Python32Parser): load ::= LOAD_FAST load ::= LOAD_NAME - funcdef ::= mkfunc designator + stmt ::= funcdeftest + funcdeftest ::= mkfunctest designator """ + def add_custom_rules(self, tokens, customize): + super(Python31Parser, self).add_custom_rules(tokens, customize) + for i, token in enumerate(tokens): + opname = token.type + if opname.startswith('MAKE_FUNCTION_A'): + args_pos, args_kw, annotate_args = token.attr + # Check that there are 2 annotated params? + # rule = ('mkfunc2 ::= %s%sEXTENDED_ARG %s' % + # ('pos_arg ' * (args_pos), 'kwargs ' * (annotate_args-1), opname)) + rule = ('mkfunctest ::= %s%sLOAD_CONST EXTENDED_ARG %s' % + (('pos_arg ' * (args_pos)), 'kwargs ', opname)) + self.add_unique_rule(rule, opname, token.attr, customize) + class Python31ParserSingle(Python31Parser, PythonParserSingle): pass diff --git a/uncompyle6/scanner.py b/uncompyle6/scanner.py index 71316d41..2d44490b 100755 --- a/uncompyle6/scanner.py +++ b/uncompyle6/scanner.py @@ -20,7 +20,7 @@ from uncompyle6.scanners.tok import Token # The byte code versions we support PYTHON_VERSIONS = (1.5, 2.1, 2.2, 2.3, 2.4, 2.5, 2.6, 2.7, - 3.1, 3.2, 3.3, 3.4, 3.5, 3.6) + 3.0, 3.1, 3.2, 3.3, 3.4, 3.5, 3.6) # FIXME: DRY if PYTHON3: diff --git a/uncompyle6/semantics/pysource.py b/uncompyle6/semantics/pysource.py index 511dae89..9b120cc8 100644 --- a/uncompyle6/semantics/pysource.py +++ b/uncompyle6/semantics/pysource.py @@ -612,10 +612,171 @@ class SourceWalker(GenericASTTraversal, object): }) - ########################## - # Python 3.2 and 3.3 only - ########################## - if 3.2 <= version <= 3.3: + if 3.1 == version: + ########################## + # Python 3.1 + ########################## + TABLE_DIRECT.update({ + 'funcdeftest': ( '\n\n%|def %c%c\n', -1, 0), + }) + + def make_function31(node, isLambda, nested=1, + codeNode=None, annotate=None): + """Dump function defintion, doc string, and function + body. This code is specialzed for Python 3.1""" + + def build_param(ast, name, default): + """build parameters: + - handle defaults + - handle format tuple parameters + """ + if default: + value = self.traverse(default, indent='') + maybe_show_ast_param_default(self.showast, name, value) + result = '%s=%s' % (name, value) + if result[-2:] == '= ': # default was 'LOAD_CONST None' + result += 'None' + return result + else: + return name + + # MAKE_FUNCTION_... or MAKE_CLOSURE_... + assert node[-1].type.startswith('MAKE_') + + args_node = node[-1] + if isinstance(args_node.attr, tuple): + defparams = node[1:args_node.attr[0]+1] + pos_args, kw_args, annotate_args = args_node.attr + else: + defparams = node[:args_node.attr] + kw_args = 0 + pass + + lambda_index = -2 + + if lambda_index and isLambda and iscode(node[lambda_index].attr): + assert node[lambda_index].type == 'LOAD_LAMBDA' + code = node[lambda_index].attr + else: + code = codeNode.attr + + assert iscode(code) + code = Code(code, self.scanner, self.currentclass) + + # add defaults values to parameter names + argc = code.co_argcount + paramnames = list(code.co_varnames[:argc]) + + try: + ast = self.build_ast(code._tokens, + code._customize, + isLambda = isLambda, + noneInNames = ('None' in code.co_names)) + except ParserError as p: + self.write(str(p)) + self.ERROR = p + return + + kw_pairs = args_node.attr[1] if self.version >= 3.0 else 0 + indent = self.indent + + params = [build_param(ast, name, default) for + name, default in zip_longest(paramnames, defparams, fillvalue=None)] + params.reverse() # back to correct order + + if 4 & code.co_flags: # flag 2 -> variable number of args + if self.version > 3.0: + params.append('*%s' % code.co_varnames[argc + kw_pairs]) + else: + params.append('*%s' % code.co_varnames[argc]) + argc += 1 + + # dump parameter list (with default values) + if isLambda: + self.write("lambda ", ", ".join(params)) + else: + self.write("(", ", ".join(params)) + # self.println(indent, '#flags:\t', int(code.co_flags)) + if kw_args > 0: + if not (4 & code.co_flags): + if argc > 0: + self.write(", *, ") + else: + self.write("*, ") + pass + else: + self.write(", ") + + kwargs = node[0] + last = len(kwargs)-1 + i = 0 + for n in node[0]: + if n == 'kwarg': + self.write('%s=' % n[0].pattr) + self.preorder(n[1]) + if i < last: + self.write(', ') + i += 1 + pass + pass + pass + + if 8 & code.co_flags: # flag 3 -> keyword args + if argc > 0: + self.write(', ') + self.write('**%s' % code.co_varnames[argc + kw_pairs]) + + if isLambda: + self.write(": ") + else: + self.write(')') + if annotate: + self.write(' -> %s' % annotate) + self.println(":") + + if (len(code.co_consts) > 0 and + code.co_consts[0] is not None and not isLambda): # ugly + # docstring exists, dump it + self.print_docstring(indent, code.co_consts[0]) + + code._tokens = None # save memory + assert ast == 'stmts' + + all_globals = find_all_globals(ast, set()) + for g in ((all_globals & self.mod_globs) | find_globals(ast, set())): + self.println(self.indent, 'global ', g) + self.mod_globs -= all_globals + has_none = 'None' in code.co_names + rn = has_none and not find_none(ast) + self.gen_source(ast, code.co_name, code._customize, isLambda=isLambda, + returnNone=rn) + code._tokens = code._customize = None # save memory + + self.make_function31 = make_function31 + + def n_mkfunctest(node): + + self.indentMore() + code = node[-3] + annotate = None + if node[-4][0][0] == 'LOAD_CONST': + annotate = node[-4][0][0].attr + self.make_function31(node, isLambda=False, + codeNode=code, annotate=annotate) + + if len(self.param_stack) > 1: + self.write('\n\n') + else: + self.write('\n\n\n') + self.indentLess() + self.prune() # stop recursing + self.n_mkfunctest = n_mkfunctest + + + elif 3.2 <= version <= 3.3: + ########################## + # Python 3.2 and 3.3 + ########################## TABLE_DIRECT.update({ 'store_locals': ( '%|# inspect.currentframe().f_locals = __locals__\n', ), }) From de65a2c25095b92f9a06d3a93ad7f15150e79679 Mon Sep 17 00:00:00 2001 From: rocky Date: Wed, 26 Oct 2016 08:22:56 -0400 Subject: [PATCH 08/23] Get ready for release 2.9.3 --- ChangeLog | 83 ++++++++++++++++++++++++++++++++++++++++++- NEWS | 15 ++++++++ uncompyle6/version.py | 2 +- 3 files changed, 98 insertions(+), 2 deletions(-) diff --git a/ChangeLog b/ChangeLog index 57d20a1e..a566d9d5 100644 --- a/ChangeLog +++ b/ChangeLog @@ -1,6 +1,87 @@ +2016-10-26 rocky + + * uncompyle6/version.py: Get ready for release 2.9.3 + +2016-10-26 rocky + + * test/simple_source/bug31/04_def_attr.py, + uncompyle6/parsers/parse31.py, uncompyle6/scanner.py, + uncompyle6/semantics/pysource.py: Start to attack Python 3.1 def() + -> xx construct Start to localize make_function routines by Python version + +2016-10-25 rocky + + * __pkginfo__.py, uncompyle6/parser.py, + uncompyle6/parsers/parse3.py, uncompyle6/parsers/parse31.py: Split + out Python 3.1 parser from rest. __pkginfo__.py: use Python 3.1 bytecode fixes + +2016-10-25 rocky + + * uncompyle6/parsers/parse3.py: Handle Python 3.1 "with ... as" + statement + +2016-10-24 rocky + + * test/Makefile: Add python 3.1 bytecode testing + +2016-10-24 rocky + + * test/simple_source/stmts/07_withstmt_fn.py, + uncompyle6/parsers/parse3.py: Python 3.1 "with" statement bug + +2016-10-24 rocky + + * uncompyle6/parsers/parse3.py, uncompyle6/parsers/parse34.py, + uncompyle6/parsers/parse35.py: Python 3.1 compile bug. DRY Python + 3.x rules ... via inheritance + +2016-10-24 rocky + + * uncompyle6/parsers/parse3.py, uncompyle6/scanners/scanner3.py: Fix + some Python 3.1 bugs + +2016-10-22 Daniel Bradburn + + * : Merge pull request #60 from rocky/buildstring Buildstring + +2016-10-22 rocky + + * pytest/test_fstring.py, test/simple_source/bug36/01_fstring.py, + uncompyle6/semantics/pysource.py: Move fstring rules inside a 3.6+ + check + +2016-10-22 rocky + + * : commit d6f7ef4e178e04d9a612d3a6c0b77a008732357f Author: rocky + Date: Fri Oct 21 07:40:35 2016 -0400 + +2016-10-20 moagstar + + * pytest/test_fstring.py, uncompyle6/parsers/parse3.py, + uncompyle6/parsers/parse36.py, uncompyle6/semantics/pysource.py: + further work on supporting single and multiple fstring decompilation + +2016-10-20 rocky + + * uncompyle6/main.py, uncompyle6/scanners/scanner2.py, + uncompyle6/scanners/scanner26.py: DRY Python 2.x unmangle_classname main.py: small typo: Disassembled -> Decompiled + +2016-10-19 moagstar + + * pytest/test_fstring.py, uncompyle6/parsers/parse3.py, + uncompyle6/parsers/parse36.py, uncompyle6/semantics/pysource.py: + urther work on fstrings for python 3.6 - there is a new opcode + build_string which is used to improve fstring performance, but broke + the fstring support in uncompyle + 2016-10-15 rocky - * uncompyle6/version.py: Get ready for release 2.9.2 + * uncompyle6/main.py: Change meta data info in uncompyle6: * Show file size of source when possible, i.e. in Python 3.x * Show full information about python interpreter used to decompile + +2016-10-15 rocky + + * ChangeLog, NEWS, __pkginfo__.py, requirements.txt, + uncompyle6/version.py: Get ready for release 2.9.2 2016-10-14 rocky diff --git a/NEWS b/NEWS index 630f278b..2fd5de75 100644 --- a/NEWS +++ b/NEWS @@ -1,3 +1,18 @@ +uncompyle6 2.9.3 2016-10-26 + +Release forced by incompatiblity change in xdis 3.2.0. + +- Python 3.1 bugs: + * handle "with ... as" + * handle "with" + * Start handling def (...) -> yy (has bugs still) + +- DRY Python 3.x via inheritance +- Python 3.6 work (from Daniel Bradburn) + * Handle 3.6 buildstring + * Handle 3.6 handle single and multiple fstring better + + uncompyle6 2.9.2 2016-10-15 - use source-code line breaks to assist in where to break diff --git a/uncompyle6/version.py b/uncompyle6/version.py index cdeaaed2..a353ce9f 100644 --- a/uncompyle6/version.py +++ b/uncompyle6/version.py @@ -1,3 +1,3 @@ # This file is suitable for sourcing inside bash as # well as importing into Python -VERSION='2.9.2' +VERSION='2.9.3' From 1a38d3d9aaadfc24236791120df68478cb3348f5 Mon Sep 17 00:00:00 2001 From: rocky Date: Wed, 26 Oct 2016 18:36:12 -0400 Subject: [PATCH 09/23] Dependencies stay within 2nd semantic level --- __pkginfo__.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/__pkginfo__.py b/__pkginfo__.py index 72659671..a48a2bdb 100644 --- a/__pkginfo__.py +++ b/__pkginfo__.py @@ -37,8 +37,8 @@ entry_points={ 'pydisassemble=uncompyle6.bin.pydisassemble:main', ]} ftp_url = None -install_requires = ['spark-parser >= 1.4.0', - 'xdis >= 3.2.0'] +install_requires = ['spark-parser >= 1.4.0, < 1.5.0', + 'xdis >= 3.2.0, < 3.3.0'] license = 'MIT' mailing_list = 'python-debugger@googlegroups.com' modname = 'uncompyle6' From 25dd67a1354244879bb22032983703ec8d317ade Mon Sep 17 00:00:00 2001 From: rocky Date: Thu, 27 Oct 2016 13:52:07 -0400 Subject: [PATCH 10/23] Clean and fix Python 3 annotate arg return --- .../{04_def_attr.py => 04_def_annotate.py} | 0 uncompyle6/parsers/parse31.py | 9 +- uncompyle6/semantics/pysource.py | 359 +++++++++--------- 3 files changed, 188 insertions(+), 180 deletions(-) rename test/simple_source/bug31/{04_def_attr.py => 04_def_annotate.py} (100%) diff --git a/test/simple_source/bug31/04_def_attr.py b/test/simple_source/bug31/04_def_annotate.py similarity index 100% rename from test/simple_source/bug31/04_def_attr.py rename to test/simple_source/bug31/04_def_annotate.py diff --git a/uncompyle6/parsers/parse31.py b/uncompyle6/parsers/parse31.py index 0356a224..08295cf7 100644 --- a/uncompyle6/parsers/parse31.py +++ b/uncompyle6/parsers/parse31.py @@ -33,8 +33,8 @@ class Python31Parser(Python32Parser): load ::= LOAD_FAST load ::= LOAD_NAME - stmt ::= funcdeftest - funcdeftest ::= mkfunctest designator + stmt ::= funcdef_annotate + funcdef_annotate ::= mkfunc_annotate designator """ def add_custom_rules(self, tokens, customize): @@ -46,8 +46,9 @@ class Python31Parser(Python32Parser): # Check that there are 2 annotated params? # rule = ('mkfunc2 ::= %s%sEXTENDED_ARG %s' % # ('pos_arg ' * (args_pos), 'kwargs ' * (annotate_args-1), opname)) - rule = ('mkfunctest ::= %s%sLOAD_CONST EXTENDED_ARG %s' % - (('pos_arg ' * (args_pos)), 'kwargs ', opname)) + rule = ('mkfunc_annotate ::= %s%sLOAD_CONST EXTENDED_ARG %s' % + (('pos_arg ' * (args_pos)), + ('kwargs ' * (annotate_args-1)), opname)) self.add_unique_rule(rule, opname, token.attr, customize) class Python31ParserSingle(Python31Parser, PythonParserSingle): diff --git a/uncompyle6/semantics/pysource.py b/uncompyle6/semantics/pysource.py index 9b120cc8..ac6f36b5 100644 --- a/uncompyle6/semantics/pysource.py +++ b/uncompyle6/semantics/pysource.py @@ -611,206 +611,213 @@ class SourceWalker(GenericASTTraversal, object): 'comp_for': ( ' for %c in %c%c', 2, 0, 3 ), }) + if version >= 3.0: + if 3.1 == version: + ########################## + # Python 3.1 + ########################## + TABLE_DIRECT.update({ + 'funcdef_annotate': ( '\n\n%|def %c%c\n', -1, 0), + }) - if 3.1 == version: - ########################## - # Python 3.1 - ########################## - TABLE_DIRECT.update({ - 'funcdeftest': ( '\n\n%|def %c%c\n', -1, 0), - }) + def make_function31(node, isLambda, nested=1, + codeNode=None, annotate=None): + """Dump function defintion, doc string, and function + body. This code is specialzed for Python 3.1""" - def make_function31(node, isLambda, nested=1, - codeNode=None, annotate=None): - """Dump function defintion, doc string, and function - body. This code is specialzed for Python 3.1""" - - def build_param(ast, name, default): - """build parameters: - - handle defaults - - handle format tuple parameters - """ - if default: - value = self.traverse(default, indent='') - maybe_show_ast_param_default(self.showast, name, value) - result = '%s=%s' % (name, value) - if result[-2:] == '= ': # default was 'LOAD_CONST None' - result += 'None' - return result - else: - return name - - # MAKE_FUNCTION_... or MAKE_CLOSURE_... - assert node[-1].type.startswith('MAKE_') - - args_node = node[-1] - if isinstance(args_node.attr, tuple): - defparams = node[1:args_node.attr[0]+1] - pos_args, kw_args, annotate_args = args_node.attr - else: - defparams = node[:args_node.attr] - kw_args = 0 - pass - - lambda_index = -2 - - if lambda_index and isLambda and iscode(node[lambda_index].attr): - assert node[lambda_index].type == 'LOAD_LAMBDA' - code = node[lambda_index].attr - else: - code = codeNode.attr - - assert iscode(code) - code = Code(code, self.scanner, self.currentclass) - - # add defaults values to parameter names - argc = code.co_argcount - paramnames = list(code.co_varnames[:argc]) - - try: - ast = self.build_ast(code._tokens, - code._customize, - isLambda = isLambda, - noneInNames = ('None' in code.co_names)) - except ParserError as p: - self.write(str(p)) - self.ERROR = p - return - - kw_pairs = args_node.attr[1] if self.version >= 3.0 else 0 - indent = self.indent - - params = [build_param(ast, name, default) for - name, default in zip_longest(paramnames, defparams, fillvalue=None)] - params.reverse() # back to correct order - - if 4 & code.co_flags: # flag 2 -> variable number of args - if self.version > 3.0: - params.append('*%s' % code.co_varnames[argc + kw_pairs]) - else: - params.append('*%s' % code.co_varnames[argc]) - argc += 1 - - # dump parameter list (with default values) - if isLambda: - self.write("lambda ", ", ".join(params)) - else: - self.write("(", ", ".join(params)) - # self.println(indent, '#flags:\t', int(code.co_flags)) - if kw_args > 0: - if not (4 & code.co_flags): - if argc > 0: - self.write(", *, ") + def build_param(ast, name, default): + """build parameters: + - handle defaults + - handle format tuple parameters + """ + if default: + value = self.traverse(default, indent='') + maybe_show_ast_param_default(self.showast, name, value) + result = '%s=%s' % (name, value) + if result[-2:] == '= ': # default was 'LOAD_CONST None' + result += 'None' + return result else: - self.write("*, ") - pass - else: - self.write(", ") + return name - kwargs = node[0] - last = len(kwargs)-1 - i = 0 - for n in node[0]: - if n == 'kwarg': - self.write('%s=' % n[0].pattr) - self.preorder(n[1]) - if i < last: - self.write(', ') + # MAKE_FUNCTION_... or MAKE_CLOSURE_... + assert node[-1].type.startswith('MAKE_') + + args_node = node[-1] + if isinstance(args_node.attr, tuple): + # positional args are before kwargs + defparams = node[:args_node.attr[0]] + pos_args, kw_args, annotate_args = args_node.attr + else: + defparams = node[:args_node.attr] + kw_args = 0 + pass + + lambda_index = -2 + + if lambda_index and isLambda and iscode(node[lambda_index].attr): + assert node[lambda_index].type == 'LOAD_LAMBDA' + code = node[lambda_index].attr + else: + code = codeNode.attr + + assert iscode(code) + code = Code(code, self.scanner, self.currentclass) + + # add defaults values to parameter names + argc = code.co_argcount + paramnames = list(code.co_varnames[:argc]) + + try: + ast = self.build_ast(code._tokens, + code._customize, + isLambda = isLambda, + noneInNames = ('None' in code.co_names)) + except ParserError as p: + self.write(str(p)) + self.ERROR = p + return + + kw_pairs = args_node.attr[1] + indent = self.indent + + if isLambda: + self.write("lambda ") + else: + self.write("(") + + if 4 & code.co_flags: # flag 2 -> variable number of args + self.write('*%s' % code.co_varnames[argc + kw_pairs]) + argc += 1 + + i = len(paramnames) - len(defparams) + self.write(",".join(paramnames[:i])) + suffix = ', ' if i > 0 else '' + for n in node: + if n == 'pos_arg': + self.write(suffix) + self.write(paramnames[i] + '=') i += 1 + self.preorder(n) + suffix = ', ' + + # self.println(indent, '#flags:\t', int(code.co_flags)) + if kw_args > 0: + if not (4 & code.co_flags): + if argc > 0: + self.write(", *, ") + else: + self.write("*, ") + pass + else: + self.write(", ") + + kwargs = node[0] + last = len(kwargs)-1 + i = 0 + for n in node[0]: + if n == 'kwarg': + self.write('%s=' % n[0].pattr) + self.preorder(n[1]) + if i < last: + self.write(', ') + i += 1 + pass pass pass - pass - if 8 & code.co_flags: # flag 3 -> keyword args - if argc > 0: - self.write(', ') - self.write('**%s' % code.co_varnames[argc + kw_pairs]) + if 8 & code.co_flags: # flag 3 -> keyword args + if argc > 0: + self.write(', ') + self.write('**%s' % code.co_varnames[argc + kw_pairs]) - if isLambda: - self.write(": ") - else: - self.write(')') - if annotate: - self.write(' -> %s' % annotate) - self.println(":") + if isLambda: + self.write(": ") + else: + self.write(')') + if annotate: + self.write(' -> %s' % annotate) + self.println(":") - if (len(code.co_consts) > 0 and - code.co_consts[0] is not None and not isLambda): # ugly - # docstring exists, dump it - self.print_docstring(indent, code.co_consts[0]) + if (len(code.co_consts) > 0 and + code.co_consts[0] is not None and not isLambda): # ugly + # docstring exists, dump it + self.print_docstring(indent, code.co_consts[0]) - code._tokens = None # save memory - assert ast == 'stmts' + code._tokens = None # save memory + assert ast == 'stmts' - all_globals = find_all_globals(ast, set()) - for g in ((all_globals & self.mod_globs) | find_globals(ast, set())): - self.println(self.indent, 'global ', g) - self.mod_globs -= all_globals - has_none = 'None' in code.co_names - rn = has_none and not find_none(ast) - self.gen_source(ast, code.co_name, code._customize, isLambda=isLambda, - returnNone=rn) - code._tokens = code._customize = None # save memory + all_globals = find_all_globals(ast, set()) + for g in ((all_globals & self.mod_globs) | find_globals(ast, set())): + self.println(self.indent, 'global ', g) + self.mod_globs -= all_globals + has_none = 'None' in code.co_names + rn = has_none and not find_none(ast) + self.gen_source(ast, code.co_name, code._customize, isLambda=isLambda, + returnNone=rn) + code._tokens = code._customize = None # save memory - self.make_function31 = make_function31 + self.make_function31 = make_function31 - def n_mkfunctest(node): + def n_mkfunc_annotate(node): - self.indentMore() - code = node[-3] - annotate = None - if node[-4][0][0] == 'LOAD_CONST': - annotate = node[-4][0][0].attr - self.make_function31(node, isLambda=False, - codeNode=code, annotate=annotate) + self.indentMore() + code = node[-3] + annotate = None + if node[-4] == 'kwargs' and node[-4][0][0] == 'LOAD_CONST': + # FIXME: Annotate args are currently stored as kwargs + # we should rename that grammar nonterminal + annotate = node[-4][0][0].attr + self.make_function31(node, isLambda=False, + codeNode=code, annotate=annotate) - if len(self.param_stack) > 1: - self.write('\n\n') - else: - self.write('\n\n\n') - self.indentLess() - self.prune() # stop recursing - self.n_mkfunctest = n_mkfunctest + if len(self.param_stack) > 1: + self.write('\n\n') + else: + self.write('\n\n\n') + self.indentLess() + self.prune() # stop recursing + self.n_mkfunc_annotate = n_mkfunc_annotate - elif 3.2 <= version <= 3.3: - ########################## - # Python 3.2 and 3.3 - ########################## - TABLE_DIRECT.update({ - 'store_locals': ( '%|# inspect.currentframe().f_locals = __locals__\n', ), - }) - elif version >= 3.4: - ######################## - # Python 3.4+ Additions - ####################### - TABLE_DIRECT.update({ - 'LOAD_CLASSDEREF': ( '%{pattr}', ), - }) - if version >= 3.6: + elif 3.2 <= version <= 3.3: + ########################## + # Python 3.2 and 3.3 + ########################## + TABLE_DIRECT.update({ + 'store_locals': ( '%|# inspect.currentframe().f_locals = __locals__\n', ), + }) + elif version >= 3.4: ######################## - # Python 3.6+ Additions + # Python 3.4+ Additions ####################### TABLE_DIRECT.update({ - 'fstring_expr': ( "{%c%{conversion}}", 0), - 'fstring_single': ( "f'{%c%{conversion}}'", 0), - 'fstring_multi': ( "f'%c'", 0), - }) + 'LOAD_CLASSDEREF': ( '%{pattr}', ), + }) + if version >= 3.6: + ######################## + # Python 3.6+ Additions + ####################### + TABLE_DIRECT.update({ + 'fstring_expr': ( "{%c%{conversion}}", 0), + 'fstring_single': ( "f'{%c%{conversion}}'", 0), + 'fstring_multi': ( "f'%c'", 0), + }) - FSTRING_CONVERSION_MAP = {1: '!s', 2: '!r', 3: '!a'} - def f_conversion(node): - node.conversion = FSTRING_CONVERSION_MAP.get(node.data[1].attr, '') + FSTRING_CONVERSION_MAP = {1: '!s', 2: '!r', 3: '!a'} + def f_conversion(node): + node.conversion = FSTRING_CONVERSION_MAP.get(node.data[1].attr, '') - def n_fstring_expr(node): - f_conversion(node) - self.default(node) - self.n_fstring_expr = n_fstring_expr + def n_fstring_expr(node): + f_conversion(node) + self.default(node) + self.n_fstring_expr = n_fstring_expr - def n_fstring_single(node): - f_conversion(node) - self.default(node) + def n_fstring_single(node): + f_conversion(node) + self.default(node) - self.n_fstring_single = n_fstring_single + self.n_fstring_single = n_fstring_single return @@ -2346,7 +2353,7 @@ class SourceWalker(GenericASTTraversal, object): indent = self.indent # build parameters - if not 3.0 <= self.version <= 3.2: + if self.version != 3.2: params = [build_param(ast, name, default) for name, default in zip_longest(paramnames, defparams, fillvalue=None)] params.reverse() # back to correct order From 0e7da031b25faea60f800615e359e7c2f1263096 Mon Sep 17 00:00:00 2001 From: rocky Date: Fri, 28 Oct 2016 07:07:18 -0400 Subject: [PATCH 11/23] Split out 3.1-3.3 parsers from parser3.py This is anticipation of extending annotation to Python 3.2+ --- uncompyle6/parser.py | 14 ++++++++------ uncompyle6/parsers/parse3.py | 33 ++------------------------------- uncompyle6/parsers/parse31.py | 2 +- uncompyle6/parsers/parse32.py | 21 +++++++++++++++++++++ uncompyle6/parsers/parse33.py | 20 ++++++++++++++++++++ uncompyle6/parsers/parse35.py | 2 +- 6 files changed, 53 insertions(+), 39 deletions(-) create mode 100644 uncompyle6/parsers/parse32.py create mode 100644 uncompyle6/parsers/parse33.py diff --git a/uncompyle6/parser.py b/uncompyle6/parser.py index 5e6a0329..af91c0f7 100644 --- a/uncompyle6/parser.py +++ b/uncompyle6/parser.py @@ -622,21 +622,23 @@ def get_python_parser( else: import uncompyle6.parsers.parse3 as parse3 if version == 3.1: + import uncompyle6.parsers.parse31 as parse31 if compile_mode == 'exec': - import uncompyle6.parsers.parse31 as parse31 p = parse31.Python31Parser(debug_parser) else: - p = parse3.Python31ParserSingle(debug_parser) + p = parse31.Python31ParserSingle(debug_parser) elif version == 3.2: + import uncompyle6.parsers.parse32 as parse32 if compile_mode == 'exec': - p = parse3.Python32Parser(debug_parser) + p = parse32.Python32Parser(debug_parser) else: - p = parse3.Python32ParserSingle(debug_parser) + p = parse32.Python32ParserSingle(debug_parser) elif version == 3.3: + import uncompyle6.parsers.parse33 as parse33 if compile_mode == 'exec': - p = parse3.Python33Parser(debug_parser) + p = parse33.Python33Parser(debug_parser) else: - p = parse3.Python33ParserSingle(debug_parser) + p = parse33.Python33ParserSingle(debug_parser) elif version == 3.4: import uncompyle6.parsers.parse34 as parse34 if compile_mode == 'exec': diff --git a/uncompyle6/parsers/parse3.py b/uncompyle6/parsers/parse3.py index da218ca4..12e4beb7 100644 --- a/uncompyle6/parsers/parse3.py +++ b/uncompyle6/parsers/parse3.py @@ -670,40 +670,9 @@ class Python3Parser(PythonParser): self.add_unique_rule(rule, opname, token.attr, customize) return - -class Python33Parser(Python3Parser): - def p_33(self, args): - """ - # Store locals is only in Python 3.0 to 3.3 - stmt ::= store_locals - store_locals ::= LOAD_FAST STORE_LOCALS - - # Python 3.3 adds yield from. - expr ::= yield_from - yield_from ::= expr expr YIELD_FROM - """ - -class Python32Parser(Python3Parser): - def p_32on(self, args): - """ - # In Python 3.2+, DUP_TOPX is DUP_TOP_TWO - binary_subscr2 ::= expr expr DUP_TOP_TWO BINARY_SUBSCR - stmt ::= store_locals - store_locals ::= LOAD_FAST STORE_LOCALS - """ - pass - class Python3ParserSingle(Python3Parser, PythonParserSingle): pass - -class Python32ParserSingle(Python32Parser, PythonParserSingle): - pass - - -class Python33ParserSingle(Python33Parser, PythonParserSingle): - pass - def info(args): # Check grammar p = Python3Parser() @@ -713,8 +682,10 @@ def info(args): from uncompyle6.parser.parse35 import Python35Parser p = Python35Parser() elif arg == '3.3': + from uncompyle6.parser.parse33 import Python33Parser p = Python33Parser() elif arg == '3.2': + from uncompyle6.parser.parse32 import Python32Parser p = Python32Parser() p.checkGrammar() if len(sys.argv) > 1 and sys.argv[1] == 'dump': diff --git a/uncompyle6/parsers/parse31.py b/uncompyle6/parsers/parse31.py index 08295cf7..f902004c 100644 --- a/uncompyle6/parsers/parse31.py +++ b/uncompyle6/parsers/parse31.py @@ -5,7 +5,7 @@ spark grammar differences over Python 3.2 for Python 3.1. from __future__ import print_function from uncompyle6.parser import PythonParserSingle -from uncompyle6.parsers.parse3 import Python32Parser +from uncompyle6.parsers.parse32 import Python32Parser class Python31Parser(Python32Parser): diff --git a/uncompyle6/parsers/parse32.py b/uncompyle6/parsers/parse32.py new file mode 100644 index 00000000..29eb5026 --- /dev/null +++ b/uncompyle6/parsers/parse32.py @@ -0,0 +1,21 @@ +# Copyright (c) 2016 Rocky Bernstein +""" +spark grammar differences over Python 3 for Python 3.2. +""" +from __future__ import print_function + +from uncompyle6.parser import PythonParserSingle +from uncompyle6.parsers.parse3 import Python3Parser + +class Python32Parser(Python3Parser): + def p_32on(self, args): + """ + # In Python 3.2+, DUP_TOPX is DUP_TOP_TWO + binary_subscr2 ::= expr expr DUP_TOP_TWO BINARY_SUBSCR + stmt ::= store_locals + store_locals ::= LOAD_FAST STORE_LOCALS + """ + pass + +class Python32ParserSingle(Python32Parser, PythonParserSingle): + pass diff --git a/uncompyle6/parsers/parse33.py b/uncompyle6/parsers/parse33.py new file mode 100644 index 00000000..38037613 --- /dev/null +++ b/uncompyle6/parsers/parse33.py @@ -0,0 +1,20 @@ +# Copyright (c) 2016 Rocky Bernstein +""" +spark grammar differences over Python 3.2 for Python 3.3. +""" +from __future__ import print_function + +from uncompyle6.parser import PythonParserSingle +from uncompyle6.parsers.parse32 import Python32Parser + +class Python33Parser(Python32Parser): + + def p_33(self, args): + """ + # Python 3.3 adds yield from. + expr ::= yield_from + yield_from ::= expr expr YIELD_FROM + """ + +class Python33ParserSingle(Python33Parser, PythonParserSingle): + pass diff --git a/uncompyle6/parsers/parse35.py b/uncompyle6/parsers/parse35.py index 39895eac..de61de25 100644 --- a/uncompyle6/parsers/parse35.py +++ b/uncompyle6/parsers/parse35.py @@ -6,7 +6,7 @@ from __future__ import print_function from uncompyle6.parser import PythonParserSingle from spark_parser import DEFAULT_DEBUG as PARSER_DEFAULT_DEBUG -from uncompyle6.parsers.parse3 import Python32Parser +from uncompyle6.parsers.parse32 import Python32Parser class Python35Parser(Python32Parser): From 9849f06ff67a5f6376032e2a4cf014c0833f4057 Mon Sep 17 00:00:00 2001 From: rocky Date: Fri, 28 Oct 2016 08:21:23 -0400 Subject: [PATCH 12/23] Expand annotate handling to 3.3 (and possibly 3.2) - DRY Python 3.1-3.3 grammar a little --- pytest/test_grammar.py | 8 +++-- uncompyle6/parsers/parse31.py | 6 +--- uncompyle6/parsers/parse32.py | 21 +++++++++++++ uncompyle6/semantics/pysource.py | 51 +++++++++++++++++++------------- 4 files changed, 59 insertions(+), 27 deletions(-) diff --git a/pytest/test_grammar.py b/pytest/test_grammar.py index c091bd6c..547b7f1e 100644 --- a/pytest/test_grammar.py +++ b/pytest/test_grammar.py @@ -1,4 +1,4 @@ -import pytest, re +import re from uncompyle6 import PYTHON_VERSION, PYTHON3, IS_PYPY # , PYTHON_VERSION from uncompyle6.parser import get_python_parser from uncompyle6.scanner import get_scanner @@ -16,7 +16,8 @@ def test_grammar(): p = get_python_parser(PYTHON_VERSION, is_pypy=IS_PYPY) lhs, rhs, tokens, right_recursive = p.checkSets() expect_lhs = set(['expr1024', 'pos_arg']) - unused_rhs = set(['build_list', 'call_function', 'mkfunc', 'mklambda', + unused_rhs = set(['build_list', 'call_function', 'mkfunc', + 'mklambda', 'unpack', 'unpack_list']) expect_right_recursive = [['designList', ('designator', 'DUP_TOP', 'designList')]] if PYTHON3: @@ -24,6 +25,9 @@ def test_grammar(): unused_rhs = unused_rhs.union(set(""" except_pop_except genexpr classdefdeco2 listcomp """.split())) + if 3.1 <= PYTHON_VERSION <= 3.3: + unused_rhs.add("mkfunc_annotate") + pass else: expect_lhs.add('kwarg') assert expect_lhs == set(lhs) diff --git a/uncompyle6/parsers/parse31.py b/uncompyle6/parsers/parse31.py index f902004c..6c910571 100644 --- a/uncompyle6/parsers/parse31.py +++ b/uncompyle6/parsers/parse31.py @@ -32,9 +32,6 @@ class Python31Parser(Python32Parser): store ::= STORE_NAME load ::= LOAD_FAST load ::= LOAD_NAME - - stmt ::= funcdef_annotate - funcdef_annotate ::= mkfunc_annotate designator """ def add_custom_rules(self, tokens, customize): @@ -48,8 +45,7 @@ class Python31Parser(Python32Parser): # ('pos_arg ' * (args_pos), 'kwargs ' * (annotate_args-1), opname)) rule = ('mkfunc_annotate ::= %s%sLOAD_CONST EXTENDED_ARG %s' % (('pos_arg ' * (args_pos)), - ('kwargs ' * (annotate_args-1)), opname)) + ('annotate_args ' * (annotate_args-1)), opname)) self.add_unique_rule(rule, opname, token.attr, customize) - class Python31ParserSingle(Python31Parser, PythonParserSingle): pass diff --git a/uncompyle6/parsers/parse32.py b/uncompyle6/parsers/parse32.py index 29eb5026..5a94dad9 100644 --- a/uncompyle6/parsers/parse32.py +++ b/uncompyle6/parsers/parse32.py @@ -14,8 +14,29 @@ class Python32Parser(Python3Parser): binary_subscr2 ::= expr expr DUP_TOP_TWO BINARY_SUBSCR stmt ::= store_locals store_locals ::= LOAD_FAST STORE_LOCALS + + stmt ::= funcdef_annotate + funcdef_annotate ::= mkfunc_annotate designator + + annotate_args ::= annotate_args annotate_arg + annotate_args ::= annotate_arg + annotate_arg ::= LOAD_CONST expr """ pass + def add_custom_rules(self, tokens, customize): + super(Python32Parser, self).add_custom_rules(tokens, customize) + for i, token in enumerate(tokens): + opname = token.type + if opname.startswith('MAKE_FUNCTION_A'): + args_pos, args_kw, annotate_args = token.attr + # Check that there are 2 annotated params? + rule = ('mkfunc_annotate ::= %s%sLOAD_CONST LOAD_CONST EXTENDED_ARG %s' % + (('pos_arg ' * (args_pos)), + ('annotate_args ' * (annotate_args-1)), opname)) + print(rule) + self.add_unique_rule(rule, opname, token.attr, customize) + + class Python32ParserSingle(Python32Parser, PythonParserSingle): pass diff --git a/uncompyle6/semantics/pysource.py b/uncompyle6/semantics/pysource.py index ac6f36b5..0079edec 100644 --- a/uncompyle6/semantics/pysource.py +++ b/uncompyle6/semantics/pysource.py @@ -612,16 +612,17 @@ class SourceWalker(GenericASTTraversal, object): }) if version >= 3.0: - if 3.1 == version: + if 3.1 <= version <= 3.3: ########################## - # Python 3.1 + # Python 3.1 - 3.3 ########################## TABLE_DIRECT.update({ 'funcdef_annotate': ( '\n\n%|def %c%c\n', -1, 0), + 'store_locals': ( '%|# inspect.currentframe().f_locals = __locals__\n', ), }) - def make_function31(node, isLambda, nested=1, - codeNode=None, annotate=None): + def make_function3(node, isLambda, nested=1, + codeNode=None, annotate=None): """Dump function defintion, doc string, and function body. This code is specialzed for Python 3.1""" @@ -653,7 +654,12 @@ class SourceWalker(GenericASTTraversal, object): kw_args = 0 pass - lambda_index = -2 + if 3.0 <= self.version <= 3.2: + lambda_index = -2 + elif 3.03 <= self.version: + lambda_index = -3 + else: + lambda_index = None if lambda_index and isLambda and iscode(node[lambda_index].attr): assert node[lambda_index].type == 'LOAD_LAMBDA' @@ -757,19 +763,31 @@ class SourceWalker(GenericASTTraversal, object): returnNone=rn) code._tokens = code._customize = None # save memory - self.make_function31 = make_function31 + self.make_function3 = make_function3 def n_mkfunc_annotate(node): + if self.version >= 3.3 or node[-2] == 'kwargs': + # LOAD_CONST code object .. + # LOAD_CONST 'x0' if >= 3.3 + # EXTENDED_ARG + # MAKE_FUNCTION .. + code = node[-4] + elif node[-3] == 'expr': + code = node[-3][0] + else: + # LOAD_CONST code object .. + # MAKE_FUNCTION .. + code = node[-3] + self.indentMore() - code = node[-3] annotate = None - if node[-4] == 'kwargs' and node[-4][0][0] == 'LOAD_CONST': - # FIXME: Annotate args are currently stored as kwargs - # we should rename that grammar nonterminal - annotate = node[-4][0][0].attr - self.make_function31(node, isLambda=False, - codeNode=code, annotate=annotate) + annotate_args = node[-4] if self.version == 3.1 else node[-5]; + + if annotate_args == 'annotate_args' and annotate_args[0][0] == 'LOAD_CONST': + annotate = annotate_args[0][0].attr + self.make_function3(node, isLambda=False, + codeNode=code, annotate=annotate) if len(self.param_stack) > 1: self.write('\n\n') @@ -780,13 +798,6 @@ class SourceWalker(GenericASTTraversal, object): self.n_mkfunc_annotate = n_mkfunc_annotate - elif 3.2 <= version <= 3.3: - ########################## - # Python 3.2 and 3.3 - ########################## - TABLE_DIRECT.update({ - 'store_locals': ( '%|# inspect.currentframe().f_locals = __locals__\n', ), - }) elif version >= 3.4: ######################## # Python 3.4+ Additions From 2e355b62459d03d145adf6ce08e269258f9c4cab Mon Sep 17 00:00:00 2001 From: rocky Date: Fri, 28 Oct 2016 11:33:54 -0400 Subject: [PATCH 13/23] Expand annotate return to Python 3.4 --- pytest/test_grammar.py | 3 ++- uncompyle6/parsers/parse3.py | 12 ++++++++++++ uncompyle6/parsers/parse32.py | 8 -------- uncompyle6/parsers/parse33.py | 4 ++-- uncompyle6/parsers/parse34.py | 12 +++--------- uncompyle6/semantics/pysource.py | 6 +++--- 6 files changed, 22 insertions(+), 23 deletions(-) diff --git a/pytest/test_grammar.py b/pytest/test_grammar.py index 547b7f1e..604298dc 100644 --- a/pytest/test_grammar.py +++ b/pytest/test_grammar.py @@ -25,7 +25,7 @@ def test_grammar(): unused_rhs = unused_rhs.union(set(""" except_pop_except genexpr classdefdeco2 listcomp """.split())) - if 3.1 <= PYTHON_VERSION <= 3.3: + if 3.1 <= PYTHON_VERSION <= 3.4: unused_rhs.add("mkfunc_annotate") pass else: @@ -47,5 +47,6 @@ def test_grammar(): check_tokens(tokens, opcode_set) elif PYTHON_VERSION == 3.4: ignore_set.add('LOAD_CLASSNAME') + ignore_set.add('STORE_LOCALS') opcode_set = set(s.opc.opname).union(ignore_set) check_tokens(tokens, opcode_set) diff --git a/uncompyle6/parsers/parse3.py b/uncompyle6/parsers/parse3.py index 12e4beb7..8eb0e04d 100644 --- a/uncompyle6/parsers/parse3.py +++ b/uncompyle6/parsers/parse3.py @@ -246,6 +246,18 @@ class Python3Parser(PythonParser): c_stmts_opt34 ::= JUMP_BACK JUMP_ABSOLUTE c_stmts_opt """ + + def p_def_annotations3(self, args): + """ + # Annotated functions + stmt ::= funcdef_annotate + funcdef_annotate ::= mkfunc_annotate designator + + annotate_args ::= annotate_args annotate_arg + annotate_args ::= annotate_arg + annotate_arg ::= LOAD_CONST expr + """ + def p_come_from3(self, args): """ opt_come_from_except ::= COME_FROM_EXCEPT diff --git a/uncompyle6/parsers/parse32.py b/uncompyle6/parsers/parse32.py index 5a94dad9..b6d5a3c6 100644 --- a/uncompyle6/parsers/parse32.py +++ b/uncompyle6/parsers/parse32.py @@ -14,13 +14,6 @@ class Python32Parser(Python3Parser): binary_subscr2 ::= expr expr DUP_TOP_TWO BINARY_SUBSCR stmt ::= store_locals store_locals ::= LOAD_FAST STORE_LOCALS - - stmt ::= funcdef_annotate - funcdef_annotate ::= mkfunc_annotate designator - - annotate_args ::= annotate_args annotate_arg - annotate_args ::= annotate_arg - annotate_arg ::= LOAD_CONST expr """ pass @@ -34,7 +27,6 @@ class Python32Parser(Python3Parser): rule = ('mkfunc_annotate ::= %s%sLOAD_CONST LOAD_CONST EXTENDED_ARG %s' % (('pos_arg ' * (args_pos)), ('annotate_args ' * (annotate_args-1)), opname)) - print(rule) self.add_unique_rule(rule, opname, token.attr, customize) diff --git a/uncompyle6/parsers/parse33.py b/uncompyle6/parsers/parse33.py index 38037613..dd43c12e 100644 --- a/uncompyle6/parsers/parse33.py +++ b/uncompyle6/parsers/parse33.py @@ -9,9 +9,9 @@ from uncompyle6.parsers.parse32 import Python32Parser class Python33Parser(Python32Parser): - def p_33(self, args): + def p_33on(self, args): """ - # Python 3.3 adds yield from. + # Python 3.3+ adds yield from. expr ::= yield_from yield_from ::= expr expr YIELD_FROM """ diff --git a/uncompyle6/parsers/parse34.py b/uncompyle6/parsers/parse34.py index cf094af6..bce24488 100644 --- a/uncompyle6/parsers/parse34.py +++ b/uncompyle6/parsers/parse34.py @@ -1,13 +1,13 @@ # Copyright (c) 2016 Rocky Bernstein """ -spark grammar differences over Python 3.2 for Python 3.4 +spark grammar differences over Python 3.3 for Python 3.4 """ from uncompyle6.parser import PythonParserSingle from spark_parser import DEFAULT_DEBUG as PARSER_DEFAULT_DEBUG -from uncompyle6.parsers.parse3 import Python3Parser +from uncompyle6.parsers.parse33 import Python33Parser -class Python34Parser(Python3Parser): +class Python34Parser(Python33Parser): def __init__(self, debug_parser=PARSER_DEFAULT_DEBUG): super(Python34Parser, self).__init__(debug_parser) @@ -28,12 +28,6 @@ class Python34Parser(Python3Parser): iflaststmt ::= testexpr c_stmts_opt34 c_stmts_opt34 ::= JUMP_BACK JUMP_ABSOLUTE c_stmts_opt - # Python 3.3 added "yield from." Do it the same way as in - # 3.3 - - expr ::= yield_from - yield_from ::= expr expr YIELD_FROM - # Is this 3.4 only? yield_from ::= expr GET_ITER LOAD_CONST YIELD_FROM diff --git a/uncompyle6/semantics/pysource.py b/uncompyle6/semantics/pysource.py index 0079edec..71d57f29 100644 --- a/uncompyle6/semantics/pysource.py +++ b/uncompyle6/semantics/pysource.py @@ -612,9 +612,9 @@ class SourceWalker(GenericASTTraversal, object): }) if version >= 3.0: - if 3.1 <= version <= 3.3: + if 3.1 <= version <= 3.4: ########################## - # Python 3.1 - 3.3 + # Python 3.1 - 3.4 ########################## TABLE_DIRECT.update({ 'funcdef_annotate': ( '\n\n%|def %c%c\n', -1, 0), @@ -798,7 +798,7 @@ class SourceWalker(GenericASTTraversal, object): self.n_mkfunc_annotate = n_mkfunc_annotate - elif version >= 3.4: + if version >= 3.4: ######################## # Python 3.4+ Additions ####################### From ccdd37611cf209389a43213a4ab43e1eb6e4815f Mon Sep 17 00:00:00 2001 From: rocky Date: Fri, 28 Oct 2016 19:52:30 -0400 Subject: [PATCH 14/23] More complete annotate handling Still have a bit of work to do though. --- pytest/test_grammar.py | 4 +- uncompyle6/parsers/parse3.py | 5 +- uncompyle6/parsers/parse31.py | 2 +- uncompyle6/parsers/parse32.py | 2 +- uncompyle6/parsers/parse35.py | 6 +- uncompyle6/semantics/pysource.py | 346 ++++++++++++++++--------------- 6 files changed, 191 insertions(+), 174 deletions(-) diff --git a/pytest/test_grammar.py b/pytest/test_grammar.py index 604298dc..3cf98358 100644 --- a/pytest/test_grammar.py +++ b/pytest/test_grammar.py @@ -22,10 +22,12 @@ def test_grammar(): expect_right_recursive = [['designList', ('designator', 'DUP_TOP', 'designList')]] if PYTHON3: expect_lhs.add('load_genexpr') + unused_rhs = unused_rhs.union(set(""" except_pop_except genexpr classdefdeco2 listcomp """.split())) - if 3.1 <= PYTHON_VERSION <= 3.4: + if 3.0 <= PYTHON_VERSION: + expect_lhs.add("annotate_arg") unused_rhs.add("mkfunc_annotate") pass else: diff --git a/uncompyle6/parsers/parse3.py b/uncompyle6/parsers/parse3.py index 8eb0e04d..233053c0 100644 --- a/uncompyle6/parsers/parse3.py +++ b/uncompyle6/parsers/parse3.py @@ -253,9 +253,8 @@ class Python3Parser(PythonParser): stmt ::= funcdef_annotate funcdef_annotate ::= mkfunc_annotate designator - annotate_args ::= annotate_args annotate_arg - annotate_args ::= annotate_arg - annotate_arg ::= LOAD_CONST expr + annotate_arg ::= LOAD_CONST + annotate_arg ::= LOAD_NAME """ def p_come_from3(self, args): diff --git a/uncompyle6/parsers/parse31.py b/uncompyle6/parsers/parse31.py index 6c910571..7bad1b30 100644 --- a/uncompyle6/parsers/parse31.py +++ b/uncompyle6/parsers/parse31.py @@ -45,7 +45,7 @@ class Python31Parser(Python32Parser): # ('pos_arg ' * (args_pos), 'kwargs ' * (annotate_args-1), opname)) rule = ('mkfunc_annotate ::= %s%sLOAD_CONST EXTENDED_ARG %s' % (('pos_arg ' * (args_pos)), - ('annotate_args ' * (annotate_args-1)), opname)) + ('annotate_arg ' * (annotate_args)), opname)) self.add_unique_rule(rule, opname, token.attr, customize) class Python31ParserSingle(Python31Parser, PythonParserSingle): pass diff --git a/uncompyle6/parsers/parse32.py b/uncompyle6/parsers/parse32.py index b6d5a3c6..9de0d3cb 100644 --- a/uncompyle6/parsers/parse32.py +++ b/uncompyle6/parsers/parse32.py @@ -26,7 +26,7 @@ class Python32Parser(Python3Parser): # Check that there are 2 annotated params? rule = ('mkfunc_annotate ::= %s%sLOAD_CONST LOAD_CONST EXTENDED_ARG %s' % (('pos_arg ' * (args_pos)), - ('annotate_args ' * (annotate_args-1)), opname)) + ('annotate_arg ' * (annotate_args)), opname)) self.add_unique_rule(rule, opname, token.attr, customize) diff --git a/uncompyle6/parsers/parse35.py b/uncompyle6/parsers/parse35.py index de61de25..996eec72 100644 --- a/uncompyle6/parsers/parse35.py +++ b/uncompyle6/parsers/parse35.py @@ -1,14 +1,14 @@ # Copyright (c) 2016 Rocky Bernstein """ -spark grammar differences over Python 3.2 for Python 3.5. +spark grammar differences over Python 3.4 for Python 3.5. """ from __future__ import print_function from uncompyle6.parser import PythonParserSingle from spark_parser import DEFAULT_DEBUG as PARSER_DEFAULT_DEBUG -from uncompyle6.parsers.parse32 import Python32Parser +from uncompyle6.parsers.parse34 import Python34Parser -class Python35Parser(Python32Parser): +class Python35Parser(Python34Parser): def __init__(self, debug_parser=PARSER_DEFAULT_DEBUG): super(Python35Parser, self).__init__(debug_parser) diff --git a/uncompyle6/semantics/pysource.py b/uncompyle6/semantics/pysource.py index 71d57f29..7819675e 100644 --- a/uncompyle6/semantics/pysource.py +++ b/uncompyle6/semantics/pysource.py @@ -611,191 +611,204 @@ class SourceWalker(GenericASTTraversal, object): 'comp_for': ( ' for %c in %c%c', 2, 0, 3 ), }) - if version >= 3.0: - if 3.1 <= version <= 3.4: - ########################## - # Python 3.1 - 3.4 - ########################## - TABLE_DIRECT.update({ - 'funcdef_annotate': ( '\n\n%|def %c%c\n', -1, 0), - 'store_locals': ( '%|# inspect.currentframe().f_locals = __locals__\n', ), - }) + if version >= 3.0: + TABLE_DIRECT.update({ + 'funcdef_annotate': ( '\n\n%|def %c%c\n', -1, 0), + 'store_locals': ( '%|# inspect.currentframe().f_locals = __locals__\n', ), + }) - def make_function3(node, isLambda, nested=1, - codeNode=None, annotate=None): - """Dump function defintion, doc string, and function - body. This code is specialzed for Python 3.1""" + def make_function3(node, isLambda, nested=1, + codeNode=None, annotate=None): + """Dump function defintion, doc string, and function + body. This code is specialzed for Python 3""" - def build_param(ast, name, default): - """build parameters: - - handle defaults - - handle format tuple parameters - """ - if default: - value = self.traverse(default, indent='') - maybe_show_ast_param_default(self.showast, name, value) - result = '%s=%s' % (name, value) - if result[-2:] == '= ': # default was 'LOAD_CONST None' - result += 'None' - return result + def build_param(ast, name, default): + """build parameters: + - handle defaults + - handle format tuple parameters + """ + if default: + value = self.traverse(default, indent='') + maybe_show_ast_param_default(self.showast, name, value) + result = '%s=%s' % (name, value) + if result[-2:] == '= ': # default was 'LOAD_CONST None' + result += 'None' + return result + else: + return name + + # MAKE_FUNCTION_... or MAKE_CLOSURE_... + assert node[-1].type.startswith('MAKE_') + + args_node = node[-1] + if isinstance(args_node.attr, tuple): + # positional args are before kwargs + defparams = node[:args_node.attr[0]] + pos_args, kw_args, annotate_args = args_node.attr + else: + defparams = node[:args_node.attr] + kw_args = 0 + pass + + if 3.0 <= self.version <= 3.2: + lambda_index = -2 + elif 3.03 <= self.version: + lambda_index = -3 + else: + lambda_index = None + + if lambda_index and isLambda and iscode(node[lambda_index].attr): + assert node[lambda_index].type == 'LOAD_LAMBDA' + code = node[lambda_index].attr + else: + code = codeNode.attr + + assert iscode(code) + code = Code(code, self.scanner, self.currentclass) + + # add defaults values to parameter names + argc = code.co_argcount + paramnames = list(code.co_varnames[:argc]) + + try: + ast = self.build_ast(code._tokens, + code._customize, + isLambda = isLambda, + noneInNames = ('None' in code.co_names)) + except ParserError as p: + self.write(str(p)) + self.ERROR = p + return + + kw_pairs = args_node.attr[1] + indent = self.indent + + if isLambda: + self.write("lambda ") + else: + self.write("(") + + last_line = self.f.getvalue().split("\n")[-1] + l = len(last_line) + indent = ' ' * l + line_number = self.line_number + + + if 4 & code.co_flags: # flag 2 -> variable number of args + self.write('*%s' % code.co_varnames[argc + kw_pairs]) + argc += 1 + + i = len(paramnames) - len(defparams) + self.write(",".join(paramnames[:i])) + suffix = ', ' if i > 0 else '' + for n in node: + if n == 'pos_arg': + self.write(suffix) + self.write(paramnames[i] + '=') + i += 1 + self.preorder(n) + if (line_number != self.line_number): + suffix = ",\n" + indent + line_number = self.line_number else: - return name - - # MAKE_FUNCTION_... or MAKE_CLOSURE_... - assert node[-1].type.startswith('MAKE_') - - args_node = node[-1] - if isinstance(args_node.attr, tuple): - # positional args are before kwargs - defparams = node[:args_node.attr[0]] - pos_args, kw_args, annotate_args = args_node.attr - else: - defparams = node[:args_node.attr] - kw_args = 0 - pass - - if 3.0 <= self.version <= 3.2: - lambda_index = -2 - elif 3.03 <= self.version: - lambda_index = -3 - else: - lambda_index = None - - if lambda_index and isLambda and iscode(node[lambda_index].attr): - assert node[lambda_index].type == 'LOAD_LAMBDA' - code = node[lambda_index].attr - else: - code = codeNode.attr - - assert iscode(code) - code = Code(code, self.scanner, self.currentclass) - - # add defaults values to parameter names - argc = code.co_argcount - paramnames = list(code.co_varnames[:argc]) - - try: - ast = self.build_ast(code._tokens, - code._customize, - isLambda = isLambda, - noneInNames = ('None' in code.co_names)) - except ParserError as p: - self.write(str(p)) - self.ERROR = p - return - - kw_pairs = args_node.attr[1] - indent = self.indent - - if isLambda: - self.write("lambda ") - else: - self.write("(") - - if 4 & code.co_flags: # flag 2 -> variable number of args - self.write('*%s' % code.co_varnames[argc + kw_pairs]) - argc += 1 - - i = len(paramnames) - len(defparams) - self.write(",".join(paramnames[:i])) - suffix = ', ' if i > 0 else '' - for n in node: - if n == 'pos_arg': - self.write(suffix) - self.write(paramnames[i] + '=') - i += 1 - self.preorder(n) suffix = ', ' - # self.println(indent, '#flags:\t', int(code.co_flags)) - if kw_args > 0: - if not (4 & code.co_flags): - if argc > 0: - self.write(", *, ") - else: - self.write("*, ") - pass + # self.println(indent, '#flags:\t', int(code.co_flags)) + if kw_args > 0: + if not (4 & code.co_flags): + if argc > 0: + self.write(", *, ") else: - self.write(", ") + self.write("*, ") + pass + else: + self.write(", ") - kwargs = node[0] - last = len(kwargs)-1 - i = 0 - for n in node[0]: - if n == 'kwarg': - self.write('%s=' % n[0].pattr) - self.preorder(n[1]) - if i < last: - self.write(', ') - i += 1 - pass + kwargs = node[0] + last = len(kwargs)-1 + i = 0 + for n in node[0]: + if n == 'kwarg': + self.write('%s=' % n[0].pattr) + self.preorder(n[1]) + if i < last: + self.write(', ') + i += 1 pass pass + pass - if 8 & code.co_flags: # flag 3 -> keyword args - if argc > 0: - self.write(', ') - self.write('**%s' % code.co_varnames[argc + kw_pairs]) + if 8 & code.co_flags: # flag 3 -> keyword args + if argc > 0: + self.write(', ') + self.write('**%s' % code.co_varnames[argc + kw_pairs]) - if isLambda: - self.write(": ") - else: - self.write(')') - if annotate: - self.write(' -> %s' % annotate) - self.println(":") + if isLambda: + self.write(": ") + else: + self.write(')') + if annotate: + self.write(' -> "%s"' % annotate) + self.println(":") - if (len(code.co_consts) > 0 and - code.co_consts[0] is not None and not isLambda): # ugly - # docstring exists, dump it - self.print_docstring(indent, code.co_consts[0]) + if (len(code.co_consts) > 0 and + code.co_consts[0] is not None and not isLambda): # ugly + # docstring exists, dump it + self.print_docstring(indent, code.co_consts[0]) - code._tokens = None # save memory - assert ast == 'stmts' + code._tokens = None # save memory + assert ast == 'stmts' - all_globals = find_all_globals(ast, set()) - for g in ((all_globals & self.mod_globs) | find_globals(ast, set())): - self.println(self.indent, 'global ', g) - self.mod_globs -= all_globals - has_none = 'None' in code.co_names - rn = has_none and not find_none(ast) - self.gen_source(ast, code.co_name, code._customize, isLambda=isLambda, - returnNone=rn) - code._tokens = code._customize = None # save memory + all_globals = find_all_globals(ast, set()) + for g in ((all_globals & self.mod_globs) | find_globals(ast, set())): + self.println(self.indent, 'global ', g) + self.mod_globs -= all_globals + has_none = 'None' in code.co_names + rn = has_none and not find_none(ast) + self.gen_source(ast, code.co_name, code._customize, isLambda=isLambda, + returnNone=rn) + code._tokens = code._customize = None # save memory - self.make_function3 = make_function3 + self.make_function3 = make_function3 - def n_mkfunc_annotate(node): + def n_mkfunc_annotate(node): - if self.version >= 3.3 or node[-2] == 'kwargs': - # LOAD_CONST code object .. - # LOAD_CONST 'x0' if >= 3.3 - # EXTENDED_ARG - # MAKE_FUNCTION .. - code = node[-4] - elif node[-3] == 'expr': - code = node[-3][0] - else: - # LOAD_CONST code object .. - # MAKE_FUNCTION .. - code = node[-3] + if self.version >= 3.3 or node[-2] == 'kwargs': + # LOAD_CONST code object .. + # LOAD_CONST 'x0' if >= 3.3 + # EXTENDED_ARG + # MAKE_FUNCTION .. + code = node[-4] + elif node[-3] == 'expr': + code = node[-3][0] + else: + # LOAD_CONST code object .. + # MAKE_FUNCTION .. + code = node[-3] - self.indentMore() - annotate = None - annotate_args = node[-4] if self.version == 3.1 else node[-5]; + self.indentMore() + annotate_return = None + annotate_last = -4 if self.version == 3.1 else -5 + annotate_arg = node[annotate_last] - if annotate_args == 'annotate_args' and annotate_args[0][0] == 'LOAD_CONST': - annotate = annotate_args[0][0].attr - self.make_function3(node, isLambda=False, - codeNode=code, annotate=annotate) + if (annotate_arg == 'annotate_arg' + and annotate_arg[0] == 'LOAD_CONST' + and isinstance(annotate_arg[0].attr, tuple)): + annotate_tup = annotate_arg[0].attr + if annotate_tup[-1] == 'return': + annotate_return = node[annotate_last-1][0].attr + pass + # FIXME: handle and pass full annotate args + self.make_function3(node, isLambda=False, + codeNode=code, annotate=annotate_return) - if len(self.param_stack) > 1: - self.write('\n\n') - else: - self.write('\n\n\n') - self.indentLess() - self.prune() # stop recursing - self.n_mkfunc_annotate = n_mkfunc_annotate + if len(self.param_stack) > 1: + self.write('\n\n') + else: + self.write('\n\n\n') + self.indentLess() + self.prune() # stop recursing + self.n_mkfunc_annotate = n_mkfunc_annotate if version >= 3.4: @@ -2285,6 +2298,9 @@ class SourceWalker(GenericASTTraversal, object): def make_function(self, node, isLambda, nested=1, codeNode=None): """Dump function defintion, doc string, and function body.""" + # FIXME: call make_function3 if we are self.version >= 3.0 + # and then simplify the below. + def build_param(ast, name, default): """build parameters: - handle defaults From 2328ca7a5546e3718e8832362054befb7217b11f Mon Sep 17 00:00:00 2001 From: rocky Date: Sat, 29 Oct 2016 06:45:41 -0400 Subject: [PATCH 15/23] Break out make_function() into its own file. It is already too complex and will get worse in Python 3.6. Note: make_function in fragments.py is still inside and probably needs fixup. --- uncompyle6/semantics/fragments.py | 7 +- uncompyle6/semantics/make_function.py | 520 ++++++++++++++++++++++++++ uncompyle6/semantics/parser_error.py | 11 + uncompyle6/semantics/pysource.py | 396 +------------------- 4 files changed, 548 insertions(+), 386 deletions(-) create mode 100644 uncompyle6/semantics/make_function.py create mode 100644 uncompyle6/semantics/parser_error.py diff --git a/uncompyle6/semantics/fragments.py b/uncompyle6/semantics/fragments.py index a1324f73..9a4c0e0a 100644 --- a/uncompyle6/semantics/fragments.py +++ b/uncompyle6/semantics/fragments.py @@ -67,7 +67,9 @@ from uncompyle6.show import ( ) from uncompyle6.semantics.pysource import AST, INDENT_PER_LEVEL, NONE, PRECEDENCE, \ - ParserError, TABLE_DIRECT, escape, find_all_globals, find_globals, find_none, minint, MAP + ParserError, TABLE_DIRECT, escape, find_globals, minint, MAP + +from uncompyle6.semantics.make_function import find_all_globals, find_none if PYTHON3: from itertools import zip_longest @@ -77,8 +79,7 @@ else: from StringIO import StringIO -from spark_parser import GenericASTTraversalPruningException, \ - DEFAULT_DEBUG as PARSER_DEFAULT_DEBUG +from spark_parser import DEFAULT_DEBUG as PARSER_DEFAULT_DEBUG from collections import namedtuple NodeInfo = namedtuple("NodeInfo", "node start finish") diff --git a/uncompyle6/semantics/make_function.py b/uncompyle6/semantics/make_function.py new file mode 100644 index 00000000..dcbd87c2 --- /dev/null +++ b/uncompyle6/semantics/make_function.py @@ -0,0 +1,520 @@ +# Copyright (c) 2015, 2016 by Rocky Bernstein +# Copyright (c) 2000-2002 by hartmut Goebel +""" +All the crazy things we have to do to handle Python functions +""" +from xdis.code import iscode +from uncompyle6.scanner import Code +from uncompyle6.parsers.astnode import AST +from uncompyle6 import PYTHON3 +from uncompyle6.semantics.parser_error import ParserError + +if PYTHON3: + from itertools import zip_longest +else: + from itertools import izip_longest as zip_longest + +from uncompyle6.show import maybe_show_ast_param_default + +def find_all_globals(node, globs): + """Find globals in this statement.""" + for n in node: + if isinstance(n, AST): + globs = find_all_globals(n, globs) + elif n.type in ('STORE_GLOBAL', 'DELETE_GLOBAL', 'LOAD_GLOBAL'): + globs.add(n.pattr) + return globs + +def find_globals(node, globs): + """Find globals in this statement.""" + for n in node: + if isinstance(n, AST): + globs = find_globals(n, globs) + elif n.type in ('STORE_GLOBAL', 'DELETE_GLOBAL'): + globs.add(n.pattr) + return globs + +def find_none(node): + for n in node: + if isinstance(n, AST): + if not n in ('return_stmt', 'return_if_stmt'): + if find_none(n): + return True + elif n.type == 'LOAD_CONST' and n.pattr is None: + return True + return False + +# FIXME: DRY the below code... + +def make_function3_annotate(self, node, isLambda, nested=1, + codeNode=None, annotate=None): + """ + Dump function defintion, doc string, and function + body. This code is specialized for Python 3""" + + def build_param(ast, name, default): + """build parameters: + - handle defaults + - handle format tuple parameters + """ + if default: + value = self.traverse(default, indent='') + maybe_show_ast_param_default(self.showast, name, value) + result = '%s=%s' % (name, value) + if result[-2:] == '= ': # default was 'LOAD_CONST None' + result += 'None' + return result + else: + return name + + # MAKE_FUNCTION_... or MAKE_CLOSURE_... + assert node[-1].type.startswith('MAKE_') + + args_node = node[-1] + if isinstance(args_node.attr, tuple): + # positional args are before kwargs + defparams = node[:args_node.attr[0]] + pos_args, kw_args, annotate_args = args_node.attr + else: + defparams = node[:args_node.attr] + kw_args = 0 + pass + + if 3.0 <= self.version <= 3.2: + lambda_index = -2 + elif 3.03 <= self.version: + lambda_index = -3 + else: + lambda_index = None + + if lambda_index and isLambda and iscode(node[lambda_index].attr): + assert node[lambda_index].type == 'LOAD_LAMBDA' + code = node[lambda_index].attr + else: + code = codeNode.attr + + assert iscode(code) + code = Code(code, self.scanner, self.currentclass) + + # add defaults values to parameter names + argc = code.co_argcount + paramnames = list(code.co_varnames[:argc]) + + try: + ast = self.build_ast(code._tokens, + code._customize, + isLambda = isLambda, + noneInNames = ('None' in code.co_names)) + except ParserError as p: + self.write(str(p)) + self.ERROR = p + return + + kw_pairs = args_node.attr[1] + indent = self.indent + + if isLambda: + self.write("lambda ") + else: + self.write("(") + + last_line = self.f.getvalue().split("\n")[-1] + l = len(last_line) + indent = ' ' * l + line_number = self.line_number + + + if 4 & code.co_flags: # flag 2 -> variable number of args + self.write('*%s' % code.co_varnames[argc + kw_pairs]) + argc += 1 + + i = len(paramnames) - len(defparams) + self.write(", ".join(paramnames[:i])) + suffix = ', ' if i > 0 else '' + for n in node: + if n == 'pos_arg': + self.write(suffix) + self.write(paramnames[i] + '=') + i += 1 + self.preorder(n) + if (line_number != self.line_number): + suffix = ",\n" + indent + line_number = self.line_number + else: + suffix = ', ' + + # self.println(indent, '#flags:\t', int(code.co_flags)) + if kw_args > 0: + if not (4 & code.co_flags): + if argc > 0: + self.write(", *, ") + else: + self.write("*, ") + pass + else: + self.write(", ") + + kwargs = node[0] + last = len(kwargs)-1 + i = 0 + for n in node[0]: + if n == 'kwarg': + self.write('%s=' % n[0].pattr) + self.preorder(n[1]) + if i < last: + self.write(', ') + i += 1 + pass + pass + pass + + if 8 & code.co_flags: # flag 3 -> keyword args + if argc > 0: + self.write(', ') + self.write('**%s' % code.co_varnames[argc + kw_pairs]) + + if isLambda: + self.write(": ") + else: + self.write(')') + if annotate: + self.write(' -> "%s"' % annotate) + self.println(":") + + if (len(code.co_consts) > 0 and + code.co_consts[0] is not None and not isLambda): # ugly + # docstring exists, dump it + self.print_docstring(indent, code.co_consts[0]) + + code._tokens = None # save memory + assert ast == 'stmts' + + all_globals = find_all_globals(ast, set()) + for g in ((all_globals & self.mod_globs) | find_globals(ast, set())): + self.println(self.indent, 'global ', g) + self.mod_globs -= all_globals + has_none = 'None' in code.co_names + rn = has_none and not find_none(ast) + self.gen_source(ast, code.co_name, code._customize, isLambda=isLambda, + returnNone=rn) + code._tokens = code._customize = None # save memory + +def make_function2(self, node, isLambda, nested=1, codeNode=None): + """ + Dump function defintion, doc string, and function body. + This code is specialied for Python 2. + """ + + # FIXME: call make_function3 if we are self.version >= 3.0 + # and then simplify the below. + + def build_param(ast, name, default): + """build parameters: + - handle defaults + - handle format tuple parameters + """ + # if formal parameter is a tuple, the paramater name + # starts with a dot (eg. '.1', '.2') + if name.startswith('.'): + # replace the name with the tuple-string + name = self.get_tuple_parameter(ast, name) + pass + + if default: + value = self.traverse(default, indent='') + maybe_show_ast_param_default(self.showast, name, value) + result = '%s=%s' % (name, value) + if result[-2:] == '= ': # default was 'LOAD_CONST None' + result += 'None' + return result + else: + return name + + # MAKE_FUNCTION_... or MAKE_CLOSURE_... + assert node[-1].type.startswith('MAKE_') + + args_node = node[-1] + if isinstance(args_node.attr, tuple): + # positional args are after kwargs + defparams = node[1:args_node.attr[0]+1] + pos_args, kw_args, annotate_args = args_node.attr + else: + defparams = node[:args_node.attr] + kw_args = 0 + pass + + lambda_index = None + + if lambda_index and isLambda and iscode(node[lambda_index].attr): + assert node[lambda_index].type == 'LOAD_LAMBDA' + code = node[lambda_index].attr + else: + code = codeNode.attr + + assert iscode(code) + code = Code(code, self.scanner, self.currentclass) + + # add defaults values to parameter names + argc = code.co_argcount + paramnames = list(code.co_varnames[:argc]) + + # defaults are for last n parameters, thus reverse + paramnames.reverse(); defparams.reverse() + + try: + ast = self.build_ast(code._tokens, + code._customize, + isLambda = isLambda, + noneInNames = ('None' in code.co_names)) + except ParserError as p: + self.write(str(p)) + self.ERROR = p + return + + kw_pairs = args_node.attr[1] if self.version >= 3.0 else 0 + indent = self.indent + + # build parameters + params = [build_param(ast, name, default) for + name, default in zip_longest(paramnames, defparams, fillvalue=None)] + params.reverse() # back to correct order + + if 4 & code.co_flags: # flag 2 -> variable number of args + if self.version > 3.0: + params.append('*%s' % code.co_varnames[argc + kw_pairs]) + else: + params.append('*%s' % code.co_varnames[argc]) + argc += 1 + + # dump parameter list (with default values) + if isLambda: + self.write("lambda ", ", ".join(params)) + else: + self.write("(", ", ".join(params)) + # self.println(indent, '#flags:\t', int(code.co_flags)) + + if kw_args > 0: + if not (4 & code.co_flags): + if argc > 0: + self.write(", *, ") + else: + self.write("*, ") + pass + else: + self.write(", ") + + for n in node: + if n == 'pos_arg': + continue + elif self.version >= 3.4 and n.type != 'kwargs': + continue + else: + self.preorder(n) + break + pass + + if 8 & code.co_flags: # flag 3 -> keyword args + if argc > 0: + self.write(', ') + self.write('**%s' % code.co_varnames[argc + kw_pairs]) + + if isLambda: + self.write(": ") + else: + self.println("):") + + if len(code.co_consts) > 0 and code.co_consts[0] is not None and not isLambda: # ugly + # docstring exists, dump it + self.print_docstring(indent, code.co_consts[0]) + + code._tokens = None # save memory + assert ast == 'stmts' + + all_globals = find_all_globals(ast, set()) + for g in ((all_globals & self.mod_globs) | find_globals(ast, set())): + self.println(self.indent, 'global ', g) + self.mod_globs -= all_globals + has_none = 'None' in code.co_names + rn = has_none and not find_none(ast) + self.gen_source(ast, code.co_name, code._customize, isLambda=isLambda, + returnNone=rn) + code._tokens = None; code._customize = None # save memory + + +def make_function3(self, node, isLambda, nested=1, codeNode=None): + """Dump function definition, doc string, and function body.""" + + # FIXME: call make_function3 if we are self.version >= 3.0 + # and then simplify the below. + + def build_param(ast, name, default): + """build parameters: + - handle defaults + - handle format tuple parameters + """ + if default: + value = self.traverse(default, indent='') + maybe_show_ast_param_default(self.showast, name, value) + result = '%s=%s' % (name, value) + if result[-2:] == '= ': # default was 'LOAD_CONST None' + result += 'None' + return result + else: + return name + + # MAKE_FUNCTION_... or MAKE_CLOSURE_... + assert node[-1].type.startswith('MAKE_') + + args_node = node[-1] + if isinstance(args_node.attr, tuple): + if self.version <= 3.3: + # positional args are after kwargs + defparams = node[1:args_node.attr[0]+1] + else: + # positional args are before kwargs + defparams = node[:args_node.attr[0]] + pos_args, kw_args, annotate_args = args_node.attr + else: + defparams = node[:args_node.attr] + kw_args = 0 + pass + + if 3.0 <= self.version <= 3.2: + lambda_index = -2 + elif 3.03 <= self.version: + lambda_index = -3 + else: + lambda_index = None + + if lambda_index and isLambda and iscode(node[lambda_index].attr): + assert node[lambda_index].type == 'LOAD_LAMBDA' + code = node[lambda_index].attr + else: + code = codeNode.attr + + assert iscode(code) + code = Code(code, self.scanner, self.currentclass) + + # add defaults values to parameter names + argc = code.co_argcount + paramnames = list(code.co_varnames[:argc]) + + # defaults are for last n parameters, thus reverse + if not 3.0 <= self.version <= 3.2: + paramnames.reverse(); defparams.reverse() + + try: + ast = self.build_ast(code._tokens, + code._customize, + isLambda = isLambda, + noneInNames = ('None' in code.co_names)) + except ParserError as p: + self.write(str(p)) + self.ERROR = p + return + + kw_pairs = args_node.attr[1] if self.version >= 3.0 else 0 + indent = self.indent + + # build parameters + if self.version != 3.2: + params = [build_param(ast, name, default) for + name, default in zip_longest(paramnames, defparams, fillvalue=None)] + params.reverse() # back to correct order + + if 4 & code.co_flags: # flag 2 -> variable number of args + if self.version > 3.0: + params.append('*%s' % code.co_varnames[argc + kw_pairs]) + else: + params.append('*%s' % code.co_varnames[argc]) + argc += 1 + + # dump parameter list (with default values) + if isLambda: + self.write("lambda ", ", ".join(params)) + else: + self.write("(", ", ".join(params)) + # self.println(indent, '#flags:\t', int(code.co_flags)) + + else: + if isLambda: + self.write("lambda ") + else: + self.write("(") + + if 4 & code.co_flags: # flag 2 -> variable number of args + self.write('*%s' % code.co_varnames[argc + kw_pairs]) + argc += 1 + + i = len(paramnames) - len(defparams) + self.write(", ".join(paramnames[:i])) + suffix = ', ' if i > 0 else '' + for n in node: + if n == 'pos_arg': + self.write(suffix) + self.write(paramnames[i] + '=') + i += 1 + self.preorder(n) + suffix = ', ' + + if kw_args > 0: + if not (4 & code.co_flags): + if argc > 0: + self.write(", *, ") + else: + self.write("*, ") + pass + else: + self.write(", ") + + if not 3.0 <= self.version <= 3.2: + for n in node: + if n == 'pos_arg': + continue + elif self.version >= 3.4 and n.type != 'kwargs': + continue + else: + self.preorder(n) + break + else: + kwargs = node[0] + last = len(kwargs)-1 + i = 0 + for n in node[0]: + if n == 'kwarg': + self.write('%s=' % n[0].pattr) + self.preorder(n[1]) + if i < last: + self.write(', ') + i += 1 + pass + pass + pass + pass + + if 8 & code.co_flags: # flag 3 -> keyword args + if argc > 0: + self.write(', ') + self.write('**%s' % code.co_varnames[argc + kw_pairs]) + + if isLambda: + self.write(": ") + else: + self.println("):") + + if len(code.co_consts) > 0 and code.co_consts[0] is not None and not isLambda: # ugly + # docstring exists, dump it + self.print_docstring(indent, code.co_consts[0]) + + code._tokens = None # save memory + assert ast == 'stmts' + + all_globals = find_all_globals(ast, set()) + for g in ((all_globals & self.mod_globs) | find_globals(ast, set())): + self.println(self.indent, 'global ', g) + self.mod_globs -= all_globals + has_none = 'None' in code.co_names + rn = has_none and not find_none(ast) + self.gen_source(ast, code.co_name, code._customize, isLambda=isLambda, + returnNone=rn) + code._tokens = None; code._customize = None # save memory diff --git a/uncompyle6/semantics/parser_error.py b/uncompyle6/semantics/parser_error.py new file mode 100644 index 00000000..a0a24ecd --- /dev/null +++ b/uncompyle6/semantics/parser_error.py @@ -0,0 +1,11 @@ +import uncompyle6.parser as python_parser +class ParserError(python_parser.ParserError): + def __init__(self, error, tokens): + self.error = error # previous exception + self.tokens = tokens + + def __str__(self): + lines = ['--- This code section failed: ---'] + lines.extend([str(i) for i in self.tokens]) + lines.extend( ['', str(self.error)] ) + return '\n'.join(lines) diff --git a/uncompyle6/semantics/pysource.py b/uncompyle6/semantics/pysource.py index 7819675e..3ef77342 100644 --- a/uncompyle6/semantics/pysource.py +++ b/uncompyle6/semantics/pysource.py @@ -79,10 +79,13 @@ from spark_parser import GenericASTTraversal, DEFAULT_DEBUG as PARSER_DEFAULT_DE from uncompyle6.scanner import Code, get_scanner from uncompyle6.scanners.tok import Token, NoneToken import uncompyle6.parser as python_parser +from uncompyle6.semantics.make_function import ( + make_function2, make_function3, make_function3_annotate, find_globals) +from uncompyle6.semantics.parser_error import ParserError + from uncompyle6.show import ( maybe_show_asm, maybe_show_ast, - maybe_show_ast_param_default, ) if PYTHON3: @@ -430,45 +433,6 @@ def is_docstring(node): except: return False -class ParserError(python_parser.ParserError): - def __init__(self, error, tokens): - self.error = error # previous exception - self.tokens = tokens - - def __str__(self): - lines = ['--- This code section failed: ---'] - lines.extend([str(i) for i in self.tokens]) - lines.extend( ['', str(self.error)] ) - return '\n'.join(lines) - -def find_globals(node, globs): - """Find globals in this statement.""" - for n in node: - if isinstance(n, AST): - globs = find_globals(n, globs) - elif n.type in ('STORE_GLOBAL', 'DELETE_GLOBAL'): - globs.add(n.pattr) - return globs - -def find_all_globals(node, globs): - """Find globals in this statement.""" - for n in node: - if isinstance(n, AST): - globs = find_all_globals(n, globs) - elif n.type in ('STORE_GLOBAL', 'DELETE_GLOBAL', 'LOAD_GLOBAL'): - globs.add(n.pattr) - return globs - -def find_none(node): - for n in node: - if isinstance(n, AST): - if not n in ('return_stmt', 'return_if_stmt'): - if find_none(n): - return True - elif n.type == 'LOAD_CONST' and n.pattr is None: - return True - return False - class SourceWalkerError(Exception): def __init__(self, errmsg): self.errmsg = errmsg @@ -617,160 +581,6 @@ class SourceWalker(GenericASTTraversal, object): 'store_locals': ( '%|# inspect.currentframe().f_locals = __locals__\n', ), }) - def make_function3(node, isLambda, nested=1, - codeNode=None, annotate=None): - """Dump function defintion, doc string, and function - body. This code is specialzed for Python 3""" - - def build_param(ast, name, default): - """build parameters: - - handle defaults - - handle format tuple parameters - """ - if default: - value = self.traverse(default, indent='') - maybe_show_ast_param_default(self.showast, name, value) - result = '%s=%s' % (name, value) - if result[-2:] == '= ': # default was 'LOAD_CONST None' - result += 'None' - return result - else: - return name - - # MAKE_FUNCTION_... or MAKE_CLOSURE_... - assert node[-1].type.startswith('MAKE_') - - args_node = node[-1] - if isinstance(args_node.attr, tuple): - # positional args are before kwargs - defparams = node[:args_node.attr[0]] - pos_args, kw_args, annotate_args = args_node.attr - else: - defparams = node[:args_node.attr] - kw_args = 0 - pass - - if 3.0 <= self.version <= 3.2: - lambda_index = -2 - elif 3.03 <= self.version: - lambda_index = -3 - else: - lambda_index = None - - if lambda_index and isLambda and iscode(node[lambda_index].attr): - assert node[lambda_index].type == 'LOAD_LAMBDA' - code = node[lambda_index].attr - else: - code = codeNode.attr - - assert iscode(code) - code = Code(code, self.scanner, self.currentclass) - - # add defaults values to parameter names - argc = code.co_argcount - paramnames = list(code.co_varnames[:argc]) - - try: - ast = self.build_ast(code._tokens, - code._customize, - isLambda = isLambda, - noneInNames = ('None' in code.co_names)) - except ParserError as p: - self.write(str(p)) - self.ERROR = p - return - - kw_pairs = args_node.attr[1] - indent = self.indent - - if isLambda: - self.write("lambda ") - else: - self.write("(") - - last_line = self.f.getvalue().split("\n")[-1] - l = len(last_line) - indent = ' ' * l - line_number = self.line_number - - - if 4 & code.co_flags: # flag 2 -> variable number of args - self.write('*%s' % code.co_varnames[argc + kw_pairs]) - argc += 1 - - i = len(paramnames) - len(defparams) - self.write(",".join(paramnames[:i])) - suffix = ', ' if i > 0 else '' - for n in node: - if n == 'pos_arg': - self.write(suffix) - self.write(paramnames[i] + '=') - i += 1 - self.preorder(n) - if (line_number != self.line_number): - suffix = ",\n" + indent - line_number = self.line_number - else: - suffix = ', ' - - # self.println(indent, '#flags:\t', int(code.co_flags)) - if kw_args > 0: - if not (4 & code.co_flags): - if argc > 0: - self.write(", *, ") - else: - self.write("*, ") - pass - else: - self.write(", ") - - kwargs = node[0] - last = len(kwargs)-1 - i = 0 - for n in node[0]: - if n == 'kwarg': - self.write('%s=' % n[0].pattr) - self.preorder(n[1]) - if i < last: - self.write(', ') - i += 1 - pass - pass - pass - - if 8 & code.co_flags: # flag 3 -> keyword args - if argc > 0: - self.write(', ') - self.write('**%s' % code.co_varnames[argc + kw_pairs]) - - if isLambda: - self.write(": ") - else: - self.write(')') - if annotate: - self.write(' -> "%s"' % annotate) - self.println(":") - - if (len(code.co_consts) > 0 and - code.co_consts[0] is not None and not isLambda): # ugly - # docstring exists, dump it - self.print_docstring(indent, code.co_consts[0]) - - code._tokens = None # save memory - assert ast == 'stmts' - - all_globals = find_all_globals(ast, set()) - for g in ((all_globals & self.mod_globs) | find_globals(ast, set())): - self.println(self.indent, 'global ', g) - self.mod_globs -= all_globals - has_none = 'None' in code.co_names - rn = has_none and not find_none(ast) - self.gen_source(ast, code.co_name, code._customize, isLambda=isLambda, - returnNone=rn) - code._tokens = code._customize = None # save memory - - self.make_function3 = make_function3 - def n_mkfunc_annotate(node): if self.version >= 3.3 or node[-2] == 'kwargs': @@ -799,8 +609,8 @@ class SourceWalker(GenericASTTraversal, object): annotate_return = node[annotate_last-1][0].attr pass # FIXME: handle and pass full annotate args - self.make_function3(node, isLambda=False, - codeNode=code, annotate=annotate_return) + make_function3_annotate(self, node, isLambda=False, + codeNode=code, annotate=annotate_return) if len(self.param_stack) > 1: self.write('\n\n') @@ -1349,6 +1159,13 @@ class SourceWalker(GenericASTTraversal, object): self.indentLess() self.prune() # stop recursing + def make_function(self, node, isLambda, nested=1, + codeNode=None, annotate=None): + if self.version >= 3.0: + make_function3(self, node, isLambda, nested, codeNode) + else: + make_function2(self, node, isLambda, nested, codeNode) + def n_mklambda(self, node): self.make_function(node, isLambda=True, codeNode=node[-2]) self.prune() # stop recursing @@ -2295,193 +2112,6 @@ class SourceWalker(GenericASTTraversal, object): # return self.traverse(node[1]) raise Exception("Can't find tuple parameter " + name) - def make_function(self, node, isLambda, nested=1, codeNode=None): - """Dump function defintion, doc string, and function body.""" - - # FIXME: call make_function3 if we are self.version >= 3.0 - # and then simplify the below. - - def build_param(ast, name, default): - """build parameters: - - handle defaults - - handle format tuple parameters - """ - if self.version < 3.0: - # if formal parameter is a tuple, the paramater name - # starts with a dot (eg. '.1', '.2') - if name.startswith('.'): - # replace the name with the tuple-string - name = self.get_tuple_parameter(ast, name) - pass - pass - - if default: - value = self.traverse(default, indent='') - maybe_show_ast_param_default(self.showast, name, value) - result = '%s=%s' % (name, value) - if result[-2:] == '= ': # default was 'LOAD_CONST None' - result += 'None' - return result - else: - return name - - # MAKE_FUNCTION_... or MAKE_CLOSURE_... - assert node[-1].type.startswith('MAKE_') - - args_node = node[-1] - if isinstance(args_node.attr, tuple): - if self.version <= 3.3: - # positional args are after kwargs - defparams = node[1:args_node.attr[0]+1] - else: - # positional args are before kwargs - defparams = node[:args_node.attr[0]] - pos_args, kw_args, annotate_args = args_node.attr - else: - defparams = node[:args_node.attr] - kw_args = 0 - pass - - if 3.0 <= self.version <= 3.2: - lambda_index = -2 - elif 3.03 <= self.version: - lambda_index = -3 - else: - lambda_index = None - - if lambda_index and isLambda and iscode(node[lambda_index].attr): - assert node[lambda_index].type == 'LOAD_LAMBDA' - code = node[lambda_index].attr - else: - code = codeNode.attr - - assert iscode(code) - code = Code(code, self.scanner, self.currentclass) - - # add defaults values to parameter names - argc = code.co_argcount - paramnames = list(code.co_varnames[:argc]) - - # defaults are for last n parameters, thus reverse - if not 3.0 <= self.version <= 3.2: - paramnames.reverse(); defparams.reverse() - - try: - ast = self.build_ast(code._tokens, - code._customize, - isLambda = isLambda, - noneInNames = ('None' in code.co_names)) - except ParserError as p: - self.write(str(p)) - self.ERROR = p - return - - kw_pairs = args_node.attr[1] if self.version >= 3.0 else 0 - indent = self.indent - - # build parameters - if self.version != 3.2: - params = [build_param(ast, name, default) for - name, default in zip_longest(paramnames, defparams, fillvalue=None)] - params.reverse() # back to correct order - - if 4 & code.co_flags: # flag 2 -> variable number of args - if self.version > 3.0: - params.append('*%s' % code.co_varnames[argc + kw_pairs]) - else: - params.append('*%s' % code.co_varnames[argc]) - argc += 1 - - # dump parameter list (with default values) - if isLambda: - self.write("lambda ", ", ".join(params)) - else: - self.write("(", ", ".join(params)) - # self.println(indent, '#flags:\t', int(code.co_flags)) - - else: - if isLambda: - self.write("lambda ") - else: - self.write("(") - - if 4 & code.co_flags: # flag 2 -> variable number of args - self.write('*%s' % code.co_varnames[argc + kw_pairs]) - argc += 1 - - i = len(paramnames) - len(defparams) - self.write(",".join(paramnames[:i])) - suffix = ', ' if i > 0 else '' - for n in node: - if n == 'pos_arg': - self.write(suffix) - self.write(paramnames[i] + '=') - i += 1 - self.preorder(n) - suffix = ', ' - - if kw_args > 0: - if not (4 & code.co_flags): - if argc > 0: - self.write(", *, ") - else: - self.write("*, ") - pass - else: - self.write(", ") - - if not 3.0 <= self.version <= 3.2: - for n in node: - if n == 'pos_arg': - continue - elif self.version >= 3.4 and n.type != 'kwargs': - continue - else: - self.preorder(n) - break - else: - kwargs = node[0] - last = len(kwargs)-1 - i = 0 - for n in node[0]: - if n == 'kwarg': - self.write('%s=' % n[0].pattr) - self.preorder(n[1]) - if i < last: - self.write(', ') - i += 1 - pass - pass - pass - pass - - if 8 & code.co_flags: # flag 3 -> keyword args - if argc > 0: - self.write(', ') - self.write('**%s' % code.co_varnames[argc + kw_pairs]) - - if isLambda: - self.write(": ") - else: - self.println("):") - - if len(code.co_consts)>0 and code.co_consts[0] is not None and not isLambda: # ugly - # docstring exists, dump it - self.print_docstring(indent, code.co_consts[0]) - - code._tokens = None # save memory - assert ast == 'stmts' - - all_globals = find_all_globals(ast, set()) - for g in ((all_globals & self.mod_globs) | find_globals(ast, set())): - self.println(self.indent, 'global ', g) - self.mod_globs -= all_globals - has_none = 'None' in code.co_names - rn = has_none and not find_none(ast) - self.gen_source(ast, code.co_name, code._customize, isLambda=isLambda, - returnNone=rn) - code._tokens = None; code._customize = None # save memory - def build_class(self, code): """Dump class definition, doc string and class body.""" From 1574bf4e1e3b9010364eea5c63edee3dd59d2d84 Mon Sep 17 00:00:00 2001 From: rocky Date: Sat, 29 Oct 2016 16:03:02 -0400 Subject: [PATCH 16/23] More annotation processing in to make_function Move return-value annotation determination from n_mkfunc_annotate to make_function_annotate which is where other kinds of annotation handling will also need to be done. --- uncompyle6/semantics/make_function.py | 17 ++++++++++++++--- uncompyle6/semantics/pysource.py | 11 +---------- 2 files changed, 15 insertions(+), 13 deletions(-) diff --git a/uncompyle6/semantics/make_function.py b/uncompyle6/semantics/make_function.py index dcbd87c2..ede09243 100644 --- a/uncompyle6/semantics/make_function.py +++ b/uncompyle6/semantics/make_function.py @@ -47,7 +47,7 @@ def find_none(node): # FIXME: DRY the below code... def make_function3_annotate(self, node, isLambda, nested=1, - codeNode=None, annotate=None): + codeNode=None, annotate_last=-1): """ Dump function defintion, doc string, and function body. This code is specialized for Python 3""" @@ -70,6 +70,17 @@ def make_function3_annotate(self, node, isLambda, nested=1, # MAKE_FUNCTION_... or MAKE_CLOSURE_... assert node[-1].type.startswith('MAKE_') + annotate_return = None + annotate_arg = node[annotate_last] + + if (annotate_arg == 'annotate_arg' + and annotate_arg[0] == 'LOAD_CONST' + and isinstance(annotate_arg[0].attr, tuple)): + annotate_tup = annotate_arg[0].attr + if annotate_tup[-1] == 'return': + annotate_return = node[annotate_last-1][0].attr + pass + args_node = node[-1] if isinstance(args_node.attr, tuple): # positional args are before kwargs @@ -177,8 +188,8 @@ def make_function3_annotate(self, node, isLambda, nested=1, self.write(": ") else: self.write(')') - if annotate: - self.write(' -> "%s"' % annotate) + if annotate_return: + self.write(' -> "%s"' % annotate_return) self.println(":") if (len(code.co_consts) > 0 and diff --git a/uncompyle6/semantics/pysource.py b/uncompyle6/semantics/pysource.py index 3ef77342..dbbbac0e 100644 --- a/uncompyle6/semantics/pysource.py +++ b/uncompyle6/semantics/pysource.py @@ -597,20 +597,11 @@ class SourceWalker(GenericASTTraversal, object): code = node[-3] self.indentMore() - annotate_return = None annotate_last = -4 if self.version == 3.1 else -5 - annotate_arg = node[annotate_last] - if (annotate_arg == 'annotate_arg' - and annotate_arg[0] == 'LOAD_CONST' - and isinstance(annotate_arg[0].attr, tuple)): - annotate_tup = annotate_arg[0].attr - if annotate_tup[-1] == 'return': - annotate_return = node[annotate_last-1][0].attr - pass # FIXME: handle and pass full annotate args make_function3_annotate(self, node, isLambda=False, - codeNode=code, annotate=annotate_return) + codeNode=code, annotate_last=annotate_last) if len(self.param_stack) > 1: self.write('\n\n') From bfd2f77fbc3a9de68980198423f752a566129772 Mon Sep 17 00:00:00 2001 From: rocky Date: Sun, 30 Oct 2016 10:39:11 -0400 Subject: [PATCH 17/23] More source-code line indention in make_function.. and remove Python 3 situations from make_function2() --- uncompyle6/semantics/make_function.py | 21 ++++++++++++--------- 1 file changed, 12 insertions(+), 9 deletions(-) diff --git a/uncompyle6/semantics/make_function.py b/uncompyle6/semantics/make_function.py index ede09243..cc2bf282 100644 --- a/uncompyle6/semantics/make_function.py +++ b/uncompyle6/semantics/make_function.py @@ -134,7 +134,6 @@ def make_function3_annotate(self, node, isLambda, nested=1, indent = ' ' * l line_number = self.line_number - if 4 & code.co_flags: # flag 2 -> variable number of args self.write('*%s' % code.co_varnames[argc + kw_pairs]) argc += 1 @@ -291,10 +290,7 @@ def make_function2(self, node, isLambda, nested=1, codeNode=None): params.reverse() # back to correct order if 4 & code.co_flags: # flag 2 -> variable number of args - if self.version > 3.0: - params.append('*%s' % code.co_varnames[argc + kw_pairs]) - else: - params.append('*%s' % code.co_varnames[argc]) + params.append('*%s' % code.co_varnames[argc]) argc += 1 # dump parameter list (with default values) @@ -302,7 +298,6 @@ def make_function2(self, node, isLambda, nested=1, codeNode=None): self.write("lambda ", ", ".join(params)) else: self.write("(", ", ".join(params)) - # self.println(indent, '#flags:\t', int(code.co_flags)) if kw_args > 0: if not (4 & code.co_flags): @@ -317,8 +312,6 @@ def make_function2(self, node, isLambda, nested=1, codeNode=None): for n in node: if n == 'pos_arg': continue - elif self.version >= 3.4 and n.type != 'kwargs': - continue else: self.preorder(n) break @@ -452,6 +445,12 @@ def make_function3(self, node, isLambda, nested=1, codeNode=None): self.write("lambda ") else: self.write("(") + pass + + last_line = self.f.getvalue().split("\n")[-1] + l = len(last_line) + indent = ' ' * l + line_number = self.line_number if 4 & code.co_flags: # flag 2 -> variable number of args self.write('*%s' % code.co_varnames[argc + kw_pairs]) @@ -466,7 +465,11 @@ def make_function3(self, node, isLambda, nested=1, codeNode=None): self.write(paramnames[i] + '=') i += 1 self.preorder(n) - suffix = ', ' + if (line_number != self.line_number): + suffix = ",\n" + indent + line_number = self.line_number + else: + suffix = ', ' if kw_args > 0: if not (4 & code.co_flags): From 7700446bb12ce4fc7f81ec9ebc586663847d1532 Mon Sep 17 00:00:00 2001 From: rocky Date: Sun, 30 Oct 2016 21:16:33 -0400 Subject: [PATCH 18/23] Note github unpyc3 and.. - Add source to bytecode_2.2/03_class_method.pyc - more ignore --- .gitignore | 3 ++- README.rst | 1 + test/bytecode_2.2/03_class_method.pyc | Bin 570 -> 570 bytes test/simple_source/def/03_class_method.py | 13 +++++++++++++ 4 files changed, 16 insertions(+), 1 deletion(-) create mode 100644 test/simple_source/def/03_class_method.py diff --git a/.gitignore b/.gitignore index 88363be6..76e99b32 100644 --- a/.gitignore +++ b/.gitignore @@ -1,7 +1,7 @@ +*.pyo *.pyc *_dis *~ -*.pyc /.cache /.eggs /.python-version @@ -13,5 +13,6 @@ /nose-*.egg /tmp /uncompyle6.egg-info +/unpyc __pycache__ build diff --git a/README.rst b/README.rst index 46c1eb08..60b654f6 100644 --- a/README.rst +++ b/README.rst @@ -132,6 +132,7 @@ See Also * https://github.com/zrax/pycdc : supports all versions of Python and is written in C++ * https://code.google.com/archive/p/unpyc3/ : supports Python 3.2 only. The above projects use a different decompiling technique what is used here. +* https://github.com/figment/unpyc3/ : fork of above, but supports Python 3.2 only. Include some fixes like supporting function annotations * The HISTORY_ file. .. |downloads| image:: https://img.shields.io/pypi/dd/uncompyle6.svg diff --git a/test/bytecode_2.2/03_class_method.pyc b/test/bytecode_2.2/03_class_method.pyc index 0304456719ac94b4c94fb26af0e994e7952a63b9..08f9ed7777fbf951f0cf981c363195e09de38945 100644 GIT binary patch delta 17 YcmdnRvWta7_bo5i^BH0h8##=a05&!SkN^Mx delta 17 ZcmdnRvWta7_bo5i@_*~XH*y#;0RTFS22B6} diff --git a/test/simple_source/def/03_class_method.py b/test/simple_source/def/03_class_method.py new file mode 100644 index 00000000..47d635d9 --- /dev/null +++ b/test/simple_source/def/03_class_method.py @@ -0,0 +1,13 @@ +# From Decompyle++ +# File: 22_class_method.pyc (Python 2.2) +# An old-style Python class. + +class MyClass: + + def method(self, i): + if i is 5: + print 'five' + elif not (i is 2): + print 'not two' + else: + print '2' From eab653afdde2d3b43fd044fc7939197b24e7360a Mon Sep 17 00:00:00 2001 From: rocky Date: Tue, 1 Nov 2016 06:05:31 -0400 Subject: [PATCH 19/23] Full Python 3 annotations --- test/bytecode_3.1/04_def_annotate.pyc | Bin 0 -> 699 bytes test/bytecode_3.4/04_def_annotate.pyc | Bin 0 -> 511 bytes test/simple_source/bug31/04_def_annotate.py | 6 ++- uncompyle6/semantics/make_function.py | 48 +++++++++++++++----- 4 files changed, 41 insertions(+), 13 deletions(-) create mode 100644 test/bytecode_3.1/04_def_annotate.pyc create mode 100644 test/bytecode_3.4/04_def_annotate.pyc diff --git a/test/bytecode_3.1/04_def_annotate.pyc b/test/bytecode_3.1/04_def_annotate.pyc new file mode 100644 index 0000000000000000000000000000000000000000..c8a5c0620cd80c980e67c98e7abb951360058a6f GIT binary patch literal 699 zcmbVIJx{|h5Iv`9T2Nu69hp!j3W5|)pgEFzE{kiAy)LB=4vAQRX{ zuuWh=ya%(Cv-tr*rj&M>tJLZ|Pdj`n2=ai1Ply|I-l&4_c<{>eiKaaxZ~z+$0mDrM z!w&(1)Xh>(GYROsL>p5ncPNmx%cOJ<0sI9qTdkWKmsWdI;5hfIo9pq_?NVa7{8o#X z&AVq2rPjRIhyX;$X@cnI>0@SbU%u5ukKfVYl?8?-Vj|SUFUjEy-(1QsJ@3k@#z59u zqG$hdUzSL1t$d$$s0uB4!{yKzZLH_d3Sa9=Vc^!%7M>hBx?}Pm;ViYGL5nt$7hU@?U literal 0 HcmV?d00001 diff --git a/test/bytecode_3.4/04_def_annotate.pyc b/test/bytecode_3.4/04_def_annotate.pyc new file mode 100644 index 0000000000000000000000000000000000000000..881b347fc6908a4a54a898a30c973220bd0023f1 GIT binary patch literal 511 zcmYk2y-ve05XaAno1~>oNEC^M2`LMuf)N1%35lf)Wk6l3B)*hLsU10v_)s_c0DTQ! z$tyb#z{EN2(9?bV<*)mlfBXBro7bb45d-*v9qW<4rKle$2>>K#AjmxsKCC&c0H&ll zh|Z1&AVLrkh%Sg8tUXu-F!kMmNxb*jPoz{PF@>C;q_J;VYFTCjji%8u^Qo0HS6$pw z)Q6M|z>=K7VCli&ivc5|NzBGEvz "IOBase" is problematic +# Python 3 annotations +def foo(a, b: 'annotating b', c: int) -> float: + print(a + b + c) + +# Python 3.1 _pyio.py uses the -> "IOBase" annotation def open(file, mode = "r", buffering = None, encoding = None, errors = None, newline = None, closefd = True) -> "IOBase": diff --git a/uncompyle6/semantics/make_function.py b/uncompyle6/semantics/make_function.py index cc2bf282..dce43bd8 100644 --- a/uncompyle6/semantics/make_function.py +++ b/uncompyle6/semantics/make_function.py @@ -70,22 +70,27 @@ def make_function3_annotate(self, node, isLambda, nested=1, # MAKE_FUNCTION_... or MAKE_CLOSURE_... assert node[-1].type.startswith('MAKE_') - annotate_return = None annotate_arg = node[annotate_last] + annotate_args = {} if (annotate_arg == 'annotate_arg' - and annotate_arg[0] == 'LOAD_CONST' + and annotate_arg[0] in ('LOAD_CONST', 'LOAD_NAME') and isinstance(annotate_arg[0].attr, tuple)): annotate_tup = annotate_arg[0].attr - if annotate_tup[-1] == 'return': - annotate_return = node[annotate_last-1][0].attr - pass + i = -1 + j = annotate_last-1 + l = -len(node) + while j >= l and node[j] == 'annotate_arg': + annotate_args[annotate_tup[i]] = (node[j][0].attr, + node[j][0] == 'LOAD_CONST') + i -= 1 + j -= 1 args_node = node[-1] if isinstance(args_node.attr, tuple): # positional args are before kwargs defparams = node[:args_node.attr[0]] - pos_args, kw_args, annotate_args = args_node.attr + pos_args, kw_args, annotate_argc = args_node.attr else: defparams = node[:args_node.attr] kw_args = 0 @@ -139,12 +144,26 @@ def make_function3_annotate(self, node, isLambda, nested=1, argc += 1 i = len(paramnames) - len(defparams) - self.write(", ".join(paramnames[:i])) + suffix = '' + for param in paramnames[:i]: + self.write(suffix, param) + if param in annotate_args: + value, string = annotate_args[param] + if string: + self.write(': "%s"' % value) + else: + self.write(': %s' % value) + suffix = ', ' + suffix = ', ' if i > 0 else '' for n in node: if n == 'pos_arg': self.write(suffix) - self.write(paramnames[i] + '=') + param = paramnames[i] + self.write(param) + if param in annotate_args: + self.write(':"%s' % annotate_args[param]) + self.write('=') i += 1 self.preorder(n) if (line_number != self.line_number): @@ -187,8 +206,13 @@ def make_function3_annotate(self, node, isLambda, nested=1, self.write(": ") else: self.write(')') - if annotate_return: - self.write(' -> "%s"' % annotate_return) + if 'return' in annotate_args: + value, string = annotate_args['return'] + if string: + self.write(' -> "%s"' % value) + else: + self.write(' -> %s' % value) + self.println(":") if (len(code.co_consts) > 0 and @@ -247,7 +271,7 @@ def make_function2(self, node, isLambda, nested=1, codeNode=None): if isinstance(args_node.attr, tuple): # positional args are after kwargs defparams = node[1:args_node.attr[0]+1] - pos_args, kw_args, annotate_args = args_node.attr + pos_args, kw_args, annotate_argc = args_node.attr else: defparams = node[:args_node.attr] kw_args = 0 @@ -377,7 +401,7 @@ def make_function3(self, node, isLambda, nested=1, codeNode=None): else: # positional args are before kwargs defparams = node[:args_node.attr[0]] - pos_args, kw_args, annotate_args = args_node.attr + pos_args, kw_args, annotate_argc = args_node.attr else: defparams = node[:args_node.attr] kw_args = 0 From 63e4c9343ff141714cb8972bf583f8a6fec4a639 Mon Sep 17 00:00:00 2001 From: rocky Date: Tue, 1 Nov 2016 15:48:21 -0400 Subject: [PATCH 20/23] Clean up annotation grammar a little --- pytest/test_grammar.py | 1 + uncompyle6/parsers/parse3.py | 10 +++++++++- uncompyle6/parsers/parse31.py | 4 ++-- uncompyle6/parsers/parse32.py | 5 +++-- uncompyle6/semantics/make_function.py | 12 ++++++------ 5 files changed, 21 insertions(+), 11 deletions(-) diff --git a/pytest/test_grammar.py b/pytest/test_grammar.py index 3cf98358..c6cc8e02 100644 --- a/pytest/test_grammar.py +++ b/pytest/test_grammar.py @@ -28,6 +28,7 @@ def test_grammar(): """.split())) if 3.0 <= PYTHON_VERSION: expect_lhs.add("annotate_arg") + expect_lhs.add("annotate_tuple") unused_rhs.add("mkfunc_annotate") pass else: diff --git a/uncompyle6/parsers/parse3.py b/uncompyle6/parsers/parse3.py index 233053c0..9b44595d 100644 --- a/uncompyle6/parsers/parse3.py +++ b/uncompyle6/parsers/parse3.py @@ -253,8 +253,16 @@ class Python3Parser(PythonParser): stmt ::= funcdef_annotate funcdef_annotate ::= mkfunc_annotate designator - annotate_arg ::= LOAD_CONST + # This has the annotation value. + # LOAD_NAME is used in an annotation type like + # int, float, str annotate_arg ::= LOAD_NAME + # LOAD_CONST is used in an annotation string + annotate_arg ::= LOAD_CONST + + # This stores the tuple of parameter names + # that have been annotated + annotate_tuple ::= LOAD_CONST """ def p_come_from3(self, args): diff --git a/uncompyle6/parsers/parse31.py b/uncompyle6/parsers/parse31.py index 7bad1b30..647ed421 100644 --- a/uncompyle6/parsers/parse31.py +++ b/uncompyle6/parsers/parse31.py @@ -43,9 +43,9 @@ class Python31Parser(Python32Parser): # Check that there are 2 annotated params? # rule = ('mkfunc2 ::= %s%sEXTENDED_ARG %s' % # ('pos_arg ' * (args_pos), 'kwargs ' * (annotate_args-1), opname)) - rule = ('mkfunc_annotate ::= %s%sLOAD_CONST EXTENDED_ARG %s' % + rule = ('mkfunc_annotate ::= %s%sannotate_tuple LOAD_CONST EXTENDED_ARG %s' % (('pos_arg ' * (args_pos)), - ('annotate_arg ' * (annotate_args)), opname)) + ('annotate_arg ' * (annotate_args-1)), opname)) self.add_unique_rule(rule, opname, token.attr, customize) class Python31ParserSingle(Python31Parser, PythonParserSingle): pass diff --git a/uncompyle6/parsers/parse32.py b/uncompyle6/parsers/parse32.py index 9de0d3cb..d88ee91d 100644 --- a/uncompyle6/parsers/parse32.py +++ b/uncompyle6/parsers/parse32.py @@ -24,9 +24,10 @@ class Python32Parser(Python3Parser): if opname.startswith('MAKE_FUNCTION_A'): args_pos, args_kw, annotate_args = token.attr # Check that there are 2 annotated params? - rule = ('mkfunc_annotate ::= %s%sLOAD_CONST LOAD_CONST EXTENDED_ARG %s' % + rule = (('mkfunc_annotate ::= %s%sannotate_tuple ' + 'LOAD_CONST LOAD_CONST EXTENDED_ARG %s') % (('pos_arg ' * (args_pos)), - ('annotate_arg ' * (annotate_args)), opname)) + ('annotate_arg ' * (annotate_args-1)), opname)) self.add_unique_rule(rule, opname, token.attr, customize) diff --git a/uncompyle6/semantics/make_function.py b/uncompyle6/semantics/make_function.py index dce43bd8..5d207ca9 100644 --- a/uncompyle6/semantics/make_function.py +++ b/uncompyle6/semantics/make_function.py @@ -70,17 +70,17 @@ def make_function3_annotate(self, node, isLambda, nested=1, # MAKE_FUNCTION_... or MAKE_CLOSURE_... assert node[-1].type.startswith('MAKE_') - annotate_arg = node[annotate_last] + annotate_tuple = node[annotate_last] annotate_args = {} - if (annotate_arg == 'annotate_arg' - and annotate_arg[0] in ('LOAD_CONST', 'LOAD_NAME') - and isinstance(annotate_arg[0].attr, tuple)): - annotate_tup = annotate_arg[0].attr + if (annotate_tuple == 'annotate_tuple' + and annotate_tuple[0] in ('LOAD_CONST', 'LOAD_NAME') + and isinstance(annotate_tuple[0].attr, tuple)): + annotate_tup = annotate_tuple[0].attr i = -1 j = annotate_last-1 l = -len(node) - while j >= l and node[j] == 'annotate_arg': + while j >= l and node[j].type in ('annotate_arg' 'annotate_tuple'): annotate_args[annotate_tup[i]] = (node[j][0].attr, node[j][0] == 'LOAD_CONST') i -= 1 From 287e98b4b1839cf211a227ead2df9d48e71d7093 Mon Sep 17 00:00:00 2001 From: rocky Date: Wed, 2 Nov 2016 20:42:31 -0400 Subject: [PATCH 21/23] Update unpyc3 info. --- README.rst | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/README.rst b/README.rst index 60b654f6..a9cb5e1d 100644 --- a/README.rst +++ b/README.rst @@ -20,9 +20,9 @@ Why this? There were a number of decompyle, uncompile, uncompyle2, uncompyle3 forks around. All of them came basically from the same code base, and almost all of them no were no longer actively maintained. Only one -handled Python 3, and even there, only 3.2. This code pulls these -together and moves forward. It also addresses a number of open issues -in the previous forks. +handled Python 3, and even there, only 3.2 or 3.3 depending on which +code is used. This code pulls these together and moves forward. It +also addresses a number of open issues in the previous forks. What makes this different from other CPython bytecode decompilers?: its ability to deparse just fragments and give source-code information @@ -132,7 +132,7 @@ See Also * https://github.com/zrax/pycdc : supports all versions of Python and is written in C++ * https://code.google.com/archive/p/unpyc3/ : supports Python 3.2 only. The above projects use a different decompiling technique what is used here. -* https://github.com/figment/unpyc3/ : fork of above, but supports Python 3.2 only. Include some fixes like supporting function annotations +* https://github.com/figment/unpyc3/ : fork of above, but supports Python 3.3 only. Include some fixes like supporting function annotations * The HISTORY_ file. .. |downloads| image:: https://img.shields.io/pypi/dd/uncompyle6.svg From 2eaea447ebcc3cc6636a98723e88e23925eef26d Mon Sep 17 00:00:00 2001 From: rocky Date: Wed, 2 Nov 2016 22:38:41 -0400 Subject: [PATCH 22/23] Get ready for release 2.9.4 --- ChangeLog | 88 ++++++++++++++++++++++++++++++++++++++++++- NEWS | 6 +++ __pkginfo__.py | 2 +- uncompyle6/version.py | 2 +- 4 files changed, 95 insertions(+), 3 deletions(-) diff --git a/ChangeLog b/ChangeLog index a566d9d5..e5b12139 100644 --- a/ChangeLog +++ b/ChangeLog @@ -1,6 +1,92 @@ +2016-11-02 rocky + + * __pkginfo__.py, uncompyle6/version.py: Get ready for release 2.9.4 + +2016-11-02 rocky + + * README.rst: Update unpyc3 info. + +2016-11-01 rocky + + * pytest/test_grammar.py, uncompyle6/parsers/parse3.py, + uncompyle6/parsers/parse31.py, uncompyle6/parsers/parse32.py, + uncompyle6/semantics/make_function.py: Clean up annotation grammar a + little + +2016-11-01 rocky + + * test/simple_source/bug31/04_def_annotate.py, + uncompyle6/semantics/make_function.py: Full Python 3 annotations + +2016-10-30 rocky + + * .gitignore, README.rst, test/simple_source/def/03_class_method.py: + Note github unpyc3 and.. - Add source to bytecode_2.2/03_class_method.pyc - more ignore + +2016-10-30 rocky + + * uncompyle6/semantics/make_function.py: More source-code line + indention in make_function.. and remove Python 3 situations from make_function2() + +2016-10-29 rocky + + * uncompyle6/semantics/make_function.py, + uncompyle6/semantics/pysource.py: More annotation processing in to + make_function Move return-value annotation determination from n_mkfunc_annotate to + make_function_annotate which is where other kinds of annotation + handling will also need to be done. + +2016-10-29 rocky + + * uncompyle6/semantics/fragments.py, + uncompyle6/semantics/make_function.py, + uncompyle6/semantics/parser_error.py, + uncompyle6/semantics/pysource.py: Break out make_function() into its + own file. It is already too complex and will get worse in Python 3.6. Note: make_function in fragments.py is still inside and probably + needs fixup. + +2016-10-28 rocky + + * pytest/test_grammar.py, uncompyle6/parsers/parse3.py, + uncompyle6/parsers/parse31.py, uncompyle6/parsers/parse32.py, + uncompyle6/parsers/parse35.py, uncompyle6/semantics/pysource.py: + More complete annotate handling Still have a bit of work to do though. + +2016-10-28 rocky + + * pytest/test_grammar.py, uncompyle6/parsers/parse3.py, + uncompyle6/parsers/parse32.py, uncompyle6/parsers/parse33.py, + uncompyle6/parsers/parse34.py, uncompyle6/semantics/pysource.py: + Expand annotate return to Python 3.4 + +2016-10-28 rocky + + * pytest/test_grammar.py, uncompyle6/parsers/parse31.py, + uncompyle6/parsers/parse32.py, uncompyle6/semantics/pysource.py: + Expand annotate handling to 3.3 (and possibly 3.2) - DRY Python 3.1-3.3 grammar a little + +2016-10-28 rocky + + * uncompyle6/parser.py, uncompyle6/parsers/parse3.py, + uncompyle6/parsers/parse31.py, uncompyle6/parsers/parse32.py, + uncompyle6/parsers/parse33.py, uncompyle6/parsers/parse35.py: Split + out 3.1-3.3 parsers from parser3.py This is anticipation of extending annotation to Python 3.2+ + +2016-10-27 rocky + + * test/simple_source/bug31/04_def_annotate.py, + test/simple_source/bug31/04_def_attr.py, + uncompyle6/parsers/parse31.py, uncompyle6/semantics/pysource.py: + Clean and fix Python 3 annotate arg return + 2016-10-26 rocky - * uncompyle6/version.py: Get ready for release 2.9.3 + * __pkginfo__.py: Dependencies stay within 2nd semantic level + +2016-10-26 rocky + + * ChangeLog, NEWS, uncompyle6/version.py: Get ready for release + 2.9.3 2016-10-26 rocky diff --git a/NEWS b/NEWS index 2fd5de75..b65d7177 100644 --- a/NEWS +++ b/NEWS @@ -1,3 +1,9 @@ +uncompyle6 2.9.4 2016-11-02 + +- Handle Python 3.x function annotations +- track def keywoard-parameter line-splitting in source code better +- bump min xdis version to mask previous xdis bug + uncompyle6 2.9.3 2016-10-26 Release forced by incompatiblity change in xdis 3.2.0. diff --git a/__pkginfo__.py b/__pkginfo__.py index a48a2bdb..13a669a1 100644 --- a/__pkginfo__.py +++ b/__pkginfo__.py @@ -38,7 +38,7 @@ entry_points={ ]} ftp_url = None install_requires = ['spark-parser >= 1.4.0, < 1.5.0', - 'xdis >= 3.2.0, < 3.3.0'] + 'xdis >= 3.2.2, < 3.3.0'] license = 'MIT' mailing_list = 'python-debugger@googlegroups.com' modname = 'uncompyle6' diff --git a/uncompyle6/version.py b/uncompyle6/version.py index a353ce9f..b539e846 100644 --- a/uncompyle6/version.py +++ b/uncompyle6/version.py @@ -1,3 +1,3 @@ # This file is suitable for sourcing inside bash as # well as importing into Python -VERSION='2.9.3' +VERSION='2.9.4' From cd3cf5ec2960a733e9fedca9c4549caf33c2d1d0 Mon Sep 17 00:00:00 2001 From: rocky Date: Thu, 3 Nov 2016 21:26:12 -0400 Subject: [PATCH 23/23] Use L. for line number prefix in asm and AST --- uncompyle6/bin/uncompile.py | 2 +- uncompyle6/main.py | 6 +++--- uncompyle6/parsers/astnode.py | 2 +- uncompyle6/scanners/scanner2.py | 2 +- uncompyle6/scanners/tok.py | 6 +++++- uncompyle6/semantics/pysource.py | 9 ++++----- 6 files changed, 15 insertions(+), 12 deletions(-) diff --git a/uncompyle6/bin/uncompile.py b/uncompyle6/bin/uncompile.py index d5284797..69a97d3d 100755 --- a/uncompyle6/bin/uncompile.py +++ b/uncompyle6/bin/uncompile.py @@ -96,7 +96,7 @@ def main_bin(): elif opt == '--verify': options['do_verify'] = True elif opt in ('--asm', '-a'): - options['showasm'] = True + options['showasm'] = 'after' options['do_verify'] = False elif opt in ('--tree', '-t'): options['showast'] = True diff --git a/uncompyle6/main.py b/uncompyle6/main.py index de82d794..e81eff26 100644 --- a/uncompyle6/main.py +++ b/uncompyle6/main.py @@ -11,7 +11,7 @@ from uncompyle6.version import VERSION from xdis.load import load_module def uncompyle( - bytecode_version, co, out=None, showasm=False, showast=False, + bytecode_version, co, out=None, showasm=None, showast=False, timestamp=None, showgrammar=False, code_objects={}, source_size=None, is_pypy=False, magic_int=None): """ @@ -53,7 +53,7 @@ def uncompyle( -def uncompyle_file(filename, outstream=None, showasm=False, showast=False, +def uncompyle_file(filename, outstream=None, showasm=None, showast=False, showgrammar=False): """ decompile Python byte-code file (.pyc) @@ -79,7 +79,7 @@ def uncompyle_file(filename, outstream=None, showasm=False, showast=False, # FIXME: combine into an options parameter def main(in_base, out_base, files, codes, outfile=None, - showasm=False, showast=False, do_verify=False, + showasm=None, showast=False, do_verify=False, showgrammar=False, raise_on_error=False): """ in_base base directory for input files diff --git a/uncompyle6/parsers/astnode.py b/uncompyle6/parsers/astnode.py index 7c66502d..69d06388 100644 --- a/uncompyle6/parsers/astnode.py +++ b/uncompyle6/parsers/astnode.py @@ -33,7 +33,7 @@ class AST(spark_AST): else: child = node.__repr1__(indent, None) else: - inst = str(node) + inst = node.format(line_prefix='L.') if inst.startswith("\n"): # Nuke leading \n inst = inst[1:] diff --git a/uncompyle6/scanners/scanner2.py b/uncompyle6/scanners/scanner2.py index 839887a3..0625de03 100644 --- a/uncompyle6/scanners/scanner2.py +++ b/uncompyle6/scanners/scanner2.py @@ -291,7 +291,7 @@ class Scanner2(scan.Scanner): if show_asm in ('both', 'after'): for t in tokens: - print(t) + print(t.format(line_prefix='L.')) print() return tokens, customize diff --git a/uncompyle6/scanners/tok.py b/uncompyle6/scanners/tok.py index f9879334..a260a24c 100644 --- a/uncompyle6/scanners/tok.py +++ b/uncompyle6/scanners/tok.py @@ -53,7 +53,11 @@ class Token: # ('%9s %-18s %r' % (self.offset, self.type, pattr))) def __str__(self): - prefix = '\n%4d ' % self.linestart if self.linestart else (' ' * 6) + return self.format(line_prefix='') + + def format(self, line_prefix=''): + prefix = ('\n%s%4d ' % (line_prefix, self.linestart) + if self.linestart else (' ' * (6 + len(line_prefix)))) offset_opname = '%6s %-17s' % (self.offset, self.type) if not self.has_arg: return "%s%s" % (prefix, offset_opname) diff --git a/uncompyle6/semantics/pysource.py b/uncompyle6/semantics/pysource.py index dbbbac0e..8b0eef72 100644 --- a/uncompyle6/semantics/pysource.py +++ b/uncompyle6/semantics/pysource.py @@ -2264,7 +2264,7 @@ class SourceWalker(GenericASTTraversal, object): return MAP.get(node, MAP_DIRECT) -def deparse_code(version, co, out=sys.stdout, showasm=False, showast=False, +def deparse_code(version, co, out=sys.stdout, showasm=None, showast=False, showgrammar=False, code_objects={}, compile_mode='exec', is_pypy=False): """ ingests and deparses a given code block 'co' @@ -2274,8 +2274,7 @@ def deparse_code(version, co, out=sys.stdout, showasm=False, showast=False, # store final output stream for case of error scanner = get_scanner(version, is_pypy=is_pypy) - tokens, customize = scanner.ingest(co, code_objects=code_objects) - maybe_show_asm(showasm, tokens) + tokens, customize = scanner.ingest(co, code_objects=code_objects, show_asm=showasm) debug_parser = dict(PARSER_DEFAULT_DEBUG) if showgrammar: @@ -2323,8 +2322,8 @@ if __name__ == '__main__': def deparse_test(co): "This is a docstring" sys_version = sys.version_info.major + (sys.version_info.minor / 10.0) - deparsed = deparse_code(sys_version, co, showasm=True, showast=True) - # deparsed = deparse_code(sys_version, co, showasm=False, showast=False, + deparsed = deparse_code(sys_version, co, showasm='after', showast=True) + # deparsed = deparse_code(sys_version, co, showasm=None, showast=False, # showgrammar=True) print(deparsed.text) return