diff --git a/test/simple_source/bug36/02_genexpr.py b/test/simple_source/bug36/02_genexpr.py index 9ea0dea8..755108dc 100644 --- a/test/simple_source/bug36/02_genexpr.py +++ b/test/simple_source/bug36/02_genexpr.py @@ -1,5 +1,5 @@ # Python 3.6, uses rule: # genexpr ::= load_closure load_genexpr LOAD_CONST -# MAKE_FUNCTION_8 expr GET_ITER CALL_FUNCTION_1 +# MAKE_FUNCTION_CLOSURE expr GET_ITER CALL_FUNCTION_1 def __sub__(self, other): # SList()-other return self.__class__(i for i in self if i not in other) diff --git a/test/simple_source/bug36/05_36lambda.py b/test/simple_source/bug36/05_36lambda.py index 3ed67f74..29bc037a 100644 --- a/test/simple_source/bug36/05_36lambda.py +++ b/test/simple_source/bug36/05_36lambda.py @@ -4,8 +4,8 @@ def __init__(self, msg = None, digestmod = None): self.digest_cons = lambda d='': digestmod.new(d) # From Python 3.6 functools.py -# Bug was handling lambda for MAKE_FUNCTION_8 (closure) -# vs to MAKE_FUNCTION_9 (pos_args + closure) +# Bug was handling lambda for MAKE_FUNCTION_CLOSURE (closure) +# vs to MAKE_FUNCTION_CLOSURE_POS (pos_args + closure) def bug(): def register(cls, func=None): return lambda f: register(cls, f) diff --git a/uncompyle6/bin/uncompile.py b/uncompyle6/bin/uncompile.py index 291066f2..bde47700 100755 --- a/uncompyle6/bin/uncompile.py +++ b/uncompyle6/bin/uncompile.py @@ -133,7 +133,8 @@ def main_bin(): elif opt in ('--tree+', '-T'): if 'showast' not in options: options['showast'] = {} - options['showast']['Full'] = True + options['showast']['after'] = True + options['showast']['before'] = True options['do_verify'] = None elif opt in ('--grammar', '-g'): options['showgrammar'] = True diff --git a/uncompyle6/main.py b/uncompyle6/main.py index c1f45778..9fd31f3d 100644 --- a/uncompyle6/main.py +++ b/uncompyle6/main.py @@ -1,4 +1,4 @@ -# Copyright (C) 2018-2021 Rocky Bernstein +# Copyright (C) 2018-2022 Rocky Bernstein # # This program is free software: you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by @@ -19,6 +19,7 @@ from xdis import iscode from xdis.version_info import IS_PYPY, PYTHON_VERSION_TRIPLE, version_tuple_to_str from uncompyle6.disas import check_object_path from uncompyle6.semantics import pysource +from uncompyle6.semantics.pysource import PARSER_DEFAULT_DEBUG from uncompyle6.parser import ParserError from uncompyle6.version import __version__ @@ -42,9 +43,9 @@ def _get_outstream(outfile): return open(outfile, 'wb') def decompile( - bytecode_version, co, - out=None, + bytecode_version = PYTHON_VERSION_TRIPLE, + out=sys.stdout, showasm=None, showast={}, timestamp=None, @@ -98,7 +99,7 @@ def decompile( write("# -*- coding: %s -*-" % source_encoding) write( "# uncompyle6 version %s\n" - "# %sPython bytecode %s%s\n# Decompiled from: %sPython %s" + "# %sPython bytecode version base %s%s\n# Decompiled from: %sPython %s" % ( __version__, co_pypy_str, @@ -107,10 +108,6 @@ def decompile( "\n# ".join(sys_version_lines), ) ) - if bytecode_version >= 3.0: - write( - "# Warning: this version of Python has problems handling the Python 3 byte type in constants properly.\n" - ) if co.co_filename: write("# Embedded file name: %s" % co.co_filename) if timestamp: @@ -120,7 +117,17 @@ def decompile( real_out.write("# Size of source mod 2**32: %d bytes\n" % source_size) - debug_opts = {"asm": showasm, "ast": showast, "grammar": showgrammar} + # maybe a second -a will do before as well + if showasm: + asm = "after" + else: + asm = None + + grammar = dict(PARSER_DEFAULT_DEBUG) + if showgrammar: + grammar["reduce"] = True + + debug_opts = {"asm": asm, "tree": showast, "grammar": grammar} try: if mapstream: @@ -128,10 +135,12 @@ def decompile( mapstream = _get_outstream(mapstream) deparsed = deparse_code_with_map( + bytecode_version, co, out, - bytecode_version, - debug_opts, + showasm, + showast, + showgrammar, code_objects=code_objects, is_pypy=is_pypy, compile_mode=compile_mode, @@ -182,7 +191,7 @@ def decompile_file( filename, outstream=None, showasm=None, - showast=False, + showast={}, showgrammar=False, source_encoding=None, mapstream=None, @@ -201,11 +210,11 @@ def decompile_file( if isinstance(co, list): deparsed = [] - for con in co: + for bytecode in co: deparsed.append( decompile( + bytecode, version, - con, outstream, showasm, showast, @@ -215,14 +224,14 @@ def decompile_file( code_objects=code_objects, is_pypy=is_pypy, magic_int=magic_int, + mapstream=mapstream, ), - mapstream=mapstream, ) else: deparsed = [ decompile( - version, co, + version, outstream, showasm, showast, @@ -235,6 +244,7 @@ def decompile_file( magic_int=magic_int, mapstream=mapstream, do_fragments=do_fragments, + compile_mode="exec", ) ] co = None @@ -249,7 +259,7 @@ def main( source_files, outfile=None, showasm=None, - showast=False, + showast={}, do_verify=False, showgrammar=False, source_encoding=None, diff --git a/uncompyle6/parsers/parse36.py b/uncompyle6/parsers/parse36.py index fb6d034e..604d9fca 100644 --- a/uncompyle6/parsers/parse36.py +++ b/uncompyle6/parsers/parse36.py @@ -219,19 +219,19 @@ class Python36Parser(Python35Parser): formatted_value2 ::= expr expr FORMAT_VALUE_ATTR """ self.add_unique_doc_rules(rules_str, customize) - elif opname == 'MAKE_FUNCTION_8': + elif opname == 'MAKE_FUNCTION_CLOSURE': if 'LOAD_DICTCOMP' in self.seen_ops: # Is there something general going on here? rule = """ dict_comp ::= load_closure LOAD_DICTCOMP LOAD_STR - MAKE_FUNCTION_8 expr + MAKE_FUNCTION_CLOSURE expr GET_ITER CALL_FUNCTION_1 """ self.addRule(rule, nop_func) elif 'LOAD_SETCOMP' in self.seen_ops: rule = """ set_comp ::= load_closure LOAD_SETCOMP LOAD_STR - MAKE_FUNCTION_8 expr + MAKE_FUNCTION_CLOSURE expr GET_ITER CALL_FUNCTION_1 """ self.addRule(rule, nop_func) diff --git a/uncompyle6/parsers/parse37.py b/uncompyle6/parsers/parse37.py index bf5a6565..57942e30 100644 --- a/uncompyle6/parsers/parse37.py +++ b/uncompyle6/parsers/parse37.py @@ -1204,19 +1204,19 @@ class Python37Parser(Python37BaseParser): formatted_value2 ::= expr expr FORMAT_VALUE_ATTR """ self.add_unique_doc_rules(rules_str, customize) - elif opname == "MAKE_FUNCTION_8": + elif opname == "MAKE_FUNCTION_CLOSURE": if "LOAD_DICTCOMP" in self.seen_ops: # Is there something general going on here? rule = """ dict_comp ::= load_closure LOAD_DICTCOMP LOAD_STR - MAKE_FUNCTION_8 expr + MAKE_FUNCTION_CLOSURE expr GET_ITER CALL_FUNCTION_1 """ self.addRule(rule, nop_func) elif "LOAD_SETCOMP" in self.seen_ops: rule = """ set_comp ::= load_closure LOAD_SETCOMP LOAD_STR - MAKE_FUNCTION_8 expr + MAKE_FUNCTION_CLOSURE expr GET_ITER CALL_FUNCTION_1 """ self.addRule(rule, nop_func) diff --git a/uncompyle6/parsers/parse37base.py b/uncompyle6/parsers/parse37base.py index de54f05a..ba51790e 100644 --- a/uncompyle6/parsers/parse37base.py +++ b/uncompyle6/parsers/parse37base.py @@ -584,6 +584,21 @@ class Python37BaseParser(PythonParser): """ self.add_unique_doc_rules(rules_str, customize) + elif opname == "GET_ANEXT": + self.addRule( + """ + func_async_prefix ::= _come_froms SETUP_FINALLY GET_ANEXT LOAD_CONST YIELD_FROM POP_BLOCK + func_async_middle ::= JUMP_FORWARD COME_FROM_EXCEPT + DUP_TOP LOAD_GLOBAL COMPARE_OP POP_JUMP_IF_TRUE + list_afor2 ::= func_async_prefix + store list_iter + JUMP_BACK COME_FROM_FINALLY + END_ASYNC_FOR + """, + nop_func, + ) + custom_ops_processed.add(opname) + elif opname == "FORMAT_VALUE_ATTR": rules_str = """ expr ::= formatted_value2 @@ -932,19 +947,19 @@ class Python37BaseParser(PythonParser): ) self.add_unique_rule(rule, opname, token.attr, customize) - elif opname == "MAKE_FUNCTION_8": + elif opname == "MAKE_FUNCTION_CLOSURE": if "LOAD_DICTCOMP" in self.seen_ops: # Is there something general going on here? rule = """ dict_comp ::= load_closure LOAD_DICTCOMP LOAD_STR - MAKE_FUNCTION_8 expr + MAKE_FUNCTION_CLOSURE expr GET_ITER CALL_FUNCTION_1 """ self.addRule(rule, nop_func) elif "LOAD_SETCOMP" in self.seen_ops: rule = """ set_comp ::= load_closure LOAD_SETCOMP LOAD_STR - MAKE_FUNCTION_8 expr + MAKE_FUNCTION_CLOSURE expr GET_ITER CALL_FUNCTION_1 """ self.addRule(rule, nop_func) diff --git a/uncompyle6/parsers/parse38.py b/uncompyle6/parsers/parse38.py index 3549ca61..01183d7a 100644 --- a/uncompyle6/parsers/parse38.py +++ b/uncompyle6/parsers/parse38.py @@ -74,6 +74,11 @@ class Python38Parser(Python37Parser): COME_FROM_FINALLY END_ASYNC_FOR + genexpr_func_async ::= LOAD_FAST func_async_prefix + store comp_iter + JUMP_BACK COME_FROM_FINALLY + END_ASYNC_FOR + # FIXME: come froms after the else_suite or END_ASYNC_FOR distinguish which of # for / forelse is used. Add come froms and check of add up control-flow detection phase. async_forelse_stmt38 ::= expr diff --git a/uncompyle6/scanners/scanner3.py b/uncompyle6/scanners/scanner3.py index cf0909a8..c82396aa 100644 --- a/uncompyle6/scanners/scanner3.py +++ b/uncompyle6/scanners/scanner3.py @@ -397,7 +397,13 @@ class Scanner3(Scanner): if self.version >= (3, 6): # 3.6+ doesn't have MAKE_CLOSURE, so opname == 'MAKE_FUNCTION' flags = argval - opname = "MAKE_FUNCTION_%d" % (flags) + # FIXME: generalize this + if flags == 8: + opname = "MAKE_FUNCTION_CLOSURE" + elif flags == 9: + opname = "MAKE_FUNCTION_CLOSURE_POS" + else: + opname = "MAKE_FUNCTION_%d" % (flags) attr = [] for flag in self.MAKE_FUNCTION_FLAGS: bit = flags & 1 diff --git a/uncompyle6/semantics/aligner.py b/uncompyle6/semantics/aligner.py index c452e93d..1cc68a56 100644 --- a/uncompyle6/semantics/aligner.py +++ b/uncompyle6/semantics/aligner.py @@ -1,4 +1,4 @@ -# Copyright (c) 2018 by Rocky Bernstein +# Copyright (c) 2018, 2022 by Rocky Bernstein # # This program is free software: you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by @@ -141,8 +141,8 @@ def code_deparse_align(co, out=sys.stderr, version=None, is_pypy=None, debug_parser=debug_parser, compile_mode=compile_mode, is_pypy = is_pypy) - isTopLevel = co.co_name == '' - deparsed.ast = deparsed.build_ast(tokens, customize, co, isTopLevel=isTopLevel) + is_top_level_module = co.co_name == '' + deparsed.ast = deparsed.build_ast(tokens, customize, co, is_top_level_module=is_top_level_module) assert deparsed.ast == 'stmts', 'Should have parsed grammar start' diff --git a/uncompyle6/semantics/consts.py b/uncompyle6/semantics/consts.py index c2236053..197bec88 100644 --- a/uncompyle6/semantics/consts.py +++ b/uncompyle6/semantics/consts.py @@ -1,4 +1,4 @@ -# Copyright (c) 2017-2021 by Rocky Bernstein +# Copyright (c) 2017-2022 by Rocky Bernstein # # This program is free software: you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by @@ -44,6 +44,9 @@ maxint = sys.maxint # say to 100, to make sure we avoid additional prenthesis in # call((.. op ..)). +NO_PARENTHESIS_EVER = 100 + +# fmt: off PRECEDENCE = { "named_expr": 40, # := "yield": 38, # Needs to be below named_expr @@ -168,11 +171,14 @@ TABLE_R = { "DELETE_ATTR": ("%|del %c.%[-1]{pattr}\n", 0), } -TABLE_R0 = { - # "BUILD_LIST": ( "[%C]", (0,-1,", ") ), - # "BUILD_TUPLE": ( "(%C)", (0,-1,", ") ), - # "CALL_FUNCTION": ( "%c(%P)", 0, (1,-1,", ") ), -} +# I'll leave this in for historical interest. +# TABLE_R0 it was like TABLE_R but the key was the *child* of the last child, +# or a grandchild of the node that this is considered. +# TABLE_R0 = { +# "BUILD_LIST": ( "[%C]", (0,-1,", ") ), +# "BUILD_TUPLE": ( "(%C)", (0,-1,", ") ), +# "CALL_FUNCTION": ( "%c(%P)", 0, (1,-1,", ") ), +# } TABLE_DIRECT = { "BINARY_ADD": ("+",), @@ -236,8 +242,19 @@ TABLE_DIRECT = { (0, "expr", PRECEDENCE["subscript"]), (1, "expr"), ), - "subscript": ("%p[%c]", (0, "expr", PRECEDENCE["subscript"]), (1, "expr")), - "subscript2": ("%p[%c]", (0, "expr", PRECEDENCE["subscript"]), (1, "expr")), + + "subscript": ( + "%p[%p]", + (0, "expr", PRECEDENCE["subscript"]), + (1, "expr", NO_PARENTHESIS_EVER) + ), + + "subscript2": ( + "%p[%p]", + (0, "expr", PRECEDENCE["subscript"]), + (1, "expr", NO_PARENTHESIS_EVER) + ), + "store_subscript": ("%p[%c]", (0, "expr", PRECEDENCE["subscript"]), (1, "expr")), "STORE_FAST": ("%{pattr}",), "STORE_NAME": ("%{pattr}",), @@ -427,7 +444,6 @@ TABLE_DIRECT = { MAP_DIRECT = (TABLE_DIRECT,) -MAP_R0 = (TABLE_R0, -1, 0) MAP_R = (TABLE_R, -1) MAP = { @@ -435,7 +451,6 @@ MAP = { "call": MAP_R, "delete": MAP_R, "store": MAP_R, - "exprlist": MAP_R0, } ASSIGN_TUPLE_PARAM = lambda param_name: SyntaxTree( diff --git a/uncompyle6/semantics/customize3.py b/uncompyle6/semantics/customize3.py index beccb2e8..07be2a36 100644 --- a/uncompyle6/semantics/customize3.py +++ b/uncompyle6/semantics/customize3.py @@ -154,6 +154,7 @@ def customize_for_version3(self, version): # recurse one step n = n[0] + # FIXME: adjust for set comprehension if n == "list_for": stores.append(n[2]) n = n[3] @@ -168,13 +169,12 @@ def customize_for_version3(self, version): c = c[0] collections.append(c) pass - elif n in ("list_if", "list_if_not"): - # FIXME: just a guess + elif n in ("list_if", "list_if_not", "list_if_or_not"): if n[0].kind == "expr": list_ifs.append(n) else: list_ifs.append([1]) - n = n[2] + n = n[-2] if n[-1] == "come_from_opt" else n[-1] pass elif n == "list_if37": list_ifs.append(n) @@ -184,7 +184,7 @@ def customize_for_version3(self, version): collections.append(n[0][0]) n = n[1] stores.append(n[1][0]) - n = n[3] + n = n[2] if n[2].kind == "list_iter" else n[3] pass assert n == "lc_body", ast diff --git a/uncompyle6/semantics/customize36.py b/uncompyle6/semantics/customize36.py index 1074fca8..9df98ad1 100644 --- a/uncompyle6/semantics/customize36.py +++ b/uncompyle6/semantics/customize36.py @@ -338,7 +338,7 @@ def customize_for_version36(self, version): kwargs = kwargs[0] call_function_ex = node[-1] assert call_function_ex == "CALL_FUNCTION_EX_KW" or ( - self.version >= 3.6 and call_function_ex == "CALL_FUNCTION_EX" + self.version >= (3, 6) and call_function_ex == "CALL_FUNCTION_EX" ) # FIXME: decide if the below test be on kwargs == 'dict' if ( diff --git a/uncompyle6/semantics/fragments.py b/uncompyle6/semantics/fragments.py index 5d484cae..d2166508 100644 --- a/uncompyle6/semantics/fragments.py +++ b/uncompyle6/semantics/fragments.py @@ -1,4 +1,4 @@ -# Copyright (c) 2015-2019, 2021 by Rocky Bernstein +# Copyright (c) 2015-2019, 2021-2022 by Rocky Bernstein # # This program is free software: you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by @@ -64,6 +64,7 @@ The node position 0 will be associated with "import". # FIXME: DRY code with pysource import re +from StringIO import StringIO from uncompyle6.semantics import pysource from uncompyle6 import parser @@ -75,7 +76,7 @@ from uncompyle6.show import maybe_show_asm, maybe_show_tree from uncompyle6.parsers.treenode import SyntaxTree -from uncompyle6.semantics.pysource import ParserError, StringIO +from uncompyle6.semantics.pysource import ParserError from xdis import iscode from xdis.version_info import IS_PYPY, PYTHON_VERSION_TRIPLE @@ -628,32 +629,6 @@ class FragmentsWalker(pysource.SourceWalker, object): self.indent_less() self.prune() # stop recursing - def n_list_comp(self, node): - """List comprehensions""" - p = self.prec - self.prec = 27 - n = node[-1] - assert n == "list_iter" - # find innermost node - while n == "list_iter": - n = n[0] # recurse one step - if n == "list_for": - n = n[3] - elif n == "list_if": - n = n[2] - elif n == "list_if_not": - n = n[2] - assert n == "lc_body" - if node[0].kind.startswith("BUILD_LIST"): - start = len(self.f.getvalue()) - self.set_pos_info(node[0], start, start + 1) - self.write("[ ") - self.preorder(n[0]) # lc_body - self.preorder(node[-1]) # for/if parts - self.write(" ]") - self.prec = p - self.prune() # stop recursing - def comprehension_walk(self, node, iter_index, code_index=-5): p = self.prec self.prec = 27 @@ -946,7 +921,7 @@ class FragmentsWalker(pysource.SourceWalker, object): self.set_pos_info(node[0], start - 1, start) self.comprehension_walk3(node, 1, 0) elif node[0].kind == "load_closure": - self.setcomprehension_walk3(node, collection_index=4) + self.closure_walk(node, collection_index=4) else: self.comprehension_walk(node, iter_index=4) self.write("}") @@ -1011,7 +986,7 @@ class FragmentsWalker(pysource.SourceWalker, object): ): self.set_pos_info(node[1], node[0][0].start, node[0][0].finish) - def setcomprehension_walk3(self, node, collection_index): + def closure_walk(self, node, collection_index): """Set comprehensions the way they are done in Python3. They're more other comprehensions, e.g. set comprehensions See if we can combine code. @@ -1185,7 +1160,7 @@ class FragmentsWalker(pysource.SourceWalker, object): code, is_lambda=False, noneInNames=False, - isTopLevel=False, + is_top_level_module=False, ): # FIXME: DRY with pysource.py @@ -1227,7 +1202,7 @@ class FragmentsWalker(pysource.SourceWalker, object): # Python 3.4's classes can add a "return None" which is # invalid syntax. if tokens[-2].kind == "LOAD_CONST": - if isTopLevel or tokens[-2].pattr is None: + if is_top_level_module or tokens[-2].pattr is None: del tokens[-2:] else: tokens.append(Token("RETURN_LAST")) @@ -2102,8 +2077,8 @@ def code_deparse( is_pypy=is_pypy, ) - isTopLevel = co.co_name == "" - deparsed.ast = deparsed.build_ast(tokens, customize, co, isTopLevel=isTopLevel) + is_top_level_module = co.co_name == "" + deparsed.ast = deparsed.build_ast(tokens, customize, co, is_top_level_module=is_top_level_module) assert deparsed.ast == "stmts", "Should have parsed grammar start" diff --git a/uncompyle6/semantics/pysource.py b/uncompyle6/semantics/pysource.py index 75e43eeb..bc4e9b98 100644 --- a/uncompyle6/semantics/pysource.py +++ b/uncompyle6/semantics/pysource.py @@ -43,7 +43,7 @@ Python. # describe rules and not have to create methods at all. # # So another other way to specify a semantic rule for a nonterminal is via -# one of the tables MAP_R0, MAP_R, or MAP_DIRECT where the key is the +# either tables MAP_R, or MAP_DIRECT where the key is the # nonterminal name. # # These dictionaries use a printf-like syntax to direct substitution @@ -63,15 +63,14 @@ Python. # parse tree for N. # # -# N&K N N -# / | ... \ / | ... \ / | ... \ -# O O O O O K O O O -# | -# K -# TABLE_DIRECT TABLE_R TABLE_R0 +# N&K N +# / | ... \ / | ... \ +# O O O O O K +# +# +# TABLE_DIRECT TABLE_R # # The default table is TABLE_DIRECT mapping By far, most rules used work this way. -# TABLE_R0 is rarely used. # # The key K is then extracted from the subtree and used to find one # of the tables, T listed above. The result after applying T[K] is @@ -139,7 +138,7 @@ from xdis.version_info import PYTHON_VERSION_TRIPLE from uncompyle6.parser import get_python_parser from uncompyle6.parsers.treenode import SyntaxTree -from spark_parser import GenericASTTraversal, DEFAULT_DEBUG as PARSER_DEFAULT_DEBUG +from spark_parser import GenericASTTraversal from uncompyle6.scanner import Code, get_scanner import uncompyle6.parser as python_parser from uncompyle6.semantics.check_ast import checker @@ -185,6 +184,25 @@ from StringIO import StringIO DEFAULT_DEBUG_OPTS = {"asm": False, "tree": False, "grammar": False} +def unicode(x): return x +from StringIO import StringIO + +PARSER_DEFAULT_DEBUG = { + "rules": False, + "transition": False, + "reduce": False, + "errorstack": "full", + "context": True, + "dups": False, +} + +TREE_DEFAULT_DEBUG = {"before": False, "after": False} + +DEFAULT_DEBUG_OPTS = { + "asm": False, + "tree": TREE_DEFAULT_DEBUG, + "grammar": dict(PARSER_DEFAULT_DEBUG), +} class SourceWalkerError(Exception): def __init__(self, errmsg): @@ -202,7 +220,7 @@ class SourceWalker(GenericASTTraversal, object): version, out, scanner, - showast=False, + showast=TREE_DEFAULT_DEBUG, debug_parser=PARSER_DEFAULT_DEBUG, compile_mode="exec", is_pypy=IS_PYPY, @@ -225,9 +243,9 @@ class SourceWalker(GenericASTTraversal, object): mode that was used to create the Syntax Tree and specifies a gramar variant within a Python version to use. - `is_pypy' should be True if the Syntax Tree was generated for PyPy. + `is_pypy` should be True if the Syntax Tree was generated for PyPy. - `linestarts' is a dictionary of line number to bytecode offset. This + `linestarts` is a dictionary of line number to bytecode offset. This can sometimes assist in determinte which kind of source-code construct to use when there is ambiguity. @@ -244,9 +262,10 @@ class SourceWalker(GenericASTTraversal, object): is_pypy=is_pypy, ) - self.treeTransform = TreeTransform( - version=version, show_ast=showast, is_pypy=is_pypy - ) + # Initialize p_lambda on demand + self.p_lambda = None + + self.treeTransform = TreeTransform(version=self.version, show_ast=showast) self.debug_parser = dict(debug_parser) self.showast = showast self.params = params @@ -286,25 +305,28 @@ class SourceWalker(GenericASTTraversal, object): # An example is: # __module__ = __name__ self.hide_internal = True - self.compile_mode = "exec" + self.compile_mode = compile_mode self.name = None self.version = version self.is_pypy = is_pypy customize_for_version(self, is_pypy, version) - return - def maybe_show_tree(self, ast): - if self.showast and self.treeTransform.showast: + def maybe_show_tree(self, ast, phase): + if self.showast.get("before", False): self.println( """ ---- end before transform +""" + ) + if self.showast.get("after", False): + self.println( + """ ---- begin after transform """ - + " " + + " " ) - - if isinstance(self.showast, dict) and self.showast.get: + if self.showast.get(phase, False): maybe_show_tree(self, ast) def str_with_template(self, ast): @@ -586,8 +608,10 @@ class SourceWalker(GenericASTTraversal, object): self.prec = 6 # print("XXX", n.kind, p, "<", self.prec) + # print(self.f.getvalue()) if p < self.prec: + # print(f"PREC {p}, {node[0].kind}") self.write("(") self.preorder(node[0]) self.write(")") @@ -1111,8 +1135,8 @@ class SourceWalker(GenericASTTraversal, object): ast = ast[0] n = ast[iter_index] - assert n == "comp_iter", n + assert n == "comp_iter", n.kind # Find the comprehension body. It is the inner-most # node that is not list_.. . while n == "comp_iter": # list_iter @@ -1154,10 +1178,24 @@ class SourceWalker(GenericASTTraversal, object): code_index = -6 if self.version > (3, 6): # Python 3.7+ adds optional "come_froms" at node[0] - iter_index = 4 + if node[0].kind in ("load_closure", "load_genexpr") and self.version >= (3, 8): + is_lambda = self.is_lambda + if node[0].kind == "load_genexpr": + self.is_lambda = False + self.closure_walk(node, collection_index=4) + self.is_lambda = is_lambda + else: + code_index = -6 + if self.version < (3, 8): + iter_index = 4 + else: + iter_index = 3 + self.comprehension_walk(node, iter_index=iter_index, code_index=code_index) + pass + pass else: code_index = -5 - self.comprehension_walk(node, iter_index=iter_index, code_index=code_index) + self.comprehension_walk(node, iter_index=iter_index, code_index=code_index) self.write(")") self.prune() @@ -1168,7 +1206,7 @@ class SourceWalker(GenericASTTraversal, object): if node[0] in ["LOAD_SETCOMP", "LOAD_DICTCOMP"]: self.comprehension_walk_newer(node, 1, 0) elif node[0].kind == "load_closure" and self.version >= (3, 0): - self.setcomprehension_walk3(node, collection_index=4) + self.closure_walk(node, collection_index=4) else: self.comprehension_walk(node, iter_index=4) self.write("}") @@ -1180,15 +1218,19 @@ class SourceWalker(GenericASTTraversal, object): """Non-closure-based comprehensions the way they are done in Python3 and some Python 2.7. Note: there are also other set comprehensions. """ + # FIXME: DRY with listcomp_closure3 p = self.prec self.prec = 27 code_obj = node[code_index].attr assert iscode(code_obj), node[code_index] + self.debug_opts["asm"] code = Code(code_obj, self.scanner, self.currentclass, self.debug_opts["asm"]) - ast = self.build_ast(code._tokens, code._customize, code) + ast = self.build_ast( + code._tokens, code._customize, code, is_lambda=self.is_lambda + ) self.customize(code._customize) # skip over: sstmt, stmt, return, return_expr @@ -1336,7 +1378,6 @@ class SourceWalker(GenericASTTraversal, object): else: self.preorder(store) - # FIXME this is all merely approximate self.write(" in ") self.preorder(node[in_node_index]) @@ -1356,6 +1397,7 @@ class SourceWalker(GenericASTTraversal, object): self.write(" if ") if have_not: self.write("not ") + pass self.prec = 27 self.preorder(if_node) pass @@ -1375,32 +1417,86 @@ class SourceWalker(GenericASTTraversal, object): self.write("]") self.prune() - def setcomprehension_walk3(self, node, collection_index): - """Set comprehensions the way they are done in Python3. - They're more other comprehensions, e.g. set comprehensions - See if we can combine code. + def get_comprehension_function(self, node, code_index): + """ + Build the body of a comprehension function and then + find the comprehension node buried in the tree which may + be surrounded with start-like symbols or dominiators,. + """ + self.prec = 27 + code_node = node[code_index] + if code_node == "load_genexpr": + code_node = code_node[0] + + code_obj = code_node.attr + assert iscode(code_obj), code_node + + code = Code(code_obj, self.scanner, self.currentclass, self.debug_opts["asm"]) + + # FIXME: is there a way we can avoid this? + # The problem is that in filterint top-level list comprehensions we can + # encounter comprehensions of other kinds, and lambdas + if self.compile_mode in ("listcomp",): # add other comprehensions to this list + p_save = self.p + self.p = get_python_parser( + self.version, compile_mode="exec", is_pypy=self.is_pypy, + ) + tree = self.build_ast( + code._tokens, code._customize, code, is_lambda=self.is_lambda + ) + self.p = p_save + else: + tree = self.build_ast( + code._tokens, code._customize, code, is_lambda=self.is_lambda + ) + + self.customize(code._customize) + + # skip over: sstmt, stmt, return, return_expr + # and other singleton derivations + if tree == "lambda_start": + if tree[0] in ("dom_start", "dom_start_opt"): + tree = tree[1] + + while len(tree) == 1 or ( + tree in ("stmt", "sstmt", "return", "return_expr", "return_expr_lambda") + ): + self.prec = 100 + tree = tree[0] + return tree + + def closure_walk(self, node, collection_index): + """Dictionary and comprehensions using closure the way they are done in Python3. """ p = self.prec self.prec = 27 - code = Code(node[1].attr, self.scanner, self.currentclass) - ast = self.build_ast(code._tokens, code._customize, code) - self.customize(code._customize) + if node[0] == "load_genexpr": + code_index = 0 + else: + code_index = 1 + tree = self.get_comprehension_function(node, code_index=code_index) # Remove single reductions as in ("stmts", "sstmt"): - while len(ast) == 1: - ast = ast[0] + while len(tree) == 1: + tree = tree[0] - store = ast[3] + store = tree[3] collection = node[collection_index] - n = ast[4] + if tree == "genexpr_func_async": + iter_index = 3 + else: + iter_index = 4 + + n = tree[iter_index] list_if = None assert n == "comp_iter" # Find inner-most node. while n == "comp_iter": n = n[0] # recurse one step + # FIXME: adjust for set comprehension if n == "list_for": store = n[2] @@ -1419,7 +1515,7 @@ class SourceWalker(GenericASTTraversal, object): pass pass - assert n == "comp_body", ast + assert n == "comp_body", tree self.preorder(n[0]) self.write(" for ") @@ -1813,6 +1909,7 @@ class SourceWalker(GenericASTTraversal, object): self.kv_map(node[-1], sep, line_number, indent) pass + pass if sep.startswith(",\n"): self.write(sep[1:]) if node[0] != "dict_entry": @@ -1874,6 +1971,7 @@ class SourceWalker(GenericASTTraversal, object): self.write("(") endchar = ")" else: + # from trepan.api import debug; debug() raise TypeError( "Internal Error: n_build_list expects list, tuple, set, or unpack" ) @@ -2044,23 +2142,22 @@ class SourceWalker(GenericASTTraversal, object): index = entry[arg] if isinstance(index, tuple): if isinstance(index[1], str): + # if node[index[0]] != index[1]: + # from trepan.api import debug; debug() assert node[index[0]] == index[1], ( "at %s[%d], expected '%s' node; got '%s'" - % (node.kind, arg, index[1], node[index[0]].kind) + % (node.kind, arg, index[1], node[index[0]].kind,) ) else: assert node[index[0]] in index[1], ( "at %s[%d], expected to be in '%s' node; got '%s'" - % (node.kind, arg, index[1], node[index[0]].kind) + % (node.kind, arg, index[1], node[index[0]].kind,) ) index = index[0] - assert isinstance( - index, int - ), "at %s[%d], %s should be int or tuple" % ( - node.kind, - arg, - type(index), + assert isinstance(index, int), ( + "at %s[%d], %s should be int or tuple" + % (node.kind, arg, type(index),) ) try: @@ -2082,10 +2179,17 @@ class SourceWalker(GenericASTTraversal, object): assert isinstance(tup, tuple) if len(tup) == 3: (index, nonterm_name, self.prec) = tup - assert node[index] == nonterm_name, ( - "at %s[%d], expected '%s' node; got '%s'" - % (node.kind, arg, nonterm_name, node[index].kind) - ) + if isinstance(tup[1], str): + assert node[index] == nonterm_name, ( + "at %s[%d], expected '%s' node; got '%s'" + % (node.kind, arg, nonterm_name, node[index].kind,) + ) + else: + assert node[tup[0]] in tup[1], ( + "at %s[%d], expected to be in '%s' node; got '%s'" + % (node.kind, arg, index[1], node[index[0]].kind,) + ) + else: assert len(tup) == 2 (index, self.prec) = entry[arg] @@ -2416,10 +2520,10 @@ class SourceWalker(GenericASTTraversal, object): # print stmt[-1] - # Add "global" declaration statements at the top globals, nonlocals = find_globals_and_nonlocals( ast, set(), set(), code, self.version ) + # Add "global" declaration statements at the top # of the function for g in sorted(globals): self.println(indent, "global ", g) @@ -2458,11 +2562,8 @@ class SourceWalker(GenericASTTraversal, object): self.println(self.indent, "pass") else: self.customize(customize) - if is_lambda: - self.write(self.traverse(ast, is_lambda=is_lambda)) - else: - self.text = self.traverse(ast, is_lambda=is_lambda) - self.println(self.text) + self.text = self.traverse(ast, is_lambda=is_lambda) + self.println(self.text) self.name = old_name self.return_none = rn @@ -2473,7 +2574,7 @@ class SourceWalker(GenericASTTraversal, object): code, is_lambda=False, noneInNames=False, - isTopLevel=False, + is_top_level_module=False, ): # FIXME: DRY with fragments.py @@ -2500,10 +2601,10 @@ class SourceWalker(GenericASTTraversal, object): raise ParserError(e, tokens, self.p.debug["reduce"]) except AssertionError, e: raise ParserError(e, tokens, self.p.debug["reduce"]) - transform_ast = self.treeTransform.transform(ast, code) - self.maybe_show_tree(ast) + transform_tree = self.treeTransform.transform(ast, code) + self.maybe_show_tree(ast, phase="after") del ast # Save memory - return transform_ast + return transform_tree # The bytecode for the end of the main routine has a # "return None". However you can't issue a "return" statement in @@ -2515,7 +2616,7 @@ class SourceWalker(GenericASTTraversal, object): # Python 3.4's classes can add a "return None" which is # invalid syntax. if tokens[-2].kind == "LOAD_CONST": - if isTopLevel or tokens[-2].pattr is None: + if is_top_level_module or tokens[-2].pattr is None: del tokens[-2:] else: tokens.append(Token("RETURN_LAST")) @@ -2540,12 +2641,12 @@ class SourceWalker(GenericASTTraversal, object): checker(ast, False, self.ast_errors) self.customize(customize) - transform_ast = self.treeTransform.transform(ast, code) + transform_tree = self.treeTransform.transform(ast, code) - self.maybe_show_tree(ast) + self.maybe_show_tree(ast, phase="before") del ast # Save memory - return transform_ast + return transform_tree @classmethod def _get_mapping(cls, node): @@ -2573,16 +2674,13 @@ def code_deparse( version = PYTHON_VERSION_TRIPLE # store final output stream for case of error - scanner = get_scanner(version, is_pypy=is_pypy) + scanner = get_scanner(version, is_pypy=is_pypy, show_asm=debug_opts["asm"]) tokens, customize = scanner.ingest( co, code_objects=code_objects, show_asm=debug_opts["asm"] ) - debug_parser = dict(PARSER_DEFAULT_DEBUG) - if debug_opts.get("grammar", None): - debug_parser["reduce"] = debug_opts["grammar"] - debug_parser["errorstack"] = "full" + debug_parser = debug_opts.get("grammar", dict(PARSER_DEFAULT_DEBUG)) # Build Syntax Tree from disassembly. linestarts = dict(scanner.opc.findlinestarts(co)) @@ -2590,27 +2688,49 @@ def code_deparse( version, out, scanner, - showast=debug_opts.get("ast", None), + showast=debug_opts.get("tree", TREE_DEFAULT_DEBUG), debug_parser=debug_parser, compile_mode=compile_mode, is_pypy=is_pypy, linestarts=linestarts, ) - isTopLevel = co.co_name == "" + is_top_level_module = co.co_name == "" if compile_mode == "eval": deparsed.hide_internal = False - deparsed.ast = deparsed.build_ast(tokens, customize, co, isTopLevel=isTopLevel) + deparsed.compile_mode = compile_mode + deparsed.ast = deparsed.build_ast( + tokens, + customize, + co, + is_lambda=(compile_mode == "lambda"), + is_top_level_module=is_top_level_module, + ) #### XXX workaround for profiling if deparsed.ast is None: return None - if compile_mode != "eval": - assert deparsed.ast == "stmts", "Should have parsed grammar start" + # FIXME use a lookup table here. + if compile_mode == "lambda": + expected_start = "lambda_start" + elif compile_mode == "eval": + expected_start = "expr_start" + elif compile_mode == "expr": + expected_start = "expr_start" + elif compile_mode == "exec": + expected_start = "stmts" + elif compile_mode == "single": + expected_start = "single_start" else: - assert deparsed.ast == "eval_expr", "Should have parsed grammar start" - + expected_start = None + if expected_start: + assert ( + deparsed.ast == expected_start + ), ( + "Should have parsed grammar start to '%s'; got: %s" % + (expected_start, deparsed.ast.kind) + ) # save memory del tokens @@ -2652,7 +2772,11 @@ def code_deparse( # What we've been waiting for: Generate source from Syntax Tree! deparsed.gen_source( - deparsed.ast, name=co.co_name, customize=customize, debug_opts=debug_opts + deparsed.ast, + name=co.co_name, + customize=customize, + is_lambda=compile_mode == "lambda", + debug_opts=debug_opts, ) for g in sorted(deparsed.mod_globs): @@ -2660,7 +2784,7 @@ def code_deparse( if deparsed.ast_errors: deparsed.write("# NOTE: have internal decompilation grammar errors.\n") - deparsed.write("# Use -t option to show full context.") + deparsed.write("# Use -T option to show full context.") for err in deparsed.ast_errors: deparsed.write(err) raise SourceWalkerError("Deparsing hit an internal grammar-rule bug") diff --git a/uncompyle6/show.py b/uncompyle6/show.py index bf5e4032..8115ca00 100644 --- a/uncompyle6/show.py +++ b/uncompyle6/show.py @@ -52,7 +52,7 @@ def maybe_show_tree(walker, ast): stream = sys.stdout if ( isinstance(walker.showast, dict) - and walker.showast.get("Full", False) + and walker.showast.get("after", False) and hasattr(walker, "str_with_template") ): walker.str_with_template(ast) diff --git a/uncompyle6/version.py b/uncompyle6/version.py index fed5a211..44c88a7d 100644 --- a/uncompyle6/version.py +++ b/uncompyle6/version.py @@ -14,4 +14,4 @@ # This file is suitable for sourcing inside POSIX shell as # well as importing into Python # fmt: off -__version__="3.8.1.dev0" # noqa +__version__="3.9.0a1" # noqa