From f7436a4ff21aa7a292eff1eac130294d63c1ef69 Mon Sep 17 00:00:00 2001 From: rocky Date: Mon, 5 Feb 2024 16:17:48 -0500 Subject: [PATCH 01/10] More lint --- uncompyle6/scanner.py | 5 +++++ uncompyle6/semantics/pysource.py | 20 ++++++++++---------- 2 files changed, 15 insertions(+), 10 deletions(-) diff --git a/uncompyle6/scanner.py b/uncompyle6/scanner.py index 912c4a0f..7384f131 100644 --- a/uncompyle6/scanner.py +++ b/uncompyle6/scanner.py @@ -96,6 +96,11 @@ class Code(object): """ def __init__(self, co, scanner, classname=None, show_asm=None): + + # Full initialization is given below, but for linters + # well set up some initial values. + self.co_code = None # Really either bytes for >= 3.0 and string in < 3.0 + for i in dir(co): if i.startswith("co_"): setattr(self, i, getattr(co, i)) diff --git a/uncompyle6/semantics/pysource.py b/uncompyle6/semantics/pysource.py index eef9591c..3b62481c 100644 --- a/uncompyle6/semantics/pysource.py +++ b/uncompyle6/semantics/pysource.py @@ -130,6 +130,7 @@ Python. # evaluating the escape code. import sys +from io import StringIO from spark_parser import GenericASTTraversal from xdis import COMPILER_FLAG_BIT, iscode @@ -174,8 +175,6 @@ def unicode(x): return x -from io import StringIO - PARSER_DEFAULT_DEBUG = { "rules": False, "transition": False, @@ -384,9 +383,9 @@ class SourceWalker(GenericASTTraversal, NonterminalActions, ComprehensionMixin): i += 1 return rv - def indent_if_source_nl(self, line_number: int, indent: int): + def indent_if_source_nl(self, line_number: int, indent_spaces: str): if line_number != self.line_number: - self.write("\n" + indent + INDENT_PER_LEVEL[:-1]) + self.write("\n" + indent_spaces + INDENT_PER_LEVEL[:-1]) return self.line_number f = property( @@ -564,6 +563,7 @@ class SourceWalker(GenericASTTraversal, NonterminalActions, ComprehensionMixin): def print_super_classes3(self, node): n = len(node) - 1 + j = 0 if node.kind != "expr": if node == "kwarg": self.template_engine(("(%[0]{attr}=%c)", 1), node) @@ -601,9 +601,9 @@ class SourceWalker(GenericASTTraversal, NonterminalActions, ComprehensionMixin): self.write("(") if kwargs: # Last arg is tuple of keyword values: omit - l = n - 1 + m = n - 1 else: - l = n + m = n if kwargs: # 3.6+ does this @@ -615,7 +615,7 @@ class SourceWalker(GenericASTTraversal, NonterminalActions, ComprehensionMixin): j += 1 j = 0 - while i < l: + while i < m: self.write(sep) value = self.traverse(node[i]) self.write("%s=%s" % (kwargs[j], value)) @@ -623,7 +623,7 @@ class SourceWalker(GenericASTTraversal, NonterminalActions, ComprehensionMixin): j += 1 i += 1 else: - while i < l: + while i < m: value = self.traverse(node[i]) i += 1 self.write(sep, value) @@ -1093,8 +1093,8 @@ class SourceWalker(GenericASTTraversal, NonterminalActions, ComprehensionMixin): # if docstring exists, dump it if code.co_consts and code.co_consts[0] is not None and len(ast) > 0: do_doc = False + i = 0 if is_docstring(ast[0], self.version, code.co_consts): - i = 0 do_doc = True elif len(ast) > 1 and is_docstring(ast[1], self.version, code.co_consts): i = 1 @@ -1427,7 +1427,6 @@ def deparse_code2str( if __name__ == "__main__": - def deparse_test(co): """This is a docstring""" s = deparse_code2str(co) @@ -1435,4 +1434,5 @@ if __name__ == "__main__": print(s) return + deparse_test(deparse_test.__code__) From 86e22bbacbeb57f666079210f4e1d7c84a937b13 Mon Sep 17 00:00:00 2001 From: rocky Date: Mon, 5 Feb 2024 16:20:15 -0500 Subject: [PATCH 02/10] One more --- uncompyle6/bin/pydisassemble.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/uncompyle6/bin/pydisassemble.py b/uncompyle6/bin/pydisassemble.py index be1cb152..29481ad1 100755 --- a/uncompyle6/bin/pydisassemble.py +++ b/uncompyle6/bin/pydisassemble.py @@ -51,7 +51,7 @@ PATTERNS = ("*.pyc", "*.pyo") def main(): - Usage_short = ( + usage_short = ( """usage: %s FILE... Type -h for for full help.""" % program @@ -59,7 +59,7 @@ Type -h for for full help.""" if len(sys.argv) == 1: print("No file(s) given", file=sys.stderr) - print(Usage_short, file=sys.stderr) + print(usage_short, file=sys.stderr) sys.exit(1) try: @@ -79,7 +79,7 @@ Type -h for for full help.""" sys.exit(0) else: print(opt) - print(Usage_short, file=sys.stderr) + print(usage_short, file=sys.stderr) sys.exit(1) for file in files: From 33bc80bb2414151b4c4dc0ee7bfbd6b615b9870b Mon Sep 17 00:00:00 2001 From: rocky Date: Mon, 5 Feb 2024 16:26:59 -0500 Subject: [PATCH 03/10] f-string convert a file --- uncompyle6/bin/pydisassemble.py | 27 +++++++++++++++++++-------- 1 file changed, 19 insertions(+), 8 deletions(-) diff --git a/uncompyle6/bin/pydisassemble.py b/uncompyle6/bin/pydisassemble.py index 29481ad1..e40e0dfc 100755 --- a/uncompyle6/bin/pydisassemble.py +++ b/uncompyle6/bin/pydisassemble.py @@ -1,9 +1,21 @@ #!/usr/bin/env python -# Mode: -*- python -*- # -# Copyright (c) 2015-2016, 2018, 2020, 2022-2023 by Rocky Bernstein +# Copyright (c) 2015-2016, 2018, 2020, 2022-2024 +# by Rocky Bernstein +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . # -from __future__ import print_function import getopt import os @@ -52,9 +64,8 @@ PATTERNS = ("*.pyc", "*.pyo") def main(): usage_short = ( - """usage: %s FILE... + f"""usage: {program} FILE... Type -h for for full help.""" - % program ) if len(sys.argv) == 1: @@ -67,7 +78,7 @@ Type -h for for full help.""" sys.argv[1:], "hVU", ["help", "version", "uncompyle6"] ) except getopt.GetoptError as e: - print("%s: %s" % (os.path.basename(sys.argv[0]), e), file=sys.stderr) + print(f"{os.path.basename(sys.argv[0])}: {e}", file=sys.stderr) sys.exit(-1) for opt, val in opts: @@ -75,7 +86,7 @@ Type -h for for full help.""" print(__doc__) sys.exit(1) elif opt in ("-V", "--version"): - print("%s %s" % (program, __version__)) + print(f"{program} {__version__}") sys.exit(0) else: print(opt) @@ -86,7 +97,7 @@ Type -h for for full help.""" if os.path.exists(files[0]): disassemble_file(file, sys.stdout) else: - print("Can't read %s - skipping" % files[0], file=sys.stderr) + print(f"Can't read {files[0]} - skipping", file=sys.stderr) pass pass return From f605f859aee4e76573c4ba309cb0752a74f308d5 Mon Sep 17 00:00:00 2001 From: rocky Date: Mon, 5 Feb 2024 16:57:59 -0500 Subject: [PATCH 04/10] Partial sync with decompyle3 --- uncompyle6/semantics/pysource.py | 107 ++++++++++++++++++++----------- 1 file changed, 70 insertions(+), 37 deletions(-) diff --git a/uncompyle6/semantics/pysource.py b/uncompyle6/semantics/pysource.py index 3b62481c..71459685 100644 --- a/uncompyle6/semantics/pysource.py +++ b/uncompyle6/semantics/pysource.py @@ -131,6 +131,7 @@ Python. import sys from io import StringIO +from typing import Optional from spark_parser import GenericASTTraversal from xdis import COMPILER_FLAG_BIT, iscode @@ -159,7 +160,11 @@ from uncompyle6.semantics.consts import ( ) from uncompyle6.semantics.customize import customize_for_version from uncompyle6.semantics.gencomp import ComprehensionMixin -from uncompyle6.semantics.helper import find_globals_and_nonlocals, print_docstring +from uncompyle6.semantics.helper import ( + find_globals_and_nonlocals, + is_lambda_mode, + print_docstring, +) from uncompyle6.semantics.make_function1 import make_function1 from uncompyle6.semantics.make_function2 import make_function2 from uncompyle6.semantics.make_function3 import make_function3 @@ -213,7 +218,7 @@ class SourceWalker(GenericASTTraversal, NonterminalActions, ComprehensionMixin): def __init__( self, - version, + version: tuple, out, scanner, showast=TREE_DEFAULT_DEBUG, @@ -223,7 +228,7 @@ class SourceWalker(GenericASTTraversal, NonterminalActions, ComprehensionMixin): linestarts={}, tolerate_errors=False, ): - """`version' is the Python version (a float) of the Python dialect + """`version' is the Python version of the Python dialect of both the syntax tree and language we should produce. `out' is IO-like file pointer to where the output should go. It @@ -235,9 +240,12 @@ class SourceWalker(GenericASTTraversal, NonterminalActions, ComprehensionMixin): If `showast' is True, we print the syntax tree. - `compile_mode' is is either 'exec' or 'single'. It is the compile - mode that was used to create the Syntax Tree and specifies a - grammar variant within a Python version to use. + `compile_mode` is is either `exec`, `single` or `lambda`. + + For `lambda`, the grammar that can be used in lambda + expressions is used. Otherwise, it is the compile mode that + was used to create the Syntax Tree and specifies a grammar + variant within a Python version to use. `is_pypy` should be True if the Syntax Tree was generated for PyPy. @@ -262,10 +270,8 @@ class SourceWalker(GenericASTTraversal, NonterminalActions, ComprehensionMixin): self.currentclass = None self.classes = [] self.debug_parser = dict(debug_parser) - # Initialize p_lambda on demand self.line_number = 1 self.linemap = {} - self.p_lambda = None self.params = params self.param_stack = [] self.ERROR = None @@ -276,11 +282,15 @@ class SourceWalker(GenericASTTraversal, NonterminalActions, ComprehensionMixin): self.pending_newlines = 0 self.linestarts = linestarts self.treeTransform = TreeTransform(version=self.version, show_ast=showast) + # FIXME: have p.insts update in a better way # modularity is broken here self.insts = scanner.insts self.offset2inst_index = scanner.offset2inst_index + # Initialize p_lambda on demand + self.p_lambda = None + # This is in Python 2.6 on. It changes the way # strings get interpreted. See n_LOAD_CONST self.FUTURE_UNICODE_LITERALS = False @@ -507,19 +517,19 @@ class SourceWalker(GenericASTTraversal, NonterminalActions, ComprehensionMixin): def pp_tuple(self, tup): """Pretty print a tuple""" last_line = self.f.getvalue().split("\n")[-1] - l = len(last_line) + 1 - indent = " " * l + ll = len(last_line) + 1 + indent = " " * ll self.write("(") sep = "" for item in tup: self.write(sep) - l += len(sep) + ll += len(sep) s = better_repr(item, self.version) - l += len(s) + ll += len(s) self.write(s) sep = "," - if l > LINE_LENGTH: - l = 0 + if ll > LINE_LENGTH: + ll = 0 sep += "\n" + indent else: sep += " " @@ -699,9 +709,10 @@ class SourceWalker(GenericASTTraversal, NonterminalActions, ComprehensionMixin): """ # print("-----") - # print(startnode) + # print(startnode.kind) # print(entry[0]) # print('======') + fmt = entry[0] arg = 1 i = 0 @@ -794,13 +805,9 @@ class SourceWalker(GenericASTTraversal, NonterminalActions, ComprehensionMixin): node[index].kind, ) else: - assert ( - node[tup[0]] in tup[1] - ), "at %s[%d], expected to be in '%s' node; got '%s'" % ( - node.kind, - arg, - index[1], - node[index[0]].kind, + assert node[tup[0]] in tup[1], ( + f"at {node.kind}[{tup[0]}], expected to be in '{tup[1]}' " + f"node; got '{node[tup[0]].kind}'" ) else: @@ -869,7 +876,7 @@ class SourceWalker(GenericASTTraversal, NonterminalActions, ComprehensionMixin): d = node.__dict__ try: self.write(eval(expr, d, d)) - except: + except Exception: raise m = escape.search(fmt, i) self.write(fmt[i:]) @@ -1190,7 +1197,7 @@ class SourceWalker(GenericASTTraversal, NonterminalActions, ComprehensionMixin): is_lambda=False, noneInNames=False, is_top_level_module=False, - ): + ) -> GenericASTTraversal: # FIXME: DRY with fragments.py # assert isinstance(tokens[0], Token) @@ -1242,7 +1249,7 @@ class SourceWalker(GenericASTTraversal, NonterminalActions, ComprehensionMixin): # Build a parse tree from a tokenized and massaged disassembly. try: # FIXME: have p.insts update in a better way - # modularity is broken here + # Modularity is broken here. p_insts = self.p.insts self.p.insts = self.scanner.insts self.p.offset2inst_index = self.scanner.offset2inst_index @@ -1255,6 +1262,7 @@ class SourceWalker(GenericASTTraversal, NonterminalActions, ComprehensionMixin): checker(ast, False, self.ast_errors) self.customize(customize) + transform_tree = self.treeTransform.transform(ast, code) self.maybe_show_tree(ast, phase="before") @@ -1270,13 +1278,15 @@ class SourceWalker(GenericASTTraversal, NonterminalActions, ComprehensionMixin): def code_deparse( co, out=sys.stdout, - version=None, + version: Optional[tuple] = None, debug_opts=DEFAULT_DEBUG_OPTS, code_objects={}, compile_mode="exec", is_pypy=IS_PYPY, walker=SourceWalker, -): + start_offset: int = 0, + stop_offset: int = -1, +) -> Optional[SourceWalker]: """ ingests and deparses a given code block 'co'. If version is None, we will use the current Python interpreter version. @@ -1284,6 +1294,9 @@ def code_deparse( assert iscode(co) + if out is None: + out = sys.stdout + if version is None: version = PYTHON_VERSION_TRIPLE @@ -1294,6 +1307,21 @@ def code_deparse( co, code_objects=code_objects, show_asm=debug_opts["asm"] ) + if start_offset > 0: + for i, t in enumerate(tokens): + # If t.offset is a string, we want to skip this. + if isinstance(t.offset, int) and t.offset >= start_offset: + tokens = tokens[i:] + break + + if stop_offset > -1: + for i, t in enumerate(tokens): + # In contrast to the test for start_offset If t.offset is + # a string, we want to extract the integer offset value. + if t.off2int() >= stop_offset: + tokens = tokens[:i] + break + debug_parser = debug_opts.get("grammar", dict(PARSER_DEFAULT_DEBUG)) # Build Syntax Tree from disassembly. @@ -1317,7 +1345,7 @@ def code_deparse( tokens, customize, co, - is_lambda=(compile_mode == "lambda"), + is_lambda=is_lambda_mode(compile_mode), is_top_level_module=is_top_level_module, ) @@ -1326,7 +1354,7 @@ def code_deparse( return None # FIXME use a lookup table here. - if compile_mode == "lambda": + if is_lambda_mode(compile_mode): expected_start = "lambda_start" elif compile_mode == "eval": expected_start = "expr_start" @@ -1339,10 +1367,12 @@ def code_deparse( expected_start = None else: expected_start = None + if expected_start: - assert ( - deparsed.ast == expected_start - ), f"Should have parsed grammar start to '{expected_start}'; got: {deparsed.ast.kind}" + assert deparsed.ast == expected_start, ( + f"Should have parsed grammar start to '{expected_start}'; " + f"got: {deparsed.ast.kind}" + ) # save memory del tokens @@ -1382,7 +1412,7 @@ def code_deparse( deparsed.ast, name=co.co_name, customize=customize, - is_lambda=compile_mode == "lambda", + is_lambda=is_lambda_mode(compile_mode), debug_opts=debug_opts, ) @@ -1410,9 +1440,12 @@ def deparse_code2str( compile_mode="exec", is_pypy=IS_PYPY, walker=SourceWalker, -): - """Return the deparsed text for a Python code object. `out` is where any intermediate - output for assembly or tree output will be sent. + start_offset: int = 0, + stop_offset: int = -1, +) -> str: + """ + Return the deparsed text for a Python code object. `out` is where + any intermediate output for assembly or tree output will be sent. """ return code_deparse( code, @@ -1427,6 +1460,7 @@ def deparse_code2str( if __name__ == "__main__": + def deparse_test(co): """This is a docstring""" s = deparse_code2str(co) @@ -1434,5 +1468,4 @@ if __name__ == "__main__": print(s) return - deparse_test(deparse_test.__code__) From 61105840afac785ae4750119fae56182095819d3 Mon Sep 17 00:00:00 2001 From: rocky Date: Mon, 5 Feb 2024 17:06:47 -0500 Subject: [PATCH 05/10] Sync with decompyle3 --- uncompyle6/scanner.py | 26 ++++++++++++-------------- 1 file changed, 12 insertions(+), 14 deletions(-) diff --git a/uncompyle6/scanner.py b/uncompyle6/scanner.py index 7384f131..ee3e73bd 100644 --- a/uncompyle6/scanner.py +++ b/uncompyle6/scanner.py @@ -1,4 +1,4 @@ -# Copyright (c) 2016, 2018-2023 by Rocky Bernstein +# Copyright (c) 2016, 2018-2024 by Rocky Bernstein # Copyright (c) 2005 by Dan Pascu # Copyright (c) 2000-2002 by hartmut Goebel # Copyright (c) 1999 John Aycock @@ -21,13 +21,11 @@ scanner/ingestion module. From here we call various version-specific scanners, e.g. for Python 2.7 or 3.4. """ -from types import ModuleType -from typing import Optional, Tuple, Union from array import array from collections import namedtuple +from types import ModuleType +from typing import Optional, Tuple, Union -from uncompyle6.scanners.tok import Token -from xdis.version_info import IS_PYPY, version_tuple_to_str import xdis from xdis import ( Bytecode, @@ -37,6 +35,9 @@ from xdis import ( instruction_size, next_offset, ) +from xdis.version_info import IS_PYPY, version_tuple_to_str + +from uncompyle6.scanners.tok import Token # The byte code versions we support. # Note: these all have to be tuples of 2 ints @@ -80,6 +81,7 @@ CANONIC2VERSION["3.5.2"] = 3.5 # FIXME: DRY L65536 = 65536 + def long(num): return num @@ -96,7 +98,6 @@ class Code(object): """ def __init__(self, co, scanner, classname=None, show_asm=None): - # Full initialization is given below, but for linters # well set up some initial values. self.co_code = None # Really either bytes for >= 3.0 and string in < 3.0 @@ -133,9 +134,7 @@ class Scanner: # FIXME: This weird Python2 behavior is not Python3 self.resetTokenClass() - def bound_collection_from_tokens( - self, tokens, t, i, collection_type - ): + def bound_collection_from_tokens(self, tokens, t, i, collection_type): count = t.attr assert isinstance(count, int) @@ -429,7 +428,7 @@ class Scanner: """ try: None in instr - except: + except Exception: instr = [instr] first = self.offset2inst_index[start] @@ -620,16 +619,14 @@ def parse_fn_counts_30_35(argc: int) -> Tuple[int, int, int]: def get_scanner(version: Union[str, tuple], is_pypy=False, show_asm=None) -> Scanner: - # If version is a string, turn that into the corresponding float. if isinstance(version, str): if version not in canonic_python_version: - raise RuntimeError("Unknown Python version in xdis %s" % version) + raise RuntimeError(f"Unknown Python version in xdis {version}") canonic_version = canonic_python_version[version] if canonic_version not in CANONIC2VERSION: raise RuntimeError( - "Unsupported Python version %s (canonic %s)" - % (version, canonic_version) + f"Unsupported Python version {version} (canonic {canonic_version})" ) version = CANONIC2VERSION[canonic_version] @@ -680,5 +677,6 @@ if __name__ == "__main__": # scanner = get_scanner('2.7.13', True) # scanner = get_scanner(sys.version[:5], False) from xdis.version_info import PYTHON_VERSION_TRIPLE + scanner = get_scanner(PYTHON_VERSION_TRIPLE, IS_PYPY, True) tokens, customize = scanner.ingest(co, {}, show_asm="after") From dd8ee1466d5871a2c11590b1bc57e0ac695b5f52 Mon Sep 17 00:00:00 2001 From: rocky Date: Sat, 10 Feb 2024 13:19:32 -0500 Subject: [PATCH 06/10] Redo uncompyel6 options ... Use click now and make more like decompyle3 --- __pkginfo__.py | 19 +- uncompyle6/bin/uncompile.py | 307 +++++++++++++++++------------- uncompyle6/main.py | 112 ++++++++--- uncompyle6/semantics/fragments.py | 18 ++ 4 files changed, 284 insertions(+), 172 deletions(-) diff --git a/__pkginfo__.py b/__pkginfo__.py index 723c15a3..98fa63d5 100644 --- a/__pkginfo__.py +++ b/__pkginfo__.py @@ -1,4 +1,4 @@ -# Copyright (C) 2018, 2020-2021 Rocky Bernstein +# Copyright (C) 2018, 2020-2021 2024 Rocky Bernstein # # This program is free software: you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by @@ -32,9 +32,11 @@ # 3.3 | pip | 10.0.1 | # 3.4 | pip | 19.1.1 | +import os.path as osp + # Things that change more often go here. copyright = """ -Copyright (C) 2015-2021 Rocky Bernstein . +Copyright (C) 2015-2021, 2024 Rocky Bernstein . """ classifiers = [ @@ -75,7 +77,7 @@ entry_points = { ] } ftp_url = None -install_requires = ["spark-parser >= 1.8.9, < 1.9.0", "xdis >= 6.0.8, < 6.2.0"] +install_requires = ["click", "spark-parser >= 1.8.9, < 1.9.0", "xdis >= 6.0.8, < 6.2.0"] license = "GPL3" mailing_list = "python-debugger@googlegroups.com" @@ -88,21 +90,18 @@ web = "https://github.com/rocky/python-uncompyle6/" zip_safe = True -import os.path - - def get_srcdir(): - filename = os.path.normcase(os.path.dirname(os.path.abspath(__file__))) - return os.path.realpath(filename) + filename = osp.normcase(osp.dirname(osp.abspath(__file__))) + return osp.realpath(filename) srcdir = get_srcdir() def read(*rnames): - return open(os.path.join(srcdir, *rnames)).read() + return open(osp.join(srcdir, *rnames)).read() -# Get info from files; set: long_description and __version__ +# Get info from files; set: long_description and VERSION long_description = read("README.rst") + "\n" exec(read("uncompyle6/version.py")) diff --git a/uncompyle6/bin/uncompile.py b/uncompyle6/bin/uncompile.py index fc7e7556..6d3a347a 100755 --- a/uncompyle6/bin/uncompile.py +++ b/uncompyle6/bin/uncompile.py @@ -1,15 +1,19 @@ #!/usr/bin/env python # Mode: -*- python -*- # -# Copyright (c) 2015-2017, 2019-2020, 2023 by Rocky Bernstein +# Copyright (c) 2015-2017, 2019-2020, 2023-2024 +# by Rocky Bernstein # Copyright (c) 2000-2002 by hartmut Goebel # from __future__ import print_function -import getopt import os import sys import time +from typing import List + +import click +from xdis.version_info import version_tuple_to_str from uncompyle6 import verify from uncompyle6.main import main, status_msg @@ -17,150 +21,162 @@ from uncompyle6.version import __version__ program = "uncompyle6" -__doc__ = """ -Usage: - %s [OPTIONS]... [ FILE | DIR]... - %s [--help | -h | --V | --version] - -Examples: - %s foo.pyc bar.pyc # decompile foo.pyc, bar.pyc to stdout - %s -o . foo.pyc bar.pyc # decompile to ./foo.pyc_dis and ./bar.pyc_dis - %s -o /tmp /usr/lib/python1.5 # decompile whole library - -Options: - -o output decompiled files to this path: - if multiple input files are decompiled, the common prefix - is stripped from these names and the remainder appended to - - uncompyle6 -o /tmp bla/fasel.pyc bla/foo.pyc - -> /tmp/fasel.pyc_dis, /tmp/foo.pyc_dis - uncompyle6 -o /tmp bla/fasel.pyc bar/foo.pyc - -> /tmp/bla/fasel.pyc_dis, /tmp/bar/foo.pyc_dis - uncompyle6 -o /tmp /usr/lib/python1.5 - -> /tmp/smtplib.pyc_dis ... /tmp/lib-tk/FixTk.pyc_dis - --compile | -c - attempts a decompilation after compiling - -d print timestamps - -p use number of processes - -r recurse directories looking for .pyc and .pyo files - --fragments use fragments deparser - --verify compare generated source with input byte-code - --verify-run compile generated source, run it and check exit code - --syntax-verify compile generated source - --linemaps generated line number correspondencies between byte-code - and generated source output - --encoding - use in generated source according to pep-0263 - --help show this message - -Debugging Options: - --asm | -a include byte-code (disables --verify) - --grammar | -g show matching grammar - --tree={before|after} - -t {before|after} include syntax before (or after) tree transformation - (disables --verify) - --tree++ | -T add template rules to --tree=before when possible - -Extensions of generated files: - '.pyc_dis' '.pyo_dis' successfully decompiled (and verified if --verify) - + '_unverified' successfully decompile but --verify failed - + '_failed' decompile failed (contact author for enhancement) -""" % ( - (program,) * 5 -) - -program = "uncompyle6" - def usage(): print(__doc__) sys.exit(1) -def main_bin(): - recurse_dirs = False - numproc = 0 - outfile = "-" - out_base = None - source_paths = [] - timestamp = False - timestampfmt = "# %Y.%m.%d %H:%M:%S %Z" +# __doc__ = """ +# Usage: +# %s [OPTIONS]... [ FILE | DIR]... +# %s [--help | -h | --V | --version] - try: - opts, pyc_paths = getopt.getopt( - sys.argv[1:], - "hac:gtTdrVo:p:", - "help asm compile= grammar linemaps recurse " - "timestamp tree= tree+ " - "fragments verify verify-run version " - "syntax-verify " - "showgrammar encoding=".split(" "), +# Examples: +# %s foo.pyc bar.pyc # decompile foo.pyc, bar.pyc to stdout +# %s -o . foo.pyc bar.pyc # decompile to ./foo.pyc_dis and ./bar.pyc_dis +# %s -o /tmp /usr/lib/python1.5 # decompile whole library + +# Options: +# -o output decompiled files to this path: +# if multiple input files are decompiled, the common prefix +# is stripped from these names and the remainder appended to +# +# uncompyle6 -o /tmp bla/fasel.pyc bla/foo.pyc +# -> /tmp/fasel.pyc_dis, /tmp/foo.pyc_dis +# uncompyle6 -o /tmp bla/fasel.pyc bar/foo.pyc +# -> /tmp/bla/fasel.pyc_dis, /tmp/bar/foo.pyc_dis +# uncompyle6 -o /tmp /usr/lib/python1.5 +# -> /tmp/smtplib.pyc_dis ... /tmp/lib-tk/FixTk.pyc_dis +# --compile | -c +# attempts a decompilation after compiling +# -d print timestamps +# -p use number of processes +# -r recurse directories looking for .pyc and .pyo files +# --fragments use fragments deparser +# --verify compare generated source with input byte-code +# --verify-run compile generated source, run it and check exit code +# --syntax-verify compile generated source +# --linemaps generated line number correspondencies between byte-code +# and generated source output +# --encoding +# use in generated source according to pep-0263 +# --help show this message + +# Debugging Options: +# --asm | -a include byte-code (disables --verify) +# --grammar | -g show matching grammar +# --tree={before|after} +# -t {before|after} include syntax before (or after) tree transformation +# (disables --verify) +# --tree++ | -T add template rules to --tree=before when possible + +# Extensions of generated files: +# '.pyc_dis' '.pyo_dis' successfully decompiled (and verified if --verify) +# + '_unverified' successfully decompile but --verify failed +# + '_failed' decompile failed (contact author for enhancement) +# """ % ( +# (program,) * 5 +# ) + + +@click.command() +@click.option( + "--asm++/--no-asm++", + "-A", + "asm_plus", + default=False, + help="show xdis assembler and tokenized assembler", +) +@click.option("--asm/--no-asm", "-a", default=False) +@click.option("--grammar/--no-grammar", "-g", "show_grammar", default=False) +@click.option("--tree/--no-tree", "-t", default=False) +@click.option( + "--tree++/--no-tree++", + "-T", + "tree_plus", + default=False, + help="show parse tree and Abstract Syntax Tree", +) +@click.option( + "--linemaps/--no-linemaps", + default=False, + help="show line number correspondencies between byte-code " + "and generated source output", +) +@click.option( + "--verify", + type=click.Choice(["run", "syntax"]), + default=None, +) +@click.option( + "--recurse/--no-recurse", + "-r", + "recurse_dirs", + default=False, +) +@click.option( + "--output", + "-o", + "outfile", + type=click.Path( + exists=True, file_okay=True, dir_okay=True, writable=True, resolve_path=True + ), + required=False, +) +@click.version_option(version=__version__) +@click.option( + "--start-offset", + "start_offset", + default=0, + help="start decomplation at offset; default is 0 or the starting offset.", +) +@click.version_option(version=__version__) +@click.option( + "--stop-offset", + "stop_offset", + default=-1, + help="stop decomplation when seeing an offset greater or equal to this; default is " + "-1 which indicates no stopping point.", +) +@click.argument("files", nargs=-1, type=click.Path(readable=True), required=True) +def main_bin( + asm: bool, + asm_plus: bool, + show_grammar, + tree: bool, + tree_plus: bool, + linemaps: bool, + verify, + recurse_dirs: bool, + outfile, + start_offset: int, + stop_offset: int, + files, +): + """ + Cross Python bytecode decompiler for Python bytecode up to Python 3.8. + """ + + version_tuple = sys.version_info[0:2] + if version_tuple < (3, 7): + print( + f"Error: This version of the {program} runs from Python 3.7 or greater." + f"You need another branch of this code for Python before 3.7." + f""" \n\tYou have version: {version_tuple_to_str()}.""" ) - except getopt.GetoptError as e: - print("%s: %s" % (os.path.basename(sys.argv[0]), e), file=sys.stderr) sys.exit(-1) - options = { - "showasm": None - } - for opt, val in opts: - if opt in ("-h", "--help"): - print(__doc__) - sys.exit(0) - elif opt in ("-V", "--version"): - print("%s %s" % (program, __version__)) - sys.exit(0) - elif opt == "--verify": - options["do_verify"] = "strong" - elif opt == "--syntax-verify": - options["do_verify"] = "weak" - elif opt == "--fragments": - options["do_fragments"] = True - elif opt == "--verify-run": - options["do_verify"] = "verify-run" - elif opt == "--linemaps": - options["do_linemaps"] = True - elif opt in ("--asm", "-a"): - if options["showasm"] == None: - options["showasm"] = "after" - else: - options["showasm"] = "both" - options["do_verify"] = None - elif opt in ("--tree", "-t"): - if "showast" not in options: - options["showast"] = {} - if val == "before": - options["showast"][val] = True - elif val == "after": - options["showast"][val] = True - else: - options["showast"]["before"] = True - options["do_verify"] = None - elif opt in ("--tree+", "-T"): - if "showast" not in options: - options["showast"] = {} - options["showast"]["after"] = True - options["showast"]["before"] = True - options["do_verify"] = None - elif opt in ("--grammar", "-g"): - options["showgrammar"] = True - elif opt == "-o": - outfile = val - elif opt in ("--timestamp", "-d"): - timestamp = True - elif opt in ("--compile", "-c"): - source_paths.append(val) - elif opt == "-p": - numproc = int(val) - elif opt in ("--recurse", "-r"): - recurse_dirs = True - elif opt == "--encoding": - options["source_encoding"] = val - else: - print(opt, file=sys.stderr) - usage() + numproc = 0 + out_base = None - # expand directory if specified + out_base = None + source_paths: List[str] = [] + timestamp = False + timestampfmt = "# %Y.%m.%d %H:%M:%S %Z" + pyc_paths = files + + # Expand directory if "recurse" was specified. if recurse_dirs: expanded_files = [] for f in pyc_paths: @@ -194,15 +210,32 @@ def main_bin(): out_base = outfile outfile = None + # A second -a turns show_asm="after" into show_asm="before" + if asm_plus or asm: + asm_opt = "both" if asm_plus else "after" + else: + asm_opt = None + if timestamp: print(time.strftime(timestampfmt)) if numproc <= 1: + show_ast = {"before": tree or tree_plus, "after": tree_plus} try: result = main( - src_base, out_base, pyc_paths, source_paths, outfile, **options + src_base, + out_base, + pyc_paths, + source_paths, + outfile, + showasm=asm_opt, + showgrammar=show_grammar, + showast=show_ast, + do_verify=verify, + do_linemaps=linemaps, + start_offset=start_offset, + stop_offset=stop_offset, ) - result = [options.get("do_verify", None)] + list(result) if len(pyc_paths) > 1: mess = status_msg(*result) print("# " + mess) diff --git a/uncompyle6/main.py b/uncompyle6/main.py index 671f0b2b..5e759190 100644 --- a/uncompyle6/main.py +++ b/uncompyle6/main.py @@ -15,9 +15,11 @@ import datetime import os +import os.path as osp import py_compile import sys -from typing import Any, Optional, Tuple +import tempfile +from typing import Any, Optional, TextIO, Tuple from xdis import iscode from xdis.load import load_module @@ -38,9 +40,9 @@ def _get_outstream(outfile: str) -> Any: """ Return an opened output file descriptor for ``outfile``. """ - dir_name = os.path.dirname(outfile) + dir_name = osp.dirname(outfile) failed_file = outfile + "_failed" - if os.path.exists(failed_file): + if osp.exists(failed_file): os.remove(failed_file) try: os.makedirs(dir_name) @@ -52,7 +54,7 @@ def _get_outstream(outfile: str) -> Any: def decompile( co, bytecode_version: Tuple[int] = PYTHON_VERSION_TRIPLE, - out=sys.stdout, + out: Optional[TextIO] = sys.stdout, showasm: Optional[str] = None, showast={}, timestamp=None, @@ -60,11 +62,13 @@ def decompile( source_encoding=None, code_objects={}, source_size=None, - is_pypy=False, + is_pypy: bool = False, magic_int=None, mapstream=None, do_fragments=False, compile_mode="exec", + start_offset: int = 0, + stop_offset: int = -1, ) -> Any: """ ingests and deparses a given code block 'co' @@ -132,11 +136,12 @@ def decompile( debug_opts=debug_opts, ) header_count = 3 + len(sys_version_lines) - linemap = [ - (line_no, deparsed.source_linemap[line_no] + header_count) - for line_no in sorted(deparsed.source_linemap.keys()) - ] - mapstream.write(f"\n\n# {linemap}\n") + if deparsed is not None: + linemap = [ + (line_no, deparsed.source_linemap[line_no] + header_count) + for line_no in sorted(deparsed.source_linemap.keys()) + ] + mapstream.write(f"\n\n# {linemap}\n") else: if do_fragments: deparse_fn = code_deparse_fragments @@ -149,8 +154,11 @@ def decompile( is_pypy=is_pypy, debug_opts=debug_opts, compile_mode=compile_mode, + start_offset=start_offset, + stop_offset=stop_offset, ) pass + real_out.write("\n") return deparsed except pysource.SourceWalkerError as e: # deparsing failed @@ -175,13 +183,15 @@ def compile_file(source_path: str) -> str: def decompile_file( filename: str, - outstream=None, - showasm=None, + outstream: Optional[TextIO] = None, + showasm: Optional[str] = None, showast={}, showgrammar=False, source_encoding=None, mapstream=None, do_fragments=False, + start_offset=0, + stop_offset=-1, ) -> Any: """ decompile Python byte-code file (.pyc). Return objects to @@ -211,6 +221,8 @@ def decompile_file( is_pypy=is_pypy, magic_int=magic_int, mapstream=mapstream, + start_offset=start_offset, + stop_offset=stop_offset, ), ) else: @@ -231,6 +243,8 @@ def decompile_file( mapstream=mapstream, do_fragments=do_fragments, compile_mode="exec", + start_offset=start_offset, + stop_offset=stop_offset, ) ] return deparsed @@ -242,13 +256,16 @@ def main( out_base: Optional[str], compiled_files: list, source_files: list, - outfile=None, + outfile: Optional[str] = None, showasm: Optional[str] = None, showast={}, - showgrammar=False, + do_verify: Optional[str] = None, + showgrammar: bool = False, source_encoding=None, do_linemaps=False, do_fragments=False, + start_offset: int = 0, + stop_offset: int = -1, ) -> Tuple[int, int, int, int]: """ in_base base directory for input files @@ -261,7 +278,8 @@ def main( - files below out_base out_base=... - stdout out_base=None, outfile=None """ - tot_files = okay_files = failed_files = verify_failed_files = 0 + tot_files = okay_files = failed_files = 0 + verify_failed_files = 0 if do_verify else 0 current_outfile = outfile linemap_stream = None @@ -269,9 +287,9 @@ def main( compiled_files.append(compile_file(source_path)) for filename in compiled_files: - infile = os.path.join(in_base, filename) + infile = osp.join(in_base, filename) # print("XXX", infile) - if not os.path.exists(infile): + if not osp.exists(infile): sys.stderr.write(f"File '{infile}' doesn't exist. Skipped\n") continue @@ -284,14 +302,19 @@ def main( if outfile: # outfile was given as parameter outstream = _get_outstream(outfile) elif out_base is None: - outstream = sys.stdout + out_base = tempfile.mkdtemp(prefix="py-dis-") + if do_verify and filename.endswith(".pyc"): + current_outfile = osp.join(out_base, filename[0:-1]) + outstream = open(current_outfile, "w") + else: + outstream = sys.stdout if do_linemaps: linemap_stream = sys.stdout else: if filename.endswith(".pyc"): - current_outfile = os.path.join(out_base, filename[0:-1]) + current_outfile = osp.join(out_base, filename[0:-1]) else: - current_outfile = os.path.join(out_base, filename) + "_dis" + current_outfile = osp.join(out_base, filename) + "_dis" pass pass @@ -299,9 +322,9 @@ def main( # print(current_outfile, file=sys.stderr) - # Try to uncompile the input file + # Try to decompile the input file. try: - deparsed = decompile_file( + deparsed_objects = decompile_file( infile, outstream, showasm, @@ -310,11 +333,13 @@ def main( source_encoding, linemap_stream, do_fragments, + start_offset, + stop_offset, ) if do_fragments: - for d in deparsed: + for deparsed_object in deparsed_objects: last_mod = None - offsets = d.offsets + offsets = deparsed_object.offsets for e in sorted( [k for k in offsets.keys() if isinstance(k[1], int)] ): @@ -323,11 +348,48 @@ def main( outstream.write(f"{line}\n{e[0]}\n{line}\n") last_mod = e[0] info = offsets[e] - extract_info = d.extract_node_info(info) + extract_info = deparse_object.extract_node_info(info) outstream.write(f"{info.node.format().strip()}" + "\n") outstream.write(extract_info.selectedLine + "\n") outstream.write(extract_info.markerLine + "\n\n") pass + + if do_verify: + for deparsed_object in deparsed_objects: + deparsed_object.f.close() + if PYTHON_VERSION_TRIPLE[:2] != deparsed_object.version[:2]: + sys.stdout.write( + f"\n# skipping running {deparsed_object.f.name}; it is" + f"{version_tuple_to_str(deparsed_object.version, end=2)}, " + "and we are " + f"{version_tuple_to_str(PYTHON_VERSION_TRIPLE, end=2)}\n" + ) + else: + check_type = "syntax check" + if do_verify == "run": + check_type = "run" + result = subprocess.run( + [sys.executable, deparsed_object.f.name], + capture_output=True, + ) + valid = result.returncode == 0 + output = result.stdout.decode() + if output: + print(output) + pass + if not valid: + print(result.stderr.decode()) + + else: + valid = syntax_check(deparsed_object.f.name) + + if not valid: + verify_failed_files += 1 + sys.stderr.write( + f"\n# {check_type} failed on file {deparsed_object.f.name}\n" + ) + + # sys.stderr.write(f"Ran {deparsed_object.f.name}\n") pass tot_files += 1 except (ValueError, SyntaxError, ParserError, pysource.SourceWalkerError) as e: diff --git a/uncompyle6/semantics/fragments.py b/uncompyle6/semantics/fragments.py index 90a2ee57..23dc83fd 100644 --- a/uncompyle6/semantics/fragments.py +++ b/uncompyle6/semantics/fragments.py @@ -2036,6 +2036,8 @@ def code_deparse( code_objects={}, compile_mode="exec", walker=FragmentsWalker, + start_offset: int = 0, + stop_offset: int = -1, ): """ Convert the code object co into a python source fragment. @@ -2070,6 +2072,22 @@ def code_deparse( tokens, customize = scanner.ingest(co, code_objects=code_objects, show_asm=show_asm) tokens, customize = scanner.ingest(co) + + if start_offset > 0: + for i, t in enumerate(tokens): + # If t.offset is a string, we want to skip this. + if isinstance(t.offset, int) and t.offset >= start_offset: + tokens = tokens[i:] + break + + if stop_offset > -1: + for i, t in enumerate(tokens): + # In contrast to the test for start_offset If t.offset is + # a string, we want to extract the integer offset value. + if t.off2int() >= stop_offset: + tokens = tokens[:i] + break + maybe_show_asm(show_asm, tokens) debug_parser = dict(PARSER_DEFAULT_DEBUG) From 5d8c40358eb9aeedd475985b61930bd875be82f3 Mon Sep 17 00:00:00 2001 From: rocky Date: Sat, 10 Feb 2024 16:29:59 -0500 Subject: [PATCH 07/10] showtree workaround until we have better sync.. with decompyle3 --- uncompyle6/semantics/transform.py | 43 ++++++++++++++++++------------- 1 file changed, 25 insertions(+), 18 deletions(-) diff --git a/uncompyle6/semantics/transform.py b/uncompyle6/semantics/transform.py index e3f96c83..0339d8af 100644 --- a/uncompyle6/semantics/transform.py +++ b/uncompyle6/semantics/transform.py @@ -13,14 +13,15 @@ # You should have received a copy of the GNU General Public License # along with this program. If not, see . -from uncompyle6.show import maybe_show_tree from copy import copy + from spark_parser import GenericASTTraversal, GenericASTTraversalPruningException -from uncompyle6.semantics.helper import find_code_node from uncompyle6.parsers.treenode import SyntaxTree from uncompyle6.scanners.tok import NoneToken, Token -from uncompyle6.semantics.consts import RETURN_NONE, ASSIGN_DOC_STRING +from uncompyle6.semantics.consts import ASSIGN_DOC_STRING, RETURN_NONE +from uncompyle6.semantics.helper import find_code_node +from uncompyle6.show import maybe_show_tree def is_docstring(node, version, co_consts): @@ -74,7 +75,9 @@ class TreeTransform(GenericASTTraversal, object): return def maybe_show_tree(self, ast): - if isinstance(self.showast, dict) and self.showast: + if isinstance(self.showast, dict) and ( + self.showast.get("before") or self.showast.get("after") + ): maybe_show_tree(self, ast) def preorder(self, node=None): @@ -121,10 +124,7 @@ class TreeTransform(GenericASTTraversal, object): if isinstance(mkfunc_pattr, tuple): assert len(mkfunc_pattr) == 4 and isinstance(mkfunc_pattr, int) - if ( - len(code.co_consts) > 0 - and isinstance(code.co_consts[0], str) - ): + if len(code.co_consts) > 0 and isinstance(code.co_consts[0], str): docstring_node = SyntaxTree( "docstring", [Token("LOAD_STR", has_arg=True, pattr=code.co_consts[0])] ) @@ -136,7 +136,7 @@ class TreeTransform(GenericASTTraversal, object): def n_ifstmt(self, node): """Here we check if we can turn an `ifstmt` or 'iflaststmtl` into - some kind of `assert` statement""" + some kind of `assert` statement""" testexpr = node[0] @@ -148,7 +148,11 @@ class TreeTransform(GenericASTTraversal, object): if ifstmts_jump == "_ifstmts_jumpl" and ifstmts_jump[0] == "_ifstmts_jump": ifstmts_jump = ifstmts_jump[0] - elif ifstmts_jump not in ("_ifstmts_jump", "_ifstmts_jumpl", "ifstmts_jumpl"): + elif ifstmts_jump not in ( + "_ifstmts_jump", + "_ifstmts_jumpl", + "ifstmts_jumpl", + ): return node stmts = ifstmts_jump[0] else: @@ -208,7 +212,7 @@ class TreeTransform(GenericASTTraversal, object): kind = "assert2not" LOAD_ASSERT = call[0].first_child() - if LOAD_ASSERT not in ( "LOAD_ASSERT", "LOAD_GLOBAL"): + if LOAD_ASSERT not in ("LOAD_ASSERT", "LOAD_GLOBAL"): return node if isinstance(call[1], SyntaxTree): expr = call[1][0] @@ -289,7 +293,12 @@ class TreeTransform(GenericASTTraversal, object): len_n = len(n) # Sometimes stmt is reduced away and n[0] can be a single reduction like continue -> CONTINUE. - if len_n == 1 and isinstance(n[0], SyntaxTree) and len(n[0]) == 1 and n[0] == "stmt": + if ( + len_n == 1 + and isinstance(n[0], SyntaxTree) + and len(n[0]) == 1 + and n[0] == "stmt" + ): n = n[0][0] elif len_n == 0: return node @@ -413,17 +422,15 @@ class TreeTransform(GenericASTTraversal, object): new_stmts = [node[0]] for i, sstmt in enumerate(node[1:]): ann_assign = sstmt[0] - if ( - ann_assign == "ann_assign" - and prev == "assign" - ): + if ann_assign == "ann_assign" and prev == "assign": annotate_var = ann_assign[-2] if annotate_var.attr == prev[-1][0].attr: node[i].kind = "deleted " + node[i].kind del new_stmts[-1] ann_assign_init = SyntaxTree( - "ann_assign_init", [ann_assign[0], copy(prev[0]), annotate_var] - ) + "ann_assign_init", + [ann_assign[0], copy(prev[0]), annotate_var], + ) if sstmt[0] == "ann_assign": sstmt[0] = ann_assign_init else: From c8b92e227576f19ad6aa29d84ee6c68934e803d5 Mon Sep 17 00:00:00 2001 From: rocky Date: Sat, 10 Feb 2024 20:04:18 -0500 Subject: [PATCH 08/10] Add needed newline separating abstract tree --- uncompyle6/semantics/pysource.py | 1 + 1 file changed, 1 insertion(+) diff --git a/uncompyle6/semantics/pysource.py b/uncompyle6/semantics/pysource.py index 71459685..4991484c 100644 --- a/uncompyle6/semantics/pysource.py +++ b/uncompyle6/semantics/pysource.py @@ -324,6 +324,7 @@ class SourceWalker(GenericASTTraversal, NonterminalActions, ComprehensionMixin): """ ---- end before transform """ + + " " ) if self.showast.get("after", False): self.println( From f1bf86088ed3559955d33ca46aa6ac1af79dc132 Mon Sep 17 00:00:00 2001 From: rocky Date: Sat, 10 Feb 2024 21:06:05 -0500 Subject: [PATCH 09/10] Sync with decompyle3 --- uncompyle6/semantics/pysource.py | 7 +++-- uncompyle6/semantics/transform.py | 51 ++++++++++++++++++++----------- 2 files changed, 38 insertions(+), 20 deletions(-) diff --git a/uncompyle6/semantics/pysource.py b/uncompyle6/semantics/pysource.py index 4991484c..f2bf4357 100644 --- a/uncompyle6/semantics/pysource.py +++ b/uncompyle6/semantics/pysource.py @@ -210,7 +210,8 @@ class SourceWalkerError(Exception): class SourceWalker(GenericASTTraversal, NonterminalActions, ComprehensionMixin): """ - Class to traverses a Parse Tree of the bytecode instruction built from parsing to produce some sort of source text. + Class to traverses a Parse Tree of the bytecode instruction built from parsing to + produce some sort of source text. The Parse tree may be turned an Abstract Syntax tree as an intermediate step. """ @@ -318,7 +319,7 @@ class SourceWalker(GenericASTTraversal, NonterminalActions, ComprehensionMixin): customize_for_version(self, is_pypy, version) return - def maybe_show_tree(self, ast, phase): + def maybe_show_tree(self, tree, phase): if self.showast.get("before", False): self.println( """ @@ -334,7 +335,7 @@ class SourceWalker(GenericASTTraversal, NonterminalActions, ComprehensionMixin): + " " ) if self.showast.get(phase, False): - maybe_show_tree(self, ast) + maybe_show_tree(self, tree) def str_with_template(self, ast): stream = sys.stdout diff --git a/uncompyle6/semantics/transform.py b/uncompyle6/semantics/transform.py index 0339d8af..430f1e0f 100644 --- a/uncompyle6/semantics/transform.py +++ b/uncompyle6/semantics/transform.py @@ -1,4 +1,4 @@ -# Copyright (c) 2019-2023 by Rocky Bernstein +# Copyright (c) 2019-2024 by Rocky Bernstein # This program is free software: you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by @@ -14,6 +14,7 @@ # along with this program. If not, see . from copy import copy +from typing import Optional from spark_parser import GenericASTTraversal, GenericASTTraversalPruningException @@ -56,29 +57,34 @@ def is_docstring(node, version, co_consts): return node == ASSIGN_DOC_STRING(co_consts[0], doc_load) -def is_not_docstring(call_stmt_node): +def is_not_docstring(call_stmt_node) -> bool: try: return ( call_stmt_node == "call_stmt" and call_stmt_node[0][0] == "LOAD_STR" and call_stmt_node[1] == "POP_TOP" ) - except: + except Exception: return False class TreeTransform(GenericASTTraversal, object): - def __init__(self, version, show_ast=None, is_pypy=False): + def __init__( + self, + version: tuple, + is_pypy=False, + show_ast: Optional[dict] = None, + ): self.version = version self.showast = show_ast self.is_pypy = is_pypy return - def maybe_show_tree(self, ast): + def maybe_show_tree(self, tree): if isinstance(self.showast, dict) and ( self.showast.get("before") or self.showast.get("after") ): - maybe_show_tree(self, ast) + maybe_show_tree(self, tree) def preorder(self, node=None): """Walk the tree in roughly 'preorder' (a bit of a lie explained below). @@ -122,6 +128,7 @@ class TreeTransform(GenericASTTraversal, object): mkfunc_pattr = node[-1].pattr if isinstance(mkfunc_pattr, tuple): + assert isinstance(mkfunc_pattr, tuple) assert len(mkfunc_pattr) == 4 and isinstance(mkfunc_pattr, int) if len(code.co_consts) > 0 and isinstance(code.co_consts[0], str): @@ -216,6 +223,7 @@ class TreeTransform(GenericASTTraversal, object): return node if isinstance(call[1], SyntaxTree): expr = call[1][0] + assert_expr.transformed_by = "n_ifstmt" node = SyntaxTree( kind, [ @@ -225,8 +233,8 @@ class TreeTransform(GenericASTTraversal, object): expr, RAISE_VARARGS_1, ], + transformed_by="n_ifstmt", ) - node.transformed_by = "n_ifstmt" pass pass else: @@ -254,9 +262,10 @@ class TreeTransform(GenericASTTraversal, object): LOAD_ASSERT = expr[0] node = SyntaxTree( - kind, [assert_expr, jump_cond, LOAD_ASSERT, RAISE_VARARGS_1] + kind, + [assert_expr, jump_cond, LOAD_ASSERT, RAISE_VARARGS_1], + transformed_by="n_ifstmt", ) - node.transformed_by = ("n_ifstmt",) pass pass return node @@ -416,6 +425,12 @@ class TreeTransform(GenericASTTraversal, object): list_for_node.transformed_by = ("n_list_for",) return list_for_node + def n_negated_testtrue(self, node): + assert node[0] == "testtrue" + test_node = node[0][0] + test_node.transformed_by = "n_negated_testtrue" + return test_node + def n_stmts(self, node): if node.first_child() == "SETUP_ANNOTATIONS": prev = node[0][0] @@ -448,26 +463,28 @@ class TreeTransform(GenericASTTraversal, object): node = self.preorder(node) return node - def transform(self, ast, code): - self.maybe_show_tree(ast) - self.ast = copy(ast) + def transform(self, parse_tree: GenericASTTraversal, code) -> GenericASTTraversal: + self.maybe_show_tree(parse_tree) + self.ast = copy(parse_tree) + del parse_tree self.ast = self.traverse(self.ast, is_lambda=False) + n = len(self.ast) try: # Disambiguate a string (expression) which appears as a "call_stmt" at # the beginning of a function versus a docstring. Seems pretty academic, # but this is Python. - call_stmt = ast[0][0] + call_stmt = self.ast[0][0] if is_not_docstring(call_stmt): call_stmt.kind = "string_at_beginning" call_stmt.transformed_by = "transform" pass - except: + except Exception: pass try: - for i in range(len(self.ast)): - sstmt = ast[i] + for i in range(n): + sstmt = self.ast[i] if len(sstmt) == 1 and sstmt == "sstmt": self.ast[i] = self.ast[i][0] @@ -493,7 +510,7 @@ class TreeTransform(GenericASTTraversal, object): if self.ast[-1] == RETURN_NONE: self.ast.pop() # remove last node # todo: if empty, add 'pass' - except: + except Exception: pass return self.ast From 147155e1d586a2db86a26b474eac5b2a723f89e6 Mon Sep 17 00:00:00 2001 From: rocky Date: Sun, 11 Feb 2024 08:42:32 -0500 Subject: [PATCH 10/10] Administrivia: automate merging --- admin-tools/merge-for-2.4.sh | 5 +++++ admin-tools/merge-for-3.0.sh | 5 +++++ admin-tools/merge-for-3.3.sh | 5 +++++ 3 files changed, 15 insertions(+) create mode 100755 admin-tools/merge-for-2.4.sh create mode 100755 admin-tools/merge-for-3.0.sh create mode 100755 admin-tools/merge-for-3.3.sh diff --git a/admin-tools/merge-for-2.4.sh b/admin-tools/merge-for-2.4.sh new file mode 100755 index 00000000..f8c55395 --- /dev/null +++ b/admin-tools/merge-for-2.4.sh @@ -0,0 +1,5 @@ +#/bin/bash +cd $(dirname ${BASH_SOURCE[0]}) +if . ./setup-python-2.4.sh; then + git merge python-3.0-to-3.2 +fi diff --git a/admin-tools/merge-for-3.0.sh b/admin-tools/merge-for-3.0.sh new file mode 100755 index 00000000..7fc1a596 --- /dev/null +++ b/admin-tools/merge-for-3.0.sh @@ -0,0 +1,5 @@ +#/bin/bash +cd $(dirname ${BASH_SOURCE[0]}) +if . ./setup-python-3.0.sh; then + git merge python-3.3-to-3.5 +fi diff --git a/admin-tools/merge-for-3.3.sh b/admin-tools/merge-for-3.3.sh new file mode 100755 index 00000000..aade2e77 --- /dev/null +++ b/admin-tools/merge-for-3.3.sh @@ -0,0 +1,5 @@ +#/bin/bash +cd $(dirname ${BASH_SOURCE[0]}) +if . ./setup-python-3.3.sh; then + git merge master +fi