From f4d21d36e5c86d120de636692e91f87e4e969752 Mon Sep 17 00:00:00 2001 From: rocky Date: Tue, 19 Nov 2024 15:27:50 -0500 Subject: [PATCH 01/12] Add BlackHat Asia 2024 and update CircleCI link --- README.rst | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/README.rst b/README.rst index 124b486f..b114d73d 100644 --- a/README.rst +++ b/README.rst @@ -282,6 +282,7 @@ to spend. See Also -------- +* https://rocky.github.io/blackhat-asia-2024-additional/all-notes-print.html : How to Read and Write a High-Level Bytecode Decompiler: ``uncompyle6`` ``decompyle3`` -- BlackHat 2024 Asia (`video `_). A big thanks to the Organizers and Reviewers for letting me speak. This kind of thing encourages me to work on projects like this. * https://github.com/rocky/python-decompile3 : Much smaller and more modern code, focusing on 3.7 and 3.8. Changes in that will get migrated back here. * https://code.google.com/archive/p/unpyc3/ : supports Python 3.2 only. The above projects use a different decompiling technique than what is used here. Currently unmaintained. * https://github.com/figment/unpyc3/ : fork of above, but supports Python 3.3 only. Includes some fixes like supporting function annotations. Currently unmaintained. @@ -306,8 +307,8 @@ See Also .. _uncompyle2: https://github.com/wibiti/uncompyle2 .. _unpyc37: https://github.com/andrew-tavera/unpyc37 .. _this: https://github.com/rocky/python-uncompyle6/wiki/Deparsing-technology-and-its-use-in-exact-location-reporting -.. |buildstatus| image:: https://travis-ci.org/rocky/python-uncompyle6.svg - :target: https://travis-ci.org/rocky/python-uncompyle6 +.. |buildstatus| image:: https://circleci.com/gh/rocky/python-uncompyle6.svg?style=svg + :target: https://app.circleci.com/pipelines/github/rocky/python-uncompyle6 .. |packagestatus| image:: https://repology.org/badge/vertical-allrepos/python:uncompyle6.svg :target: https://repology.org/project/python:uncompyle6/versions .. _PJOrion: http://www.koreanrandom.com/forum/topic/15280-pjorion-%D1%80%D0%B5%D0%B4%D0%B0%D0%BA%D1%82%D0%B8%D1%80%D0%BE%D0%B2%D0%B0%D0%BD%D0%B8%D0%B5-%D0%BA%D0%BE%D0%BC%D0%BF%D0%B8%D0%BB%D1%8F%D1%86%D0%B8%D1%8F-%D0%B4%D0%B5%D0%BA%D0%BE%D0%BC%D0%BF%D0%B8%D0%BB%D1%8F%D1%86%D0%B8%D1%8F-%D0%BE%D0%B1%D1%84 From 4181bcbc79eaf05d56bb7d1d1f78220cee502559 Mon Sep 17 00:00:00 2001 From: rocky Date: Tue, 26 Nov 2024 09:43:32 -0500 Subject: [PATCH 02/12] Correct getting code node on mkfunc --- uncompyle6/semantics/fragments.py | 15 ++------------- 1 file changed, 2 insertions(+), 13 deletions(-) diff --git a/uncompyle6/semantics/fragments.py b/uncompyle6/semantics/fragments.py index 8bb7aad1..dead329f 100644 --- a/uncompyle6/semantics/fragments.py +++ b/uncompyle6/semantics/fragments.py @@ -79,14 +79,13 @@ from uncompyle6.semantics import pysource from uncompyle6.semantics.check_ast import checker from uncompyle6.semantics.consts import ( INDENT_PER_LEVEL, - MAP, NONE, PASS, PRECEDENCE, TABLE_DIRECT, - TABLE_R, escape, ) +from uncompyle6.semantics.helper import find_code_node from uncompyle6.semantics.pysource import ( DEFAULT_DEBUG_OPTS, TREE_DEFAULT_DEBUG, @@ -596,17 +595,7 @@ class FragmentsWalker(pysource.SourceWalker, object): def n_mkfunc(self, node): start = len(self.f.getvalue()) - if self.version >= (3, 3) or node[-2] == "kwargs": - # LOAD_CONST code object .. - # LOAD_CONST 'x0' if >= 3.3 - # MAKE_FUNCTION .. - code_node = node[-3] - elif node[-2] == "expr": - code_node = node[-2][0] - else: - # LOAD_CONST code object .. - # MAKE_FUNCTION .. - code_node = node[-2] + code_node = find_code_node(node, -2) func_name = code_node.attr.co_name self.write(func_name) self.set_pos_info(code_node, start, len(self.f.getvalue())) From addddf82f51f0ac6e04a58cfd02d5f6db5849edb Mon Sep 17 00:00:00 2001 From: rocky Date: Tue, 26 Nov 2024 09:44:49 -0500 Subject: [PATCH 03/12] Correct getting code node on mkfunc --- uncompyle6/semantics/fragments.py | 15 ++------------- 1 file changed, 2 insertions(+), 13 deletions(-) diff --git a/uncompyle6/semantics/fragments.py b/uncompyle6/semantics/fragments.py index 537ca7f5..c27fed37 100644 --- a/uncompyle6/semantics/fragments.py +++ b/uncompyle6/semantics/fragments.py @@ -80,14 +80,13 @@ from uncompyle6.semantics import pysource from uncompyle6.semantics.check_ast import checker from uncompyle6.semantics.consts import ( INDENT_PER_LEVEL, - MAP, NONE, PASS, PRECEDENCE, TABLE_DIRECT, - TABLE_R, escape, ) +from uncompyle6.semantics.helper import find_code_node from uncompyle6.semantics.pysource import ( DEFAULT_DEBUG_OPTS, TREE_DEFAULT_DEBUG, @@ -597,17 +596,7 @@ class FragmentsWalker(pysource.SourceWalker, object): def n_mkfunc(self, node): start = len(self.f.getvalue()) - if self.version >= (3, 3) or node[-2] == "kwargs": - # LOAD_CONST code object .. - # LOAD_CONST 'x0' if >= 3.3 - # MAKE_FUNCTION .. - code_node = node[-3] - elif node[-2] == "expr": - code_node = node[-2][0] - else: - # LOAD_CONST code object .. - # MAKE_FUNCTION .. - code_node = node[-2] + code_node = find_code_node(node, -2) func_name = code_node.attr.co_name self.write(func_name) self.set_pos_info(code_node, start, len(self.f.getvalue())) From 4ac5564df3a58a75b9382b328602fc5e4312e0d7 Mon Sep 17 00:00:00 2001 From: rocky Date: Thu, 28 Nov 2024 07:27:13 -0500 Subject: [PATCH 04/12] Don't remove LOAD_CONST RETURN_VALUE when... the LOAD_CONST has a non-None value, or the LOAD_CONST has a line associated with it. --- uncompyle6/scanners/tok.py | 2 +- uncompyle6/semantics/pysource.py | 20 +++++++++++++++----- 2 files changed, 16 insertions(+), 6 deletions(-) diff --git a/uncompyle6/scanners/tok.py b/uncompyle6/scanners/tok.py index 0e701e97..75a56912 100644 --- a/uncompyle6/scanners/tok.py +++ b/uncompyle6/scanners/tok.py @@ -1,4 +1,4 @@ -# Copyright (c) 2016-2021, 2023 by Rocky Bernstein +# Copyright (c) 2016-2021, 2023-2024 by Rocky Bernstein # Copyright (c) 2000-2002 by hartmut Goebel # Copyright (c) 1999 John Aycock # diff --git a/uncompyle6/semantics/pysource.py b/uncompyle6/semantics/pysource.py index 00138023..78fd4e8c 100644 --- a/uncompyle6/semantics/pysource.py +++ b/uncompyle6/semantics/pysource.py @@ -1251,11 +1251,21 @@ class SourceWalker(GenericASTTraversal, NonterminalActions, ComprehensionMixin): if tokens[-1].kind in ("RETURN_VALUE", "RETURN_VALUE_LAMBDA"): # Python 3.4's classes can add a "return None" which is # invalid syntax. - if tokens[-2].kind == "LOAD_CONST": - if is_top_level_module or tokens[-2].pattr is None: - del tokens[-2:] - else: - tokens.append(Token("RETURN_LAST")) + load_const = tokens[-2] + # We should have: + # LOAD_CONST None + # with *no* line number associated the token. + # A line number on the token or a non-None + # token value a token based on user source + # text. + if ( + load_const.kind == "LOAD_CONST" + and load_const.linestart is None + and load_const.attr is None + or is_top_level_module + ): + # Delete LOAD_CONST (None) RETURN_VALUE + del tokens[-2:] else: tokens.append(Token("RETURN_LAST")) if len(tokens) == 0: From 74b39e22627b8dac4b599be1d8c1d856d937ed19 Mon Sep 17 00:00:00 2001 From: rocky Date: Thu, 28 Nov 2024 07:29:37 -0500 Subject: [PATCH 05/12] Administriva --- admin-tools/merge-for-3.6.sh | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/admin-tools/merge-for-3.6.sh b/admin-tools/merge-for-3.6.sh index 7915bda3..82c74335 100755 --- a/admin-tools/merge-for-3.6.sh +++ b/admin-tools/merge-for-3.6.sh @@ -1,7 +1,7 @@ #/bin/bash -uncompyle6_merge_33_owd=$(pwd) +uncompyle6_merge_36_owd=$(pwd) cd $(dirname ${BASH_SOURCE[0]}) -if . ./setup-python-3.3.sh; then +if . ./setup-python-3.6.sh; then git merge master fi -cd $uncompyle6_merge_33_owd +cd $uncompyle6_merge_36_owd From b71cd88b7317d4b4bea71380031e63ad9e500b9f Mon Sep 17 00:00:00 2001 From: rocky Date: Thu, 28 Nov 2024 08:01:15 -0500 Subject: [PATCH 06/12] Show return value when not None... And fixup setup.py --- setup.py | 63 +++++++++++++++++++++++++++++++- uncompyle6/semantics/pysource.py | 3 +- 2 files changed, 63 insertions(+), 3 deletions(-) diff --git a/setup.py b/setup.py index 2642edf8..88a7072d 100755 --- a/setup.py +++ b/setup.py @@ -1,6 +1,65 @@ #!/usr/bin/env python """Setup script for the 'uncompyle6' distribution.""" +import sys -from setuptools import setup +import setuptools -setup(packages=["uncompyle6"]) +SYS_VERSION = sys.version_info[0:2] +if not ((3, 6) <= SYS_VERSION < (3, 11)): + mess = "Python Release 3.6 .. 3.10 are supported in this code branch." + if (2, 4) <= SYS_VERSION <= (2, 7): + mess += ( + "\nFor your Python, version %s, use the python-2.4 code/branch." + % sys.version[0:3] + ) + elif SYS_VERSION >= (3, 10): + mess += ( + "\nFor your Python, version %s, use the master code/branch." + % sys.version[0:3] + ) + elif (3, 0) >= SYS_VERSION < (3, 3): + mess += ( + "\nFor your Python, version %s, use the python-3.0-to-3.2 code/branch." + % sys.version[0:3] + ) + elif SYS_VERSION < (2, 4): + mess += ( + "\nThis package is not supported for Python version %s." % sys.version[0:3] + ) + print(mess) + raise Exception(mess) + +from __pkginfo__ import ( + __version__, + author, + author_email, + classifiers, + entry_points, + install_requires, + license, + long_description, + modname, + py_modules, + short_desc, + web, + zip_safe, +) + +setuptools.setup( + author=author, + author_email=author_email, + classifiers=classifiers, + description=short_desc, + entry_points=entry_points, + install_requires=install_requires, + license=license, + long_description=long_description, + long_description_content_type="text/x-rst", + name=modname, + packages=setuptools.find_packages(), + py_modules=py_modules, + test_suite="nose.collector", + url=web, + version=__version__, + zip_safe=zip_safe, +) diff --git a/uncompyle6/semantics/pysource.py b/uncompyle6/semantics/pysource.py index 78fd4e8c..f31d5a55 100644 --- a/uncompyle6/semantics/pysource.py +++ b/uncompyle6/semantics/pysource.py @@ -1212,6 +1212,7 @@ class SourceWalker(GenericASTTraversal, NonterminalActions, ComprehensionMixin): is_lambda=False, noneInNames=False, is_top_level_module=False, + compile_mode="exec" ) -> GenericASTTraversal: # FIXME: DRY with fragments.py @@ -1262,7 +1263,6 @@ class SourceWalker(GenericASTTraversal, NonterminalActions, ComprehensionMixin): load_const.kind == "LOAD_CONST" and load_const.linestart is None and load_const.attr is None - or is_top_level_module ): # Delete LOAD_CONST (None) RETURN_VALUE del tokens[-2:] @@ -1371,6 +1371,7 @@ def code_deparse( co, is_lambda=is_lambda_mode(compile_mode), is_top_level_module=is_top_level_module, + compile_mode=compile_mode, ) # XXX workaround for profiling From e4e3743de5116bd9e5f04b65fb59dfeae7d8ea78 Mon Sep 17 00:00:00 2001 From: rocky Date: Thu, 28 Nov 2024 08:03:32 -0500 Subject: [PATCH 07/12] Tweak when we delete LOAD_CONST RETURN_VALUE --- uncompyle6/semantics/pysource.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/uncompyle6/semantics/pysource.py b/uncompyle6/semantics/pysource.py index 78fd4e8c..5fb77771 100644 --- a/uncompyle6/semantics/pysource.py +++ b/uncompyle6/semantics/pysource.py @@ -1212,6 +1212,7 @@ class SourceWalker(GenericASTTraversal, NonterminalActions, ComprehensionMixin): is_lambda=False, noneInNames=False, is_top_level_module=False, + compile_mode="exec", ) -> GenericASTTraversal: # FIXME: DRY with fragments.py @@ -1262,7 +1263,6 @@ class SourceWalker(GenericASTTraversal, NonterminalActions, ComprehensionMixin): load_const.kind == "LOAD_CONST" and load_const.linestart is None and load_const.attr is None - or is_top_level_module ): # Delete LOAD_CONST (None) RETURN_VALUE del tokens[-2:] @@ -1371,6 +1371,7 @@ def code_deparse( co, is_lambda=is_lambda_mode(compile_mode), is_top_level_module=is_top_level_module, + compile_mode=compile_mode, ) # XXX workaround for profiling From f72b2c1153a94b201d526043c4a40772c4ed473c Mon Sep 17 00:00:00 2001 From: c10udlnk <73152692+c10udlnk@users.noreply.github.com> Date: Fri, 29 Nov 2024 11:03:13 +0800 Subject: [PATCH 08/12] Fix when parsing NOP Fix error when parsing NOP opcode --- uncompyle6/parser.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/uncompyle6/parser.py b/uncompyle6/parser.py index 2409f9f1..35d9bb55 100644 --- a/uncompyle6/parser.py +++ b/uncompyle6/parser.py @@ -391,6 +391,10 @@ class PythonParser(GenericASTBuilder): returns ::= return returns ::= _stmts return + + # NOP + stmt ::= nop_stmt + nop_stmt ::= NOP """ pass From efd28710cece31d891945b53690d03c50499edc2 Mon Sep 17 00:00:00 2001 From: c10udlnk <73152692+c10udlnk@users.noreply.github.com> Date: Sun, 1 Dec 2024 16:22:54 +0800 Subject: [PATCH 09/12] Add NOP test cases Add test cases for check NOP opcode --- test/bytecode_2.7/06_nop.pyc | Bin 0 -> 163 bytes test/bytecode_3.6/06_nop.pyc | Bin 0 -> 160 bytes 2 files changed, 0 insertions(+), 0 deletions(-) create mode 100644 test/bytecode_2.7/06_nop.pyc create mode 100644 test/bytecode_3.6/06_nop.pyc diff --git a/test/bytecode_2.7/06_nop.pyc b/test/bytecode_2.7/06_nop.pyc new file mode 100644 index 0000000000000000000000000000000000000000..0f6f966d16b7537f730aedf29e7f7d1e4ce83822 GIT binary patch literal 163 zcmZSn%*&O@>XV+#00oRd+5w1*<$wey2&6DDL_wLU3=HlbU?PQ)Ay|VMq>dj*`1uDY zX#m9-G(bj`fQVuUA1nwWin)PAUVcG5NU>f)WeEq6Zks`^+~s2U|@I*#Bjg_WH|tFu^N!z1j7`DD8^KVRK{jTMyLoU1B{=-7|fu_ zRK@S-AE1z5ker{As!)Bh9YL5VPN7HcV2!$JV=RNLFFwD Yo80`A(wtN~Mv#phK!S~tg^>dY0r(;wN&o-= literal 0 HcmV?d00001 From b2cf041ec37ff316ca84b62538f1ea28741684f5 Mon Sep 17 00:00:00 2001 From: rocky Date: Mon, 2 Dec 2024 19:39:56 -0500 Subject: [PATCH 10/12] Remove unused import --- uncompyle6/semantics/customize37.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/uncompyle6/semantics/customize37.py b/uncompyle6/semantics/customize37.py index 41e375ab..60210800 100644 --- a/uncompyle6/semantics/customize37.py +++ b/uncompyle6/semantics/customize37.py @@ -17,7 +17,7 @@ import re -from uncompyle6.semantics.consts import INDENT_PER_LEVEL, PRECEDENCE, TABLE_DIRECT +from uncompyle6.semantics.consts import INDENT_PER_LEVEL, PRECEDENCE from uncompyle6.semantics.helper import flatten_list # FIXME get from a newer xdis From 7ca4363602daa50a5978302b085a6c2539cece95 Mon Sep 17 00:00:00 2001 From: rocky Date: Thu, 12 Dec 2024 18:03:31 -0500 Subject: [PATCH 11/12] Administrivia --- .pre-commit-config.yaml | 8 ++++---- admin-tools/setup-python-2.4.sh | 2 +- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index b51fd593..4cbf372a 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -6,17 +6,17 @@ repos: hooks: - id: check-merge-conflict - id: debug-statements - stages: [commit] + stages: [pre-commit] - id: end-of-file-fixer - stages: [commit] + stages: [pre-commit] - repo: https://github.com/pycqa/isort rev: 5.13.2 hooks: - id: isort - stages: [commit] + stages: [pre-commit] - repo: https://github.com/psf/black rev: 23.12.1 hooks: - id: black language_version: python3 - stages: [commit] + stages: [pre-commit] diff --git a/admin-tools/setup-python-2.4.sh b/admin-tools/setup-python-2.4.sh index 843cf285..71e4ee04 100755 --- a/admin-tools/setup-python-2.4.sh +++ b/admin-tools/setup-python-2.4.sh @@ -17,7 +17,7 @@ cd $mydir (cd $fulldir/.. && \ setup_version python-spark python-2.4 && \ - setup_verseion python-xdis python-2.4-to-2.7) + setup_version python-xdis python-2.4-to-2.7) checkout_finish python-2.4-to-2.7 From 2f1ab4f63eb76d7826e3ea3f88cc4ea086ce99ca Mon Sep 17 00:00:00 2001 From: rocky Date: Thu, 12 Dec 2024 18:05:29 -0500 Subject: [PATCH 12/12] Merge hell --- uncompyle6/semantics/pysource.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/uncompyle6/semantics/pysource.py b/uncompyle6/semantics/pysource.py index f31d5a55..5fb77771 100644 --- a/uncompyle6/semantics/pysource.py +++ b/uncompyle6/semantics/pysource.py @@ -1212,7 +1212,7 @@ class SourceWalker(GenericASTTraversal, NonterminalActions, ComprehensionMixin): is_lambda=False, noneInNames=False, is_top_level_module=False, - compile_mode="exec" + compile_mode="exec", ) -> GenericASTTraversal: # FIXME: DRY with fragments.py