From bec88e4aaaf9b3abf69eabc95126e0756e98341d Mon Sep 17 00:00:00 2001 From: rocky Date: Thu, 14 Mar 2024 15:31:53 -0400 Subject: [PATCH 1/2] Name phases "disassembly" and "tokenization" --- uncompyle6/scanners/scanner2.py | 35 ++++++++++++++++++-------------- uncompyle6/scanners/scanner26.py | 6 +++--- uncompyle6/scanners/scanner3.py | 4 ++-- 3 files changed, 25 insertions(+), 20 deletions(-) diff --git a/uncompyle6/scanners/scanner2.py b/uncompyle6/scanners/scanner2.py index bd1f2769..4bcb30fd 100644 --- a/uncompyle6/scanners/scanner2.py +++ b/uncompyle6/scanners/scanner2.py @@ -1,4 +1,4 @@ -# Copyright (c) 2015-2023 by Rocky Bernstein +# Copyright (c) 2015-2024 by Rocky Bernstein # Copyright (c) 2005 by Dan Pascu # Copyright (c) 2000-2002 by hartmut Goebel # @@ -36,13 +36,13 @@ Finally we save token information. from __future__ import print_function from copy import copy - -from xdis import code2num, iscode, op_has_argument, instruction_size -from xdis.bytecode import _get_const_info -from uncompyle6.scanner import Scanner, Token - from sys import intern +from xdis import code2num, instruction_size, iscode, op_has_argument +from xdis.bytecode import _get_const_info + +from uncompyle6.scanner import Scanner, Token + class Scanner2(Scanner): def __init__(self, version, show_asm=None, is_pypy=False): @@ -206,7 +206,7 @@ class Scanner2(Scanner): bytecode = self.build_instructions(co) if show_asm in ("both", "before"): - print("\n# ---- before tokenization:") + print("\n# ---- disassembly:") bytecode.disassemble_bytes( co.co_code, varnames=co.co_varnames, @@ -235,7 +235,6 @@ class Scanner2(Scanner): # 'LOAD_ASSERT' is used in assert statements. self.load_asserts = set() for i in self.op_range(0, codelen): - # We need to detect the difference between: # raise AssertionError # and @@ -327,9 +326,14 @@ class Scanner2(Scanner): "BUILD_SET", ): t = Token( - op_name, oparg, pattr, offset, + op_name, + oparg, + pattr, + offset, self.linestarts.get(offset, None), - op, has_arg, self.opc + op, + has_arg, + self.opc, ) collection_type = op_name.split("_")[1] next_tokens = self.bound_collection_from_tokens( @@ -490,7 +494,7 @@ class Scanner2(Scanner): pass if show_asm in ("both", "after"): - print("\n# ---- after tokenization:") + print("\n# ---- tokenization:") for t in new_tokens: print(t.format(line_prefix="")) print() @@ -540,14 +544,17 @@ class Scanner2(Scanner): for s in stmt_list: if code[s] == self.opc.JUMP_ABSOLUTE and s not in pass_stmts: target = self.get_target(s) - if target > s or (self.lines and self.lines[last_stmt].l_no == self.lines[s].l_no): + if target > s or ( + self.lines and self.lines[last_stmt].l_no == self.lines[s].l_no + ): stmts.remove(s) continue j = self.prev[s] while code[j] == self.opc.JUMP_ABSOLUTE: j = self.prev[j] if ( - self.version >= (2, 3) and self.opname_for_offset(j) == "LIST_APPEND" + self.version >= (2, 3) + and self.opname_for_offset(j) == "LIST_APPEND" ): # list comprehension stmts.remove(s) continue @@ -924,7 +931,6 @@ class Scanner2(Scanner): # Is it an "and" inside an "if" or "while" block if op == self.opc.PJIF: - # Search for other POP_JUMP_IF_...'s targeting the # same target, of the current POP_JUMP_... instruction, # starting from current offset, and filter everything inside inner 'or' @@ -1116,7 +1122,6 @@ class Scanner2(Scanner): # Is this a loop and not an "if" statement? if (if_end < pre_rtarget) and (pre[if_end] in self.setup_loop_targets): - if if_end > start: return else: diff --git a/uncompyle6/scanners/scanner26.py b/uncompyle6/scanners/scanner26.py index 208e94b1..b15c0d2f 100755 --- a/uncompyle6/scanners/scanner26.py +++ b/uncompyle6/scanners/scanner26.py @@ -1,4 +1,4 @@ -# Copyright (c) 2015-2017, 2021-2022 by Rocky Bernstein +# Copyright (c) 2015-2017, 2021-2022, 2024 by Rocky Bernstein # Copyright (c) 2005 by Dan Pascu # Copyright (c) 2000-2002 by hartmut Goebel # @@ -80,7 +80,7 @@ class Scanner26(scan.Scanner2): # show_asm = 'after' if show_asm in ("both", "before"): - print("\n# ---- before tokenization:") + print("\n# ---- disassembly:") for instr in bytecode.get_instructions(co): print(instr.disassemble(self.opc)) @@ -346,7 +346,7 @@ class Scanner26(scan.Scanner2): pass if show_asm in ("both", "after"): - print("\n# ---- after tokenization:") + print("\n# ---- tokenization:") for t in tokens: print(t.format(line_prefix="")) print() diff --git a/uncompyle6/scanners/scanner3.py b/uncompyle6/scanners/scanner3.py index 634417af..6804b04c 100644 --- a/uncompyle6/scanners/scanner3.py +++ b/uncompyle6/scanners/scanner3.py @@ -418,7 +418,7 @@ class Scanner3(Scanner): # show_asm = 'both' if show_asm in ("both", "before"): - print("\n# ---- before tokenization:") + print("\n# ---- disassembly:") bytecode.disassemble_bytes( co.co_code, varnames=co.co_varnames, @@ -788,7 +788,7 @@ class Scanner3(Scanner): pass if show_asm in ("both", "after"): - print("\n# ---- after tokenization:") + print("\n# ---- tokenization:") for t in new_tokens: print(t.format(line_prefix="")) print() From ee72f6d685a2c675a5f477c5c6f7f55218d2069e Mon Sep 17 00:00:00 2001 From: rocky Date: Fri, 15 Mar 2024 22:02:53 -0400 Subject: [PATCH 2/2] Get ready for release 3.9.1 --- .circleci/config.yml | 4 +-- .github/workflows/osx.yml | 4 +-- .github/workflows/ubuntu.yml | 3 +- .github/workflows/windows.yml | 4 +-- NEWS.md | 20 ++++++++++++ __pkginfo__.py | 2 ++ setup.py | 61 ++--------------------------------- uncompyle6/version.py | 2 +- 8 files changed, 33 insertions(+), 67 deletions(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index f05ee99b..4850625c 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -43,9 +43,9 @@ jobs: - run: command: | # Use pip to install dependengcies sudo pip install --user --upgrade setuptools - # Until the next release - sudo pip install git+https://github.com/rocky/python-xdis#egg=xdis pip install --user -e . + # Not sure why "pip install -e" doesn't work above + pip install click spark-parser xdis pip install --user -r requirements-dev.txt # Save dependency cache diff --git a/.github/workflows/osx.yml b/.github/workflows/osx.yml index 38eac9b0..724517d6 100644 --- a/.github/workflows/osx.yml +++ b/.github/workflows/osx.yml @@ -22,9 +22,9 @@ jobs: - name: Install dependencies run: | python -m pip install --upgrade pip - # Until the next xdis release - pip install git+https://github.com/rocky/python-xdis#egg=xdis pip install -e . + # Not sure why "pip install -e" doesn't work above + pip install click spark-parser xdis pip install -r requirements-dev.txt - name: Test uncompyle6 run: | diff --git a/.github/workflows/ubuntu.yml b/.github/workflows/ubuntu.yml index bfabdfee..d5093c4c 100644 --- a/.github/workflows/ubuntu.yml +++ b/.github/workflows/ubuntu.yml @@ -21,9 +21,8 @@ jobs: - name: Install dependencies run: | python -m pip install --upgrade pip - # Until the next xdis release - pip install git+https://github.com/rocky/python-xdis#egg=xdis pip install -e . + pip install click spark-parser xdis pip install -r requirements-dev.txt - name: Test uncompyle6 run: | diff --git a/.github/workflows/windows.yml b/.github/workflows/windows.yml index 34d1ea55..82d36e92 100644 --- a/.github/workflows/windows.yml +++ b/.github/workflows/windows.yml @@ -22,9 +22,9 @@ jobs: - name: Install dependencies run: | python -m pip install --upgrade pip - # Until the next xdis release - pip install git+https://github.com/rocky/python-xdis#egg=xdis pip install -e . + # Not sure why "pip install -e" doesn't work above + pip install click spark-parser xdis pip install -r requirements-dev.txt - name: Test uncompyle6 run: | diff --git a/NEWS.md b/NEWS.md index 887a597c..853dd00c 100644 --- a/NEWS.md +++ b/NEWS.md @@ -1,3 +1,23 @@ +3.9.1: 2024-05-15 +================= + +Lots of changes major changes. track xdis API has changes. + +Separate Phases more clearly: +* disassembly +* tokenization +* parsing +* abstracting to AST (more is done in newer projects) +* printing + +Although we do not decompile bytecode greater than 3.8, code supports running from up to 3.12. + +Many bugs fixed. + +A lot of Linting and coding style modernization. + +Work done in preparation for Blackhat Asia 2024 + 3.9.0: 2022-12-22 ================= diff --git a/__pkginfo__.py b/__pkginfo__.py index 98fa63d5..2f220105 100644 --- a/__pkginfo__.py +++ b/__pkginfo__.py @@ -62,6 +62,8 @@ classifiers = [ "Programming Language :: Python :: 3.8", "Programming Language :: Python :: 3.9", "Programming Language :: Python :: 3.10", + "Programming Language :: Python :: 3.11", + "Programming Language :: Python :: 3.12", "Programming Language :: Python :: Implementation :: PyPy", "Topic :: Software Development :: Debuggers", "Topic :: Software Development :: Libraries :: Python Modules", diff --git a/setup.py b/setup.py index 515ee367..2ff64d0f 100755 --- a/setup.py +++ b/setup.py @@ -1,61 +1,6 @@ #!/usr/bin/env python -import setuptools -import sys +"""Setup script for the 'xdis' distribution.""" -"""Setup script for the 'uncompyle6' distribution.""" +from setuptools import setup -SYS_VERSION = sys.version_info[0:2] -if not ((2, 4) <= SYS_VERSION < (3, 13)): - mess = "Python Release 2.6 .. 3.12 are supported in this code branch." - if (2, 4) <= SYS_VERSION <= (2, 7): - mess += ( - "\nFor your Python, version %s, use the python-2.4 code/branch." - % sys.version[0:3] - ) - if (3, 3) <= SYS_VERSION < (3, 6): - mess += ( - "\nFor your Python, version %s, use the python-3.3-to-3.5 code/branch." - % sys.version[0:3] - ) - elif SYS_VERSION < (2, 4): - mess += ( - "\nThis package is not supported for Python version %s." % sys.version[0:3] - ) - print(mess) - raise Exception(mess) - -from __pkginfo__ import ( - author, - author_email, - install_requires, - license, - long_description, - classifiers, - entry_points, - modname, - py_modules, - short_desc, - __version__, - web, - zip_safe, -) - -setuptools.setup( - author=author, - author_email=author_email, - classifiers=classifiers, - description=short_desc, - entry_points=entry_points, - install_requires=install_requires, - license=license, - long_description=long_description, - long_description_content_type="text/x-rst", - name=modname, - packages=setuptools.find_packages(), - py_modules=py_modules, - test_suite="nose.collector", - url=web, - tests_require=["nose>=1.0"], - version=__version__, - zip_safe=zip_safe, -) +setup(packages=["uncompyle6"]) diff --git a/uncompyle6/version.py b/uncompyle6/version.py index 03e22cc7..669350ed 100644 --- a/uncompyle6/version.py +++ b/uncompyle6/version.py @@ -14,4 +14,4 @@ # This file is suitable for sourcing inside POSIX shell as # well as importing into Python # fmt: off -__version__="3.9.1.dev0" # noqa +__version__="3.9.1" # noqa