Merge branch 'master' into python-3.6-to-3.10

Fix when parsing NOP

.circleci/config.yml

@@ -64,7 +64,7 @@ jobs:
       # Test
       #   This would typically be a build job when using workflows, possibly combined with build
       # This is based on your 1.0 configuration file or project settings
-      - run: sudo python ./setup.py develop && make check-3.6
+      - run: sudo pip install -e . && make check-3.6
       - run: cd ./test/stdlib && bash ./runtests.sh 'test_[p-z]*.py'
       # Teardown
       #   If you break your build into multiple jobs with workflows, you will probably want to do the parts of this that are relevant in each

.pre-commit-config.yaml

@@ -6,17 +6,17 @@ repos:
     hooks:
     - id: check-merge-conflict
     - id: debug-statements
-      stages: [commit]
+      stages: [pre-commit]
     - id: end-of-file-fixer
-      stages: [commit]
+      stages: [pre-commit]
 -   repo: https://github.com/pycqa/isort
     rev: 5.13.2
     hooks:
       - id: isort
-        stages: [commit]
+        stages: [pre-commit]
 -   repo: https://github.com/psf/black
     rev: 23.12.1
     hooks:
     - id: black
       language_version: python3
-      stages: [commit]
+      stages: [pre-commit]

README.rst

@@ -282,6 +282,7 @@ to spend.
 See Also
 --------
 
+* https://rocky.github.io/blackhat-asia-2024-additional/all-notes-print.html :  How to Read and Write a High-Level Bytecode Decompiler: ``uncompyle6`` ``decompyle3`` -- BlackHat 2024 Asia (`video <https://www.youtube.com/watch?v=NA77SFncppE>`_). A big thanks to the Organizers and Reviewers for letting me speak. This kind of thing encourages me to work on projects like this.
 * https://github.com/rocky/python-decompile3 : Much smaller and more modern code, focusing on 3.7 and 3.8. Changes in that will get migrated back here.
 * https://code.google.com/archive/p/unpyc3/ : supports Python 3.2 only. The above projects use a different decompiling technique than what is used here. Currently unmaintained.
 * https://github.com/figment/unpyc3/ : fork of above, but supports Python 3.3 only. Includes some fixes like supporting function annotations. Currently unmaintained.
@@ -306,8 +307,8 @@ See Also
 .. _uncompyle2: https://github.com/wibiti/uncompyle2
 .. _unpyc37: https://github.com/andrew-tavera/unpyc37
 .. _this: https://github.com/rocky/python-uncompyle6/wiki/Deparsing-technology-and-its-use-in-exact-location-reporting
-.. |buildstatus| image:: https://travis-ci.org/rocky/python-uncompyle6.svg
-		 :target: https://travis-ci.org/rocky/python-uncompyle6
+.. |buildstatus| image:: https://circleci.com/gh/rocky/python-uncompyle6.svg?style=svg
+		 :target: https://app.circleci.com/pipelines/github/rocky/python-uncompyle6
 .. |packagestatus| image:: https://repology.org/badge/vertical-allrepos/python:uncompyle6.svg
 		 :target: https://repology.org/project/python:uncompyle6/versions
 .. _PJOrion: http://www.koreanrandom.com/forum/topic/15280-pjorion-%D1%80%D0%B5%D0%B4%D0%B0%D0%BA%D1%82%D0%B8%D1%80%D0%BE%D0%B2%D0%B0%D0%BD%D0%B8%D0%B5-%D0%BA%D0%BE%D0%BC%D0%BF%D0%B8%D0%BB%D1%8F%D1%86%D0%B8%D1%8F-%D0%B4%D0%B5%D0%BA%D0%BE%D0%BC%D0%BF%D0%B8%D0%BB%D1%8F%D1%86%D0%B8%D1%8F-%D0%BE%D0%B1%D1%84

__pkginfo__.py

@@ -79,7 +79,7 @@ entry_points = {
     ]
 }
 ftp_url = None
-install_requires = ["click", "spark-parser >= 1.8.9, < 1.9.0", "xdis >= 6.1.1, < 6.2.0"]
+install_requires = ["click", "spark-parser >= 1.8.9, < 1.9.2", "xdis >= 6.1.1, < 6.2.0"]
 
 license = "GPL3"
 mailing_list = "python-debugger@googlegroups.com"

admin-tools/.gitignore

@@ -0,0 +1 @@
+/.python-version

admin-tools/checkout_common.sh

@@ -0,0 +1,21 @@
+# Common checkout routine
+export PATH=$HOME/.pyenv/bin/pyenv:$PATH
+bs=${BASH_SOURCE[0]}
+mydir=$(dirname $bs)
+fulldir=$(readlink -f $mydir)
+
+function setup_version {
+    local repo=$1
+    version=$2
+    echo Running setup $version on $repo ...
+    (cd ../$repo && . ./admin-tools/setup-${version}.sh)
+    return $?
+}
+
+function checkout_finish {
+    branch=$1
+    cd $uncompyle6_owd
+    git checkout $branch && pyenv local $PYTHON_VERSION && git pull
+    rc=$?
+    return $rc
+}

admin-tools/merge-for-3.3.sh

@@ -2,6 +2,6 @@
 uncompyle6_merge_33_owd=$(pwd)
 cd $(dirname ${BASH_SOURCE[0]})
 if . ./setup-python-3.3.sh; then
-    git merge master
+    git merge python-3.6-to-3.10
 fi
 cd $uncompyle6_merge_33_owd

admin-tools/merge-for-3.6.sh

@@ -0,0 +1,7 @@
+#/bin/bash
+uncompyle6_merge_36_owd=$(pwd)
+cd $(dirname ${BASH_SOURCE[0]})
+if . ./setup-python-3.6.sh; then
+    git merge master
+fi
+cd $uncompyle6_merge_36_owd

admin-tools/setup-master.sh

@@ -1,32 +1,20 @@
 #!/bin/bash
 # Check out master branch and dependent development master branches
-PYTHON_VERSION=3.8.18
-
 bs=${BASH_SOURCE[0]}
 if [[ $0 == $bs ]] ; then
     echo "This script should be *sourced* rather than run directly through bash"
     exit 1
 fi
 
-function checkout_version {
-    local repo=$1
-    version=${2:-master}
-    echo Checking out $version on $repo ...
-    (cd ../$repo && git checkout $version && pyenv local $PYTHON_VERSION) && \
-	git pull
-    return $?
-}
-
-owd=$(pwd)
-
-export PATH=$HOME/.pyenv/bin/pyenv:$PATH
+PYTHON_VERSION=3.12
 
+uncompyle6_owd=$(pwd)
 mydir=$(dirname $bs)
 fulldir=$(readlink -f $mydir)
+cd $mydir
+. ./checkout_common.sh
 cd $fulldir/..
-(cd $fulldir/.. && checkout_version python-spark && checkout_version python-xdis &&
-     checkout_version python-uncompyle6)
-
-git pull
-rm -v */.python-version || true
-cd $owd
+(cd $fulldir/.. && \
+     setup_version python-spark master && \
+     setup_version python-xdis master )
+checkout_finish master

admin-tools/setup-python-2.4.sh

@@ -1,32 +1,23 @@
 #!/bin/bash
 # Check out python-2.4-to-2.7 and dependent development branches.
 
-PYTHON_VERSION=2.4.6
-
 bs=${BASH_SOURCE[0]}
 if [[ $0 == $bs ]] ; then
     echo "This script should be *sourced* rather than run directly through bash"
     exit 1
 fi
 
-function checkout_version {
-    local repo=$1
-    version=${2:-python-2.4-to-2.7}
-    echo Checking out $version on $repo ...
-    (cd ../$repo && git checkout $version && pyenv local $PYTHON_VERSION) && \
-	git pull
-    return $?
-}
-
-owd=$(pwd)
-
-export PATH=$HOME/.pyenv/bin/pyenv:$PATH
+PYTHON_VERSION=2.4
 
+uncompyle6_owd=$(pwd)
 mydir=$(dirname $bs)
 fulldir=$(readlink -f $mydir)
-(cd $fulldir/.. && checkout_version python-spark && checkout_version python-xdis python-2.4-to-2.7 &&
-     checkout_version python-uncompyle6)
+cd $mydir
+. ./checkout_common.sh
 
-git pull
-rm -v */.python-version || true
-cd $owd
+(cd $fulldir/.. && \
+     setup_version python-spark python-2.4 && \
+     setup_version python-xdis python-2.4)
+
+
+checkout_finish python-2.4-to-2.7

admin-tools/setup-python-3.0.sh

@@ -1,35 +1,20 @@
 #!/bin/bash
 # Check out python-3.0-to-3.2 and dependent development branches.
-
-PYTHON_VERSION=3.0.1
-
 bs=${BASH_SOURCE[0]}
 if [[ $0 == $bs ]] ; then
     echo "This script should be *sourced* rather than run directly through bash"
     exit 1
 fi
 
-# FIXME put some of the below in a common routine
-function checkout_version {
-    local repo=$1
-    version=${2:-python-3.0-to-3.2}
-    echo Checking out $version on $repo ...
-    (cd ../$repo && git checkout $version && pyenv local $PYTHON_VERSION) && \
-	git pull
-    return $?
-}
-
-owd=$(pwd)
-trap finish EXIT
-
-export PATH=$HOME/.pyenv/bin/pyenv:$PATH
+PYTHON_VERSION=3.0
 
+uncompyle6_owd=$(pwd)
 mydir=$(dirname $bs)
 fulldir=$(readlink -f $mydir)
-cd $fulldir/..
-(cd $fulldir/.. && checkout_version python-spark master && checkout_version python-xdis &&
-      checkout_version python-uncompyle6)
+cd $mydir
+. ./checkout_common.sh
+(cd $fulldir/.. && \
+     setup_version python-spark python-3.0 && \
+     setup_version python-xdis python-3.0)
 
-git pull
-rm -v */.python-version || true
-cd $owd
+checkout_finish python-3.0-to-3.2

admin-tools/setup-python-3.3.sh

@@ -1,34 +1,21 @@
 #!/bin/bash
 # Check out python-3.3-to-3.5 and dependent development branches.
-PYTHON_VERSION=3.3.7
-
 bs=${BASH_SOURCE[0]}
 if [[ $0 == $bs ]] ; then
     echo "This script should be *sourced* rather than run directly through bash"
     exit 1
 fi
 
-# FIXME put some of the below in a common routine
-function checkout_version {
-    local repo=$1
-    version=${2:-python-3.3-to-3.5}
-    echo Checking out $version on $repo ...
-    (cd ../$repo && git checkout $version && pyenv local $PYTHON_VERSION) && \
-	git pull
-    return $?
-}
-
-owd=$(pwd)
-
-export PATH=$HOME/.pyenv/bin/pyenv:$PATH
+PYTHON_VERSION=3.3
 
+uncompyle6_owd=$(pwd)
 mydir=$(dirname $bs)
+cd $mydir
 fulldir=$(readlink -f $mydir)
+. ./checkout_common.sh
 cd $fulldir/..
-(cd $fulldir/.. && checkout_version python-spark master && checkout_version python-xdis &&
-     checkout_version python-uncompyle6)
-rm -v */.python-version || true
+(cd $fulldir/.. && \
+     setup_version python-spark python-3.3 && \
+     setup_version python-xdis python-3.3 )
 
-git pull
-rm -v */.python-version || true
-cd $owd
+checkout_finish python-3.3-to-3.5

admin-tools/setup-python-3.6.sh

@@ -0,0 +1,21 @@
+#!/bin/bash
+# Check out python-3.6-to-3.10 and dependent development branches.
+bs=${BASH_SOURCE[0]}
+if [[ $0 == $bs ]] ; then
+    echo "This script should be *sourced* rather than run directly through bash"
+    exit 1
+fi
+
+PYTHON_VERSION=3.6
+
+uncompyle6_owd=$(pwd)
+mydir=$(dirname $bs)
+cd $mydir
+fulldir=$(readlink -f $mydir)
+. ./checkout_common.sh
+cd $fulldir/..
+(cd $fulldir/.. && \
+     setup_version python-spark python-3.6 && \
+     setup_version python-xdis python-3.6 )
+
+checkout_finish python-3.6-to-3.10

pyproject.toml

@@ -1,6 +1,7 @@
 [build-system]
 requires = [
-    "setuptools>=71.0.3",
+    "setuptools",
+    # "setuptools>=59.6.0", # for 3.6
 ]
 
 build-backend = "setuptools.build_meta"
@@ -14,7 +15,7 @@ name = "uncompyle6"
 description = "Python cross-version byte-code library and disassembler"
 dependencies = [
     "click",
-    "spark-parser >= 1.8.9, < 1.9.0",
+    "spark-parser >= 1.8.9, < 1.9.2",
     "xdis >= 6.1.0, < 6.2.0",
 ]
 readme = "README.rst"

setup.py

@@ -1,6 +1,65 @@
 #!/usr/bin/env python
 """Setup script for the 'uncompyle6' distribution."""
+import sys
 
-from setuptools import setup
+import setuptools
 
-setup(packages=["uncompyle6"])
+SYS_VERSION = sys.version_info[0:2]
+if not ((3, 6) <= SYS_VERSION < (3, 11)):
+    mess = "Python Release 3.6 .. 3.10 are supported in this code branch."
+    if (2, 4) <= SYS_VERSION <= (2, 7):
+        mess += (
+            "\nFor your Python, version %s, use the python-2.4 code/branch."
+            % sys.version[0:3]
+        )
+    elif SYS_VERSION >= (3, 10):
+        mess += (
+            "\nFor your Python, version %s, use the master code/branch."
+            % sys.version[0:3]
+        )
+    elif (3, 0) >= SYS_VERSION < (3, 3):
+        mess += (
+            "\nFor your Python, version %s, use the python-3.0-to-3.2 code/branch."
+            % sys.version[0:3]
+        )
+    elif SYS_VERSION < (2, 4):
+        mess += (
+            "\nThis package is not supported for Python version %s." % sys.version[0:3]
+        )
+    print(mess)
+    raise Exception(mess)
+
+from __pkginfo__ import (
+    __version__,
+    author,
+    author_email,
+    classifiers,
+    entry_points,
+    install_requires,
+    license,
+    long_description,
+    modname,
+    py_modules,
+    short_desc,
+    web,
+    zip_safe,
+)
+
+setuptools.setup(
+    author=author,
+    author_email=author_email,
+    classifiers=classifiers,
+    description=short_desc,
+    entry_points=entry_points,
+    install_requires=install_requires,
+    license=license,
+    long_description=long_description,
+    long_description_content_type="text/x-rst",
+    name=modname,
+    packages=setuptools.find_packages(),
+    py_modules=py_modules,
+    test_suite="nose.collector",
+    url=web,
+    version=__version__,
+    zip_safe=zip_safe,
+)

test/simple_source/stmts/16_bytestring_docstring.py

@@ -0,0 +1,45 @@
+"""Module docstring"""
+class A:
+    b"""Got \xe7\xfe Bytes?"""
+    assert __doc__ == b"""Got \xe7\xfe Bytes?"""
+
+    def class_func(self):
+       b"""Got \xe7\xfe Bytes?"""
+       assert __doc__ == """Module docstring"""
+
+class B:
+    """Got no Bytes?"""
+    assert __doc__ == """Got no Bytes?"""
+
+    def class_func(self):
+        """Got no Bytes?"""
+        assert __doc__ == """Module docstring"""
+
+def single_func():
+    """single docstring?"""
+    assert __doc__ == """Module docstring"""
+
+def single_byte_func():
+    b"""Got \xe7\xfe Bytes?"""
+    assert __doc__ == """Module docstring"""
+
+assert __doc__ == """Module docstring"""
+
+assert single_func.__doc__ == """single docstring?"""
+single_func()
+
+assert single_byte_func.__doc__ == b"""Got \xe7\xfe Bytes?"""
+single_byte_func()
+
+assert A.__doc__ == b"""Got \xe7\xfe Bytes?"""
+assert A.class_func.__doc__ == b"""Got \xe7\xfe Bytes?"""
+a = A()
+assert a.class_func.__doc__ == b"""Got \xe7\xfe Bytes?"""
+a.class_func()
+
+assert B.__doc__ == """Got no Bytes?"""
+assert B.class_func.__doc__ == """Got no Bytes?"""
+b = B()
+assert b.class_func.__doc__ == """Got no Bytes?"""
+b.class_func()
+

test/simple_source/stmts/16_no_bytestring_docstring.py

@@ -0,0 +1,45 @@
+"""Module docstring"""
+class A:
+    b"""Got \xe7\xfe Bytes?"""
+    assert __doc__ == """Module docstring"""
+
+    def class_func(self):
+       b"""Got \xe7\xfe Bytes?"""
+       assert __doc__ == """Module docstring"""
+
+class B:
+    """Got no Bytes?"""
+    assert __doc__ == """Got no Bytes?"""
+
+    def class_func(self):
+        """Got no Bytes?"""
+        assert __doc__ == """Module docstring"""
+
+def single_func():
+    """single docstring?"""
+    assert __doc__ == """Module docstring"""
+
+def single_byte_func():
+    b"""Got \xe7\xfe Bytes?"""
+    assert __doc__ == """Module docstring"""
+
+assert __doc__ == """Module docstring"""
+
+assert single_func.__doc__ == """single docstring?"""
+single_func()
+
+assert single_byte_func.__doc__ is None
+single_byte_func()
+
+assert A.__doc__ is None
+assert A.class_func.__doc__ is None
+a = A()
+assert a.class_func.__doc__ is None
+a.class_func()
+
+assert B.__doc__ == """Got no Bytes?"""
+assert B.class_func.__doc__ == """Got no Bytes?"""
+b = B()
+assert b.class_func.__doc__ == """Got no Bytes?"""
+b.class_func()
+

uncompyle6/bin/uncompile.py

@@ -30,7 +30,7 @@ def usage():
 # __doc__ = """
 # Usage:
 #   %s [OPTIONS]... [ FILE | DIR]...
-#   %s [--help | -h | --V | --version]
+#   %s [--help | --version]
 
 # Examples:
 #   %s      foo.pyc bar.pyc       # decompile foo.pyc, bar.pyc to stdout

uncompyle6/parser.py

@@ -391,6 +391,10 @@ class PythonParser(GenericASTBuilder):
 
         returns ::= return
         returns ::= _stmts return
+        
+        # NOP
+        stmt     ::= nop_stmt
+        nop_stmt ::= NOP
 
         """
         pass

uncompyle6/scanners/tok.py

@@ -1,4 +1,4 @@
-#  Copyright (c) 2016-2021, 2023 by Rocky Bernstein
+#  Copyright (c) 2016-2021, 2023-2024 by Rocky Bernstein
 #  Copyright (c) 2000-2002 by hartmut Goebel <h.goebel@crazy-compilers.com>
 #  Copyright (c) 1999 John Aycock
 #

uncompyle6/semantics/customize.py

@@ -1,4 +1,4 @@
-#  Copyright (c) 2018-2019, 2021-2022 by Rocky Bernstein
+#  Copyright (c) 2018-2019, 2021-2022 2024 by Rocky Bernstein
 #
 #  This program is free software: you can redistribute it and/or modify
 #  it under the terms of the GNU General Public License as published by
@@ -17,15 +17,15 @@
 """
 
 from uncompyle6.parsers.treenode import SyntaxTree
+from uncompyle6.scanners.tok import Token
 from uncompyle6.semantics.consts import (
     INDENT_PER_LEVEL,
     NO_PARENTHESIS_EVER,
     PRECEDENCE,
-    TABLE_R,
     TABLE_DIRECT,
+    TABLE_R,
 )
 from uncompyle6.semantics.helper import flatten_list
-from uncompyle6.scanners.tok import Token
 
 
 def customize_for_version(self, is_pypy, version):
@@ -34,7 +34,7 @@ def customize_for_version(self, is_pypy, version):
         # PyPy changes
         #######################
         # fmt: off
-        TABLE_DIRECT.update({
+        self.TABLE_DIRECT.update({
 
             "assert":           ("%|assert %c\n", 0),
             # This can happen as a result of an if transformation
@@ -114,7 +114,7 @@ def customize_for_version(self, is_pypy, version):
         ########################
         # Without PyPy
         #######################
-        TABLE_DIRECT.update(
+        self.TABLE_DIRECT.update(
             {
                 # "assert" and "assert_expr" are added via transform rules.
                 "assert": ("%|assert %c\n", 0),
@@ -133,23 +133,23 @@ def customize_for_version(self, is_pypy, version):
         )
     if version >= (3, 0):
         if version >= (3, 2):
-            TABLE_DIRECT.update(
+            self.TABLE_DIRECT.update(
                 {"del_deref_stmt": ("%|del %c\n", 0), "DELETE_DEREF": ("%{pattr}", 0)}
             )
         from uncompyle6.semantics.customize3 import customize_for_version3
 
         customize_for_version3(self, version)
     else:  # < 3.0
-        TABLE_DIRECT.update(
+        self.TABLE_DIRECT.update(
             {"except_cond3": ("%|except %c, %c:\n", (1, "expr"), (-2, "store"))}
         )
         if version <= (2, 6):
-            TABLE_DIRECT["testtrue_then"] = TABLE_DIRECT["testtrue"]
+            self.TABLE_DIRECT["testtrue_then"] = self.TABLE_DIRECT["testtrue"]
 
         if (2, 4) <= version <= (2, 6):
-            TABLE_DIRECT.update({"comp_for": (" for %c in %c", 3, 1)})
+            self.TABLE_DIRECT.update({"comp_for": (" for %c in %c", 3, 1)})
         else:
-            TABLE_DIRECT.update({"comp_for": (" for %c in %c%c", 2, 0, 3)})
+            self.TABLE_DIRECT.update({"comp_for": (" for %c in %c%c", 2, 0, 3)})
 
         if version >= (2, 5):
             from uncompyle6.semantics.customize25 import customize_for_version25
@@ -197,7 +197,7 @@ def customize_for_version(self, is_pypy, version):
                     )
                 ],
             )
-            TABLE_DIRECT.update(
+            self.TABLE_DIRECT.update(
                 {
                     "importmultiple": ("%|import %c%c\n", 2, 3),
                     "import_cont": (", %c", 2),
@@ -247,9 +247,9 @@ def customize_for_version(self, is_pypy, version):
                 self.n_call = n_call
 
             else:  # 1.0 <= version <= 2.3:
-                TABLE_DIRECT.update({"if1_stmt": ("%|if 1\n%+%c%-", 5)})
+                self.TABLE_DIRECT.update({"if1_stmt": ("%|if 1\n%+%c%-", 5)})
                 if version <= (2, 1):
-                    TABLE_DIRECT.update(
+                    self.TABLE_DIRECT.update(
                         {
                             "importmultiple": ("%c", 2),
                             # FIXME: not quite right. We have indiividual imports
@@ -263,7 +263,7 @@ def customize_for_version(self, is_pypy, version):
 
         # < 3.0 continues
 
-        TABLE_R.update(
+        self.TABLE_R.update(
             {
                 "STORE_SLICE+0": ("%c[:]", 0),
                 "STORE_SLICE+1": ("%c[%p:]", 0, (1, -1)),
@@ -275,7 +275,7 @@ def customize_for_version(self, is_pypy, version):
                 "DELETE_SLICE+3": ("%|del %c[%c:%c]\n", 0, 1, 2),
             }
         )
-        TABLE_DIRECT.update({"raise_stmt2": ("%|raise %c, %c\n", 0, 1)})
+        self.TABLE_DIRECT.update({"raise_stmt2": ("%|raise %c, %c\n", 0, 1)})
 
         # exec as a built-in statement is only in Python 2.x
         def n_exec_stmt(node):

uncompyle6/semantics/customize14.py

@@ -1,4 +1,4 @@
-#  Copyright (c) 2022 by Rocky Bernstein
+#  Copyright (c) 2022, 2024 by Rocky Bernstein
 #
 #  This program is free software: you can redistribute it and/or modify
 #  it under the terms of the GNU General Public License as published by
@@ -15,16 +15,13 @@
 """Isolate Python 1.4- version-specific semantic actions here.
 """
 
-from uncompyle6.semantics.consts import TABLE_DIRECT
 
 #######################
 # Python 1.4- Changes #
 #######################
-def customize_for_version14(self, version):
-    TABLE_DIRECT.update(
+def customize_for_version14(self, version: tuple):
+    self.TABLE_DIRECT.update(
         {
-            "print_expr_stmt": (
-                ("%|print %c\n", 0)
-            ),
+            "print_expr_stmt": (("%|print %c\n", 0)),
         }
     )

uncompyle6/semantics/customize25.py

@@ -25,7 +25,7 @@ def customize_for_version25(self, version):
     ########################
     # Import style for 2.5+
     ########################
-    TABLE_DIRECT.update(
+    self.TABLE_DIRECT.update(
         {
             "importmultiple": ("%|import %c%c\n", 2, 3),
             "import_cont": (", %c", 2),

uncompyle6/semantics/customize26_27.py

@@ -1,4 +1,4 @@
-#  Copyright (c) 2019 2021 by Rocky Bernstein
+#  Copyright (c) 2019 2021, 2024 by Rocky Bernstein
 #
 #  This program is free software: you can redistribute it and/or modify
 #  it under the terms of the GNU General Public License as published by
@@ -17,8 +17,8 @@
 
 from uncompyle6.semantics.consts import TABLE_DIRECT
 
-def customize_for_version26_27(self, version):
 
+def customize_for_version26_27(self, version: tuple):
     ########################################
     # Python 2.6+
     #    except <condition> as <var>
@@ -29,16 +29,20 @@ def customize_for_version26_27(self, version):
     # matches how we parse this in bytecode
     ########################################
     if version > (2, 6):
-        TABLE_DIRECT.update({
-            "except_cond2":	( "%|except %c as %c:\n", 1, 5 ),
-            # When a generator is a single parameter of a function,
-            # it doesn't need the surrounding parenethesis.
-            "call_generator": ('%c%P', 0, (1, -1, ', ', 100)),
-        })
+        self.TABLE_DIRECT.update(
+            {
+                "except_cond2": ("%|except %c as %c:\n", 1, 5),
+                # When a generator is a single parameter of a function,
+                # it doesn't need the surrounding parenethesis.
+                "call_generator": ("%c%P", 0, (1, -1, ", ", 100)),
+            }
+        )
     else:
-        TABLE_DIRECT.update({
-            'testtrue_then': ( 'not %p', (0, 22) ),
-        })
+        self.TABLE_DIRECT.update(
+            {
+                "testtrue_then": ("not %p", (0, 22)),
+            }
+        )
 
     # FIXME: this should be a transformation
     def n_call(node):
@@ -47,22 +51,24 @@ def customize_for_version26_27(self, version):
         for i in mapping[1:]:
             key = key[i]
             pass
-        if key.kind == 'CALL_FUNCTION_1':
+        if key.kind == "CALL_FUNCTION_1":
             # A function with one argument. If this is a generator,
             # no parenthesis is needed.
             args_node = node[-2]
-            if args_node == 'expr':
+            if args_node == "expr":
                 n = args_node[0]
-                if n == 'generator_exp':
-                    node.kind = 'call_generator'
+                if n == "generator_exp":
+                    node.kind = "call_generator"
                     pass
                 pass
 
         self.default(node)
+
     self.n_call = n_call
 
     def n_import_from(node):
         if node[0].pattr > 0:
             node[2].pattr = ("." * node[0].pattr) + node[2].pattr
         self.default(node)
+
     self.n_import_from = n_import_from

uncompyle6/semantics/customize3.py

@@ -29,8 +29,8 @@ from uncompyle6.semantics.make_function3 import make_function3_annotate
 from uncompyle6.util import get_code_name
 
 
-def customize_for_version3(self, version):
-    TABLE_DIRECT.update(
+def customize_for_version3(self, version: tuple):
+    self.TABLE_DIRECT.update(
         {
             "comp_for": (" for %c in %c", (2, "store"), (0, "expr")),
             "if_exp_not": (
@@ -183,7 +183,7 @@ def customize_for_version3(self, version):
         # the iteration variable. These rules we can ignore
         # since we pick up the iteration variable some other way and
         # we definitely don't include in the source  _[dd].
-        TABLE_DIRECT.update(
+        self.TABLE_DIRECT.update(
             {
                 "ifstmt30": (
                     "%|if %c:\n%+%c%-",
@@ -335,7 +335,7 @@ def customize_for_version3(self, version):
 
     self.n_mkfunc_annotate = n_mkfunc_annotate
 
-    TABLE_DIRECT.update(
+    self.TABLE_DIRECT.update(
         {
             "tryelsestmtl3": (
                 "%|try:\n%+%c%-%c%|else:\n%+%c%-",
@@ -350,7 +350,7 @@ def customize_for_version3(self, version):
         #######################
         # Python 3.4+ Changes #
         #######################
-        TABLE_DIRECT.update(
+        self.TABLE_DIRECT.update(
             {
                 "LOAD_CLASSDEREF": ("%{pattr}",),
                 "yield_from": ("yield from %c", (0, "expr")),

uncompyle6/semantics/customize35.py

@@ -1,4 +1,4 @@
-#  Copyright (c) 2019-2020, 2022 by Rocky Bernstein
+#  Copyright (c) 2019-2020, 2022, 2024 by Rocky Bernstein
 #
 #  This program is free software: you can redistribute it and/or modify
 #  it under the terms of the GNU General Public License as published by
@@ -16,21 +16,17 @@
 """
 
 from xdis import co_flags_is_async, iscode
-from uncompyle6.semantics.consts import (
-    INDENT_PER_LEVEL,
-    PRECEDENCE,
-    TABLE_DIRECT,
-)
 
+from uncompyle6.semantics.consts import INDENT_PER_LEVEL, PRECEDENCE, TABLE_DIRECT
 from uncompyle6.semantics.helper import flatten_list, gen_function_parens_adjust
 
 
 #######################
 # Python 3.5+ Changes #
 #######################
-def customize_for_version35(self, version):
+def customize_for_version35(self, version: tuple):
     # fmt: off
-    TABLE_DIRECT.update(
+    self.TABLE_DIRECT.update(
         {
             # nested await expressions like:
             #   return await (await bar())
@@ -197,7 +193,11 @@ def customize_for_version35(self, version):
                 self.template_engine(template, args_node)
             else:
                 if len(node) - nargs > 3:
-                    template = ("*%c, %P)", nargs + 1, (nargs + kwargs + 1, -1, ", ", 100))
+                    template = (
+                        "*%c, %P)",
+                        nargs + 1,
+                        (nargs + kwargs + 1, -1, ", ", 100),
+                    )
                 else:
                     template = ("*%c)", nargs + 1)
                 self.template_engine(template, node)

uncompyle6/semantics/customize36.py

@@ -38,7 +38,7 @@ def escape_format(s):
 #######################
 
 
-def customize_for_version36(self, version):
+def customize_for_version36(self, version: tuple):
     # fmt: off
     PRECEDENCE["call_kw"]          =   0
     PRECEDENCE["call_kw36"]        =   1
@@ -50,7 +50,7 @@ def customize_for_version36(self, version):
     PRECEDENCE["dict_pack"]        =   0  # **{ ... }
     PRECEDENCE["formatted_value1"] = 100
 
-    TABLE_DIRECT.update(
+    self.TABLE_DIRECT.update(
         {
             "ann_assign_init_value": (
                 "%|%c = %p\n",
@@ -96,7 +96,7 @@ def customize_for_version36(self, version):
         }
     )
 
-    TABLE_R.update(
+    self.TABLE_R.update(
         {
             "CALL_FUNCTION_EX":    ("%c(*%P)", 0, (1, 2, ", ", 100)),
             # Not quite right

uncompyle6/semantics/customize37.py

@@ -1,4 +1,4 @@
-#  Copyright (c) 2019-2023 by Rocky Bernstein
+#  Copyright (c) 2019-2024 by Rocky Bernstein
 #
 #  This program is free software: you can redistribute it and/or modify
 #  it under the terms of the GNU General Public License as published by
@@ -17,7 +17,7 @@
 
 import re
 
-from uncompyle6.semantics.consts import INDENT_PER_LEVEL, PRECEDENCE, TABLE_DIRECT
+from uncompyle6.semantics.consts import INDENT_PER_LEVEL, PRECEDENCE
 from uncompyle6.semantics.helper import flatten_list
 
 # FIXME get from a newer xdis
@@ -25,7 +25,7 @@ FSTRING_CONVERSION_MAP = {1: "!s", 2: "!r", 3: "!a", "X": ":X"}
 
 
 #######################
-def customize_for_version37(self, version):
+def customize_for_version37(self, version: tuple):
     ########################
     # Python 3.7+ changes
     #######################
@@ -47,7 +47,7 @@ def customize_for_version37(self, version):
     PRECEDENCE["dict_unpack"]      =   0  # **{...}
 
     # fmt: on
-    TABLE_DIRECT.update(
+    self.TABLE_DIRECT.update(
         {
             "and_not": ("%c and not %c", (0, "expr"), (2, "expr")),
             "ann_assign": (

uncompyle6/semantics/customize38.py

@@ -1,4 +1,4 @@
-#  Copyright (c) 2019-2020, 2022 by Rocky Bernstein
+#  Copyright (c) 2019-2020, 2022, 2024 by Rocky Bernstein
 #
 #  This program is free software: you can redistribute it and/or modify
 #  it under the terms of the GNU General Public License as published by
@@ -24,13 +24,12 @@ from uncompyle6.semantics.customize37 import FSTRING_CONVERSION_MAP
 from uncompyle6.semantics.helper import escape_string, strip_quotes
 
 
-def customize_for_version38(self, version):
+def customize_for_version38(self, version: tuple):
     # FIXME: pytest doesn't add proper keys in testing. Reinstate after we have fixed pytest.
     # for lhs in 'for forelsestmt forelselaststmt '
     #             'forelselaststmtc tryfinally38'.split():
     #     del TABLE_DIRECT[lhs]
-
-    TABLE_DIRECT.update(
+    self.TABLE_DIRECT.update(
         {
             "async_for_stmt38": (
                 "%|async for %c in %c:\n%+%c%-%-\n\n",

uncompyle6/semantics/fragments.py

@@ -80,13 +80,13 @@ from uncompyle6.semantics import pysource
 from uncompyle6.semantics.check_ast import checker
 from uncompyle6.semantics.consts import (
     INDENT_PER_LEVEL,
-    MAP,
     NONE,
     PASS,
     PRECEDENCE,
     TABLE_DIRECT,
     escape,
 )
+from uncompyle6.semantics.helper import find_code_node
 from uncompyle6.semantics.pysource import (
     DEFAULT_DEBUG_OPTS,
     TREE_DEFAULT_DEBUG,
@@ -189,8 +189,7 @@ class FragmentsWalker(pysource.SourceWalker, object):
         self.is_pypy = is_pypy
 
         # FIXME: is there a better way?
-        global MAP_DIRECT_FRAGMENT
-        MAP_DIRECT_FRAGMENT = (dict(TABLE_DIRECT, **TABLE_DIRECT_FRAGMENT),)
+        self.MAP_DIRECT_FRAGMENT = (dict(TABLE_DIRECT, **TABLE_DIRECT_FRAGMENT),)
         return
 
     f = property(
@@ -597,17 +596,7 @@ class FragmentsWalker(pysource.SourceWalker, object):
     def n_mkfunc(self, node):
         start = len(self.f.getvalue())
 
-        if self.version >= (3, 3) or node[-2] == "kwargs":
-            # LOAD_CONST code object ..
-            # LOAD_CONST        'x0'  if >= 3.3
-            # MAKE_FUNCTION ..
-            code_node = node[-3]
-        elif node[-2] == "expr":
-            code_node = node[-2][0]
-        else:
-            # LOAD_CONST code object ..
-            # MAKE_FUNCTION ..
-            code_node = node[-2]
+        code_node = find_code_node(node, -2)
         func_name = code_node.attr.co_name
         self.write(func_name)
         self.set_pos_info(code_node, start, len(self.f.getvalue()))
@@ -655,6 +644,17 @@ class FragmentsWalker(pysource.SourceWalker, object):
 
         code = Code(cn.attr, self.scanner, self.currentclass)
         ast = self.build_ast(code._tokens, code._customize, code)
+
+        self.MAP_DIRECT = (self.TABLE_DIRECT,)
+        self.MAP_R = (self.TABLE_R, -1)
+
+        self.MAP = {
+            "stmt": self.MAP_R,
+            "call": self.MAP_R,
+            "delete": self.MAP_R,
+            "store": self.MAP_R,
+        }
+
         self.customize(code._customize)
 
         # Remove single reductions as in ("stmts", "sstmt"):
@@ -2003,8 +2003,7 @@ class FragmentsWalker(pysource.SourceWalker, object):
             self.set_pos_info(last_node, startnode_start, self.last_finish)
         return
 
-    @classmethod
-    def _get_mapping(cls, node):
+    def _get_mapping(self, node):
         if (
             hasattr(node, "data")
             and len(node) > 0
@@ -2012,7 +2011,7 @@ class FragmentsWalker(pysource.SourceWalker, object):
             and not hasattr(node[-1], "parent")
         ):
             node[-1].parent = node
-        return MAP.get(node, MAP_DIRECT_FRAGMENT)
+        return self.MAP.get(node, self.MAP_DIRECT_FRAGMENT)
 
     pass
 

uncompyle6/semantics/helper.py

@@ -152,6 +152,9 @@ def is_lambda_mode(compile_mode: str) -> bool:
 
 
 def print_docstring(self, indent, docstring):
+    if isinstance(docstring, bytes):
+        docstring = docstring.decode("utf8", errors="backslashreplace")
+
     quote = '"""'
     if docstring.find(quote) >= 0:
         if docstring.find("'''") == -1:

uncompyle6/semantics/n_actions.py

@@ -26,7 +26,7 @@ from uncompyle6.semantics.consts import (
     PRECEDENCE,
     minint,
 )
-from uncompyle6.semantics.helper import find_code_node, flatten_list
+from uncompyle6.semantics.helper import find_code_node, flatten_list, print_docstring
 from uncompyle6.util import better_repr, get_code_name
 
 
@@ -541,70 +541,7 @@ class NonterminalActions:
         else:
             docstring = node[0].pattr
 
-        quote = '"""'
-        if docstring.find(quote) >= 0:
-            if docstring.find("'''") == -1:
-                quote = "'''"
-
-        self.write(indent)
-        docstring = repr(docstring.expandtabs())[1:-1]
-
-        for orig, replace in (
-            ("\\\\", "\t"),
-            ("\\r\\n", "\n"),
-            ("\\n", "\n"),
-            ("\\r", "\n"),
-            ('\\"', '"'),
-            ("\\'", "'"),
-        ):
-            docstring = docstring.replace(orig, replace)
-
-        # Do a raw string if there are backslashes but no other escaped characters:
-        # also check some edge cases
-        if (
-            "\t" in docstring
-            and "\\" not in docstring
-            and len(docstring) >= 2
-            and docstring[-1] != "\t"
-            and (docstring[-1] != '"' or docstring[-2] == "\t")
-        ):
-            self.write("r")  # raw string
-            # Restore backslashes unescaped since raw
-            docstring = docstring.replace("\t", "\\")
-        else:
-            # Escape the last character if it is the same as the
-            # triple quote character.
-            quote1 = quote[-1]
-            if len(docstring) and docstring[-1] == quote1:
-                docstring = docstring[:-1] + "\\" + quote1
-
-            # Escape triple quote when needed
-            if quote == '"""':
-                replace_str = '\\"""'
-            else:
-                assert quote == "'''"
-                replace_str = "\\'''"
-
-            docstring = docstring.replace(quote, replace_str)
-            docstring = docstring.replace("\t", "\\\\")
-
-        lines = docstring.split("\n")
-
-        self.write(quote)
-        if len(lines) == 0:
-            self.println(quote)
-        elif len(lines) == 1:
-            self.println(lines[0], quote)
-        else:
-            self.println(lines[0])
-            for line in lines[1:-1]:
-                if line:
-                    self.println(line)
-                else:
-                    self.println("\n\n")
-                    pass
-                pass
-            self.println(lines[-1], quote)
+        print_docstring(self, indent, docstring)
         self.prune()
 
     def n_elifelsestmtr(self, node: SyntaxTree):

uncompyle6/semantics/pysource.py

@@ -146,8 +146,6 @@ from uncompyle6.semantics.consts import (
     ASSIGN_TUPLE_PARAM,
     INDENT_PER_LEVEL,
     LINE_LENGTH,
-    MAP,
-    MAP_DIRECT,
     NAME_MODULE,
     NO_PARENTHESIS_EVER,
     NONE,
@@ -156,6 +154,7 @@ from uncompyle6.semantics.consts import (
     RETURN_LOCALS,
     RETURN_NONE,
     TAB,
+    TABLE_DIRECT,
     TABLE_R,
     escape,
 )
@@ -316,7 +315,21 @@ class SourceWalker(GenericASTTraversal, NonterminalActions, ComprehensionMixin):
         # An example is:
         # __module__ = __name__
         self.hide_internal = True
+
+        self.TABLE_DIRECT = TABLE_DIRECT.copy()
+        self.TABLE_R = TABLE_R.copy()
+        self.MAP_DIRECT = (self.TABLE_DIRECT,)
+        self.MAP_R = (self.TABLE_R, -1)
+
+        self.MAP = {
+            "stmt": self.MAP_R,
+            "call": self.MAP_R,
+            "delete": self.MAP_R,
+            "store": self.MAP_R,
+        }
+
         customize_for_version(self, is_pypy, version)
+
         return
 
     def maybe_show_tree(self, tree, phase):
@@ -902,17 +915,17 @@ class SourceWalker(GenericASTTraversal, NonterminalActions, ComprehensionMixin):
         of arguments -- we add a new entry for each in TABLE_R.
         """
         for k, v in list(customize.items()):
-            if k in TABLE_R:
+            if k in self.TABLE_R:
                 continue
             op = k[: k.rfind("_")]
 
             if k.startswith("CALL_METHOD"):
                 # This happens in PyPy and Python 3.7+
-                TABLE_R[k] = ("%c(%P)", (0, "expr"), (1, -1, ", ", 100))
+                self.TABLE_R[k] = ("%c(%P)", (0, "expr"), (1, -1, ", ", 100))
             elif self.version >= (3, 6) and k.startswith("CALL_FUNCTION_KW"):
-                TABLE_R[k] = ("%c(%P)", (0, "expr"), (1, -1, ", ", 100))
+                self.TABLE_R[k] = ("%c(%P)", (0, "expr"), (1, -1, ", ", 100))
             elif op == "CALL_FUNCTION":
-                TABLE_R[k] = (
+                self.TABLE_R[k] = (
                     "%c(%P)",
                     (0, "expr"),
                     (1, -1, ", ", PRECEDENCE["yield"] - 1),
@@ -971,13 +984,13 @@ class SourceWalker(GenericASTTraversal, NonterminalActions, ComprehensionMixin):
                 else:
                     assert False, "Unhandled CALL_FUNCTION %s" % op
 
-                TABLE_R[k] = entry
+                self.TABLE_R[k] = entry
                 pass
             # handled by n_dict:
-            # if op == 'BUILD_SLICE':	TABLE_R[k] = ('%C'    ,    (0,-1,':'))
+            # if op == 'BUILD_SLICE':	self.TABLE_R[k] = ('%C'    ,    (0,-1,':'))
             # handled by n_list:
-            # if   op == 'BUILD_LIST':	TABLE_R[k] = ('[%C]'  ,    (0,-1,', '))
-            # elif op == 'BUILD_TUPLE':	TABLE_R[k] = ('(%C%,)',    (0,-1,', '))
+            # if   op == 'BUILD_LIST':	self.TABLE_R[k] = ('[%C]'  ,    (0,-1,', '))
+            # elif op == 'BUILD_TUPLE':	self.TABLE_R[k] = ('(%C%,)',    (0,-1,', '))
             pass
         return
 
@@ -1199,6 +1212,7 @@ class SourceWalker(GenericASTTraversal, NonterminalActions, ComprehensionMixin):
         is_lambda=False,
         noneInNames=False,
         is_top_level_module=False,
+        compile_mode="exec",
     ) -> GenericASTTraversal:
         # FIXME: DRY with fragments.py
 
@@ -1238,11 +1252,20 @@ class SourceWalker(GenericASTTraversal, NonterminalActions, ComprehensionMixin):
                 if tokens[-1].kind in ("RETURN_VALUE", "RETURN_VALUE_LAMBDA"):
                     # Python 3.4's classes can add a "return None" which is
                     # invalid syntax.
-                    if tokens[-2].kind == "LOAD_CONST":
-                        if is_top_level_module or tokens[-2].pattr is None:
-                            del tokens[-2:]
-                        else:
-                            tokens.append(Token("RETURN_LAST"))
+                    load_const = tokens[-2]
+                    # We should have:
+                    #   LOAD_CONST None
+                    # with *no* line number associated the token.
+                    # A line number on the token or a non-None
+                    # token value a token based on user source
+                    # text.
+                    if (
+                        load_const.kind == "LOAD_CONST"
+                        and load_const.linestart is None
+                        and load_const.attr is None
+                    ):
+                        # Delete LOAD_CONST (None) RETURN_VALUE
+                        del tokens[-2:]
                     else:
                         tokens.append(Token("RETURN_LAST"))
             if len(tokens) == 0:
@@ -1272,9 +1295,8 @@ class SourceWalker(GenericASTTraversal, NonterminalActions, ComprehensionMixin):
         del ast  # Save memory
         return transform_tree
 
-    @classmethod
-    def _get_mapping(cls, node):
-        return MAP.get(node, MAP_DIRECT)
+    def _get_mapping(self, node):
+        return self.MAP.get(node, self.MAP_DIRECT)
 
 
 def code_deparse(
@@ -1349,6 +1371,7 @@ def code_deparse(
         co,
         is_lambda=is_lambda_mode(compile_mode),
         is_top_level_module=is_top_level_module,
+        compile_mode=compile_mode,
     )
 
     # XXX workaround for profiling

uncompyle6/version.py

@@ -14,4 +14,4 @@
 # This file is suitable for sourcing inside POSIX shell as
 # well as importing into Python
 # fmt: off
-__version__="3.9.2"  # noqa
+__version__="3.9.3.dev0"  # noqa