Remove setuptools pin

Change copyright year to 2018

.circleci/config.yml

@@ -1,12 +1,7 @@
-# This configuration was automatically generated from a CircleCI 1.0 config.
-# It should include any build commands you had along with commands that CircleCI
-# inferred from your project structure. We strongly recommend you read all the
-# comments in this file to understand the structure of CircleCI 2.0, as the idiom
-# for configuration has changed substantially in 2.0 to allow arbitrary jobs rather
-# than the prescribed lifecycle of 1.0. In general, we recommend using this generated
-# configuration as a reference rather than using it in production, though in most
-# cases it should duplicate the execution of your original 1.0 config.
 version: 2
+filters:
+  branches:
+    only: master
 jobs:
   build:
     working_directory: ~/rocky/python-uncompyle6
@@ -18,72 +13,68 @@ jobs:
       CIRCLE_ARTIFACTS: /tmp/circleci-artifacts
       CIRCLE_TEST_REPORTS: /tmp/circleci-test-results
       COMPILE: --compile
-    # In CircleCI 1.0 we used a pre-configured image with a large number of languages and other packages.
-    # In CircleCI 2.0 you can now specify your own image, or use one of our pre-configured images.
-    # The following configuration line tells CircleCI to use the specified docker image as the runtime environment for you job.
-    # We have selected a pre-built image that mirrors the build environment we use on
-    # the 1.0 platform, but we recommend you choose an image more tailored to the needs
-    # of each job. For more information on choosing an image (or alternatively using a
-    # VM instead of a container) see https://circleci.com/docs/2.0/executor-types/
     # To see the list of pre-built images that CircleCI provides for most common languages see
     # https://circleci.com/docs/2.0/circleci-images/
     docker:
-    - image: circleci/build-image:ubuntu-14.04-XXL-upstart-1189-5614f37
-      command: /sbin/init
+      - image: circleci/python:3.8
     steps:
-    # Machine Setup
-    #   If you break your build into multiple jobs with workflows, you will probably want to do the parts of this that are relevant in each
-    # The following `checkout` command checks out your code to your working directory. In 1.0 we did this implicitly. In 2.0 you can choose where in the course of a job your code should be checked out.
-    - checkout
-    # Prepare for artifact and test results  collection equivalent to how it was done on 1.0.
-    # In many cases you can simplify this from what is generated here.
-    # 'See docs on artifact collection here https://circleci.com/docs/2.0/artifacts/'
-    - run: mkdir -p $CIRCLE_ARTIFACTS $CIRCLE_TEST_REPORTS
-    # This is based on your 1.0 configuration file or project settings
-    - run:
-        working_directory: ~/rocky/python-uncompyle6
-        command: pyenv local 2.7.11 && pyenv rehash && pip install virtualenv && pip install nose && pip install pep8 && pip install six && pyenv rehash
-    # Dependencies
-    #   This would typically go in either a build or a build-and-test job when using workflows
-    # Restore the dependency cache
-    - restore_cache:
-        keys:
-        # This branch if available
-        - v1-dep-{{ .Branch }}-
-        # Default branch if not
-        - v1-dep-master-
-        # Any branch if there are none on the default branch - this should be unnecessary if you have your default branch configured correctly
-        - v1-dep-
-    # This is based on your 1.0 configuration file or project settings
-    - run: pip install --upgrade setuptools
-    - run: pip install -e .
-    - run: pip install pytest==3.2.5 hypothesis==3.0.0
-    # Save dependency cache
-    - save_cache:
-        key: v1-dep-{{ .Branch }}-{{ epoch }}
-        paths:
-        # This is a broad list of cache paths to include many possible development environments
-        # You can probably delete some of these entries
-        - vendor/bundle
-        - ~/virtualenvs
-        - ~/.m2
-        - ~/.ivy2
-        - ~/.bundle
-        - ~/.go_workspace
-        - ~/.gradle
-        - ~/.cache/bower
-    # Test
-    #   This would typically be a build job when using workflows, possibly combined with build
-    # This is based on your 1.0 configuration file or project settings
-    - run: python ./setup.py develop && make check-2.7
-    - run: cd ./test/stdlib && pyenv local 2.7.11 && bash ./runtests.sh 'test_[p-z]*.py'
-    # Teardown
-    #   If you break your build into multiple jobs with workflows, you will probably want to do the parts of this that are relevant in each
-    # Save test results
-    - store_test_results:
-        path: /tmp/circleci-test-results
-    # Save artifacts
-    - store_artifacts:
-        path: /tmp/circleci-artifacts
-    - store_artifacts:
-        path: /tmp/circleci-test-results
+      # Machine Setup
+      #   If you break your build into multiple jobs with workflows, you will probably want to do the parts of this that are relevant in each
+      # The following `checkout` command checks out your code to your working directory. In 1.0 we did this implicitly. In 2.0 you can choose where in the course of a job your code should be checked out.
+      - checkout
+      # Prepare for artifact and test results  collection equivalent to how it was done on 1.0.
+      # In many cases you can simplify this from what is generated here.
+      # 'See docs on artifact collection here https://circleci.com/docs/2.0/artifacts/'
+      - run: mkdir -p $CIRCLE_ARTIFACTS $CIRCLE_TEST_REPORTS
+      # This is based on your 1.0 configuration file or project settings
+      - run:
+          working_directory: ~/rocky/python-uncompyle6
+          command: pip install --user virtualenv && pip install --user nose && pip install
+            --user pep8
+      # Dependencies
+      #   This would typically go in either a build or a build-and-test job when using workflows
+      # Restore the dependency cache
+      - restore_cache:
+          keys:
+            - v2-dependencies-{{ .Branch }}-
+            # fallback to using the latest cache if no exact match is found
+            - v2-dependencies-
+
+      - run:
+          command: | # Use pip to install dependengcies
+            sudo pip install --user --upgrade setuptools
+            pip install --user -e .
+            # Not sure why "pip install -e" doesn't work above
+            # pip install click spark-parser xdis
+            pip install --user -r requirements-dev.txt
+
+      # Save dependency cache
+      - save_cache:
+          key: v2-dependencies-{{ .Branch }}-{{ epoch }}
+          paths:
+            # This is a broad list of cache paths to include many possible development environments
+            # You can probably delete some of these entries
+            - vendor/bundle
+            - ~/virtualenvs
+            - ~/.m2
+            - ~/.ivy2
+            - ~/.bundle
+            - ~/.cache/bower
+
+      # Test
+      #   This would typically be a build job when using workflows, possibly combined with build
+      # This is based on your 1.0 configuration file or project settings
+      - run: sudo pip install -e . && make check-3.6
+      - run: cd ./test/stdlib && bash ./runtests.sh 'test_[p-z]*.py'
+      # Teardown
+      #   If you break your build into multiple jobs with workflows, you will probably want to do the parts of this that are relevant in each
+      # Save test results
+      - store_test_results:
+          path: /tmp/circleci-test-results
+      # Save artifacts
+      - store_artifacts:
+          path: /tmp/circleci-artifacts
+      - store_artifacts:
+          path: /tmp/circleci-test-results
+    # The resource_class feature allows configuring CPU and RAM resources for each job. Different resource classes are available for different executors. https://circleci.com/docs/2.0/configuration-reference/#resourceclass
+    resource_class: large

.editorconfig

@@ -0,0 +1,28 @@
+# THis is an EditorConfig file
+# https://EditorConfig.org
+
+root = true
+
+[*]
+end_of_line = lf
+insert_final_newline = true
+charset = utf-8
+indent_style = tab
+indent_size = 4
+insert_final_newline = true
+
+[*.yml]
+indent_style = space
+indent_size = 2
+end_of_line = lf
+insert_final_newline = true
+
+[*.py]
+indent_style = space
+indent_size = 4
+end_of_line = lf
+insert_final_newline = true
+
+# Tab indentation (no size specified)
+[Makefile]
+indent_style = tab

.github/FUNDING.yml

@@ -0,0 +1,12 @@
+# These are supported funding model platforms
+
+github: [rocky]
+patreon: # Replace with a single Patreon username
+open_collective: # Replace with a single Open Collective username
+ko_fi: # Replace with a single Ko-fi username
+tidelift: # Replace with a single Tidelift platform-name/package-name e.g., npm/babel
+community_bridge: # Replace with a single Community Bridge project-name e.g., cloud-foundry
+liberapay: rocky
+issuehunt: # Replace with a single IssueHunt username
+otechie: # Replace with a single Otechie username
+custom: # Replace with up to 4 custom sponsorship URLs e.g., ['link1', 'link2']

.github/ISSUE_TEMPLATE/bug-report.md

@@ -4,38 +4,78 @@ about: Tell us about uncompyle6 bugs
 
 ---
 
-<!-- __Note:__ Have you read https://github.com/rocky/python-uncompyle6/blob/master/HOW-TO-REPORT-A-BUG.md ?
+<!-- __Note:__ If you are using this program to do something illegal - don't.
+The issue may be flagged to make it easier for those looking for illegal activity.
 
+If you are reporting a bug in decompilation, it will probably not be acted upon
+unless it is narrowed to a small example. You may have to do some work remove
+extraneous code from the source example. Most bugs can be expressed in 30 lines of
+code.
+
+Issues are not for asking questions about a problem you
+are trying to solve that involve the use of uncompyle6 along the way,
+although I may be more tolerant of this if you sponsor the project.
+
+Bugs are also not for general or novice kind help on how to install
+this Python program and its dependencies in your environment, or in
+the way you would like to have it set up, or how to interpret a Python
+traceback e.g. that winds up saying Python X.Y.Z is not supported.
+
+For these kinds of things, you will save yourself time by asking
+instead on forums like StackOverflow that are geared to helping people
+for such general or novice kinds questions and tasks. And unless you
+are a sponsor of the project, if your question seems to be of this
+category, the issue may just be closed.
+
+Also, the unless you are a sponsor of the project, it may take a
+while, maybe a week or so, before the bug report is noticed, let alone
+acted upon.
+
+To set expectations, some legitimate bugs can take years to fix, but
+they eventually do get fixed.
+
+Funding the project was added to partially address the problem that there are
+lots of people seeking help and reporting bugs, but few people who are
+willing or capable of providing help or fixing bugs.
+
+Tasks or the kinds of things others can do, but you can't do or don't
+want to do yourself are typically the kind of thing that you pay
+someone to do, especially when you are the primary beneficiary of the
+work, or the task is complex, long, or tedious. If your code is over
+30 lines long, it fits into this category.
+
+
+See also https://github.com/rocky/python-uncomp[yle6/blob/master/HOW-TO-REPORT-A-BUG.md ?
+-->
+
+<!--
 Please remove any of the optional sections if they are not applicable.
 
-Prerequisites
+Prerequisites/Caveats
 
 * Make sure the bytecode you have can be disassembled with a
-  disassembler.
+  disassembler and produces valid results.
+* Try to make the bytecode that exhibits a bug as small as possible.
 * Don't put bytecode and corresponding source code on any service that
-  requires registration to download.
-* When you open a bug report there is no privacy. If the legitimacy of
-  the activity is deemed suspicous, I may flag it as suspicious,
+  requires registration to download. Instead attach it as a zip file.
+* When you open a bug report there is no privacy. If you need privacy, then
+  contact me by email and explain who you are and the need for privacy.
+  But be mindful that you may be asked to sponsor the project for the
+  personal and private help that you are requesting.
+* If the legitimacy of the activity is deemed suspicious, I may flag it as suspicious,
   making the issue even more easy to detect.
 
-Bug reports that violate a prerequisite may be discarded.
-
-Note that there are way more bug-fix requestors than there are bug
-fixers. If you want you need more immediate, confidential or urgent
-assistance
-
-http://www.crazy-compilers.com/decompyle/ offers a byte-code
-decompiler service for versions of Python up to 2.6.
+Bug reports that violate the above may be discarded.
 
 -->
 
 ## Description
 
-<!-- Add a clear and concise description of the bug. -->
+<!-- Please add a clear and concise description of the bug. Try to narrow the problem down to the smallest that exhibits the bug.-->
 
 ## How to Reproduce
 
-<!-- Please show both the input you gave and the
+<!-- Please show both the *input* you gave and the
 output you got in describing how to reproduce the bug:
 
 or give a complete console log with input and output
@@ -46,12 +86,22 @@ $ uncompyle6 <command-line-options>
 $
 ```
 
-Provide links to the Python bytecode. For example you can create a
+Attach a zip file to the Python bytecode or a
 gist with the information. If you have the correct source code, you
 can add that too.
 
 -->
 
+## Output Given
+
+<!--
+Please include not just the error message but all output leading to the message which includes echoing input and messages up to the error.
+For a command-line environment include command invocation and all the output produced.
+
+If this is too long, then try narrowing the problem to something short.
+-->
+
+
 ## Expected behavior
 
 <!-- Add a clear and concise description of what you expected to happen. -->
@@ -63,12 +113,25 @@ can add that too.
 Please modify for your setup
 
 - Uncompyle6 version: output from  `uncompyle6 --version` or `pip show uncompyle6`
-- Python version: `python -V`
+- xdis version: output from `pydisasm --version` or or `pip show xdis`
+- Python version for the version of Python the byte-compiled the file: `python -c "import sys; print(sys.version)"` where `python` is the correct CPython or PyPy binary.
 - OS and Version: [e.g. Ubuntu bionic]
 
 -->
 
-## Additional Environment or Context
+## Workarounds
+
+<!-- If there is a workaround for the problem, describe that here. -->
+
+## Priority
+
+<!-- If this is important for a particular public good state that here.
+     If this is blocking some important activity let us know what activity it blocks.
+
+	 Otherwise, we'll assume this has the lowest priority in addressing.
+	 -->
+
+## Additional Context
 
 <!-- _This section is optional._
 

.github/ISSUE_TEMPLATE/config.yml

@@ -0,0 +1 @@
+blank_issues_enabled: False

.github/workflows/osx.yml

@@ -0,0 +1,31 @@
+name: uncompyle6 (osx)
+
+on:
+  push:
+    branches: [ master ]
+  pull_request:
+    branches: [ master ]
+
+jobs:
+  build:
+    runs-on: macos-latest
+    strategy:
+      matrix:
+        os: [macOS]
+        python-version: [3.8]
+    steps:
+    - uses: actions/checkout@v2
+    - name: Set up Python ${{ matrix.python-version }}
+      uses: actions/setup-python@v2
+      with:
+        python-version: ${{ matrix.python-version }}
+    - name: Install dependencies
+      run: |
+        python -m pip install --upgrade pip
+        pip install -e .
+        # Not sure why "pip install -e" doesn't work above
+        # pip install click spark-parser xdis
+        pip install -r requirements-dev.txt
+    - name: Test uncompyle6
+      run: |
+        make check

.github/workflows/ubuntu.yml

@@ -0,0 +1,29 @@
+name: uncompyle6 (ubuntu)
+
+on:
+  push:
+    branches: [ master ]
+  pull_request:
+    branches: [ master ]
+
+jobs:
+  build:
+    runs-on: ubuntu-latest
+    strategy:
+      matrix:
+        python-version: [3.8]
+    steps:
+    - uses: actions/checkout@v2
+    - name: Set up Python ${{ matrix.python-version }}
+      uses: actions/setup-python@v2
+      with:
+        python-version: ${{ matrix.python-version }}
+    - name: Install dependencies
+      run: |
+        python -m pip install --upgrade pip
+        pip install -e .
+        # pip install click spark-parser xdis
+        pip install -r requirements-dev.txt
+    - name: Test uncompyle6
+      run: |
+        make check

.github/workflows/windows.yml

@@ -0,0 +1,31 @@
+name: uncompyle6 (windows)
+
+on:
+  push:
+    branches: [ master ]
+  pull_request:
+    branches: [ master ]
+
+jobs:
+  build:
+    runs-on: macos-latest
+    strategy:
+      matrix:
+        os: [windows]
+        python-version: [3.8]
+    steps:
+    - uses: actions/checkout@v2
+    - name: Set up Python ${{ matrix.python-version }}
+      uses: actions/setup-python@v2
+      with:
+        python-version: ${{ matrix.python-version }}
+    - name: Install dependencies
+      run: |
+        python -m pip install --upgrade pip
+        pip install -e .
+        # Not sure why "pip install -e" doesn't work above
+        # pip install click spark-parser xdis
+        pip install -r requirements-dev.txt
+    - name: Test uncompyle6
+      run: |
+        make check

.gitignore

@@ -2,10 +2,12 @@
 *.pyo
 *_dis
 *~
+.mypy_cache
 /.cache
 /.eggs
 /.hypothesis
 /.idea
+/.mypy_cache
 /.pytest_cache
 /.python-version
 /.tox
@@ -15,9 +17,11 @@
 /dist
 /how-to-make-a-release.txt
 /nose-*.egg
+/pycharm-venv
 /tmp
 /uncompyle6.egg-info
 /unpyc
+/venv
 ChangeLog
 __pycache__
 build

.isort.cfg

@@ -0,0 +1,11 @@
+[settings]
+multi_line_output = 3
+include_trailing_comma = True
+force_grid_wrap = 0
+use_parentheses = True
+line_length = 88
+known_crunch = cr, zz9d, zz9lib, pycrunch, silhouette
+sections = FUTURE,STDLIB,THIRDPARTY,FIRSTPARTY,CRUNCH,LOCALFOLDER
+default_section = THIRDPARTY
+combine_as_imports = 1
+profile = black

.pre-commit-config.yaml

@@ -0,0 +1,22 @@
+default_language_version:
+  python: python
+repos:
+-   repo: https://github.com/pre-commit/pre-commit-hooks
+    rev: v4.5.0
+    hooks:
+    - id: check-merge-conflict
+    - id: debug-statements
+      stages: [pre-commit]
+    - id: end-of-file-fixer
+      stages: [pre-commit]
+-   repo: https://github.com/pycqa/isort
+    rev: 5.13.2
+    hooks:
+      - id: isort
+        stages: [pre-commit]
+-   repo: https://github.com/psf/black
+    rev: 23.12.1
+    hooks:
+    - id: black
+      language_version: python3
+      stages: [pre-commit]

.travis.yml

@@ -1,11 +1,11 @@
 language: python
 
 python:
-  - '3.5'
-  - '2.7'
-  - '2.6'
-  - '3.4'
+#  - '3.5'
+#  - '2.7'
+#  - '3.4'
   - '3.6'
+  - '3.8'
 
 matrix:
   include:
@@ -13,6 +13,8 @@ matrix:
       dist: xenial  # required for Python >= 3.7 (travis-ci/travis-ci#9069)
 
 install:
+# Remove the next line when xdis 6.0.0 is released
+# - pip install git://github.com/rocky/python-xdis.git#egg=xdis
 - pip install -e .
 - pip install -r requirements-dev.txt
 

HISTORY.md

@@ -1,213 +1,93 @@
-This project has history of over 18 years spanning back to Python 1.5
+# Introduction
 
-There have been a number of people who have worked on this. I am awed
-by the amount of work, number of people who have contributed to this,
-and the cleverness in the code.
+This project started around 1999 spanning back to Python 1.5
 
-The below is an annotated history from talking to participants
-involved and my reading of the code and sources cited.
+In the interest of shortening what is written here, I am going to start where we left off where [decompyle 2.4's history](https://github.com/rocky/decompile-2.4/blob/master/HISTORY.md) ends.
 
-In 1998, John Aycock first wrote a grammar parser in Python,
-eventually called SPARK, that was usable inside a Python program. This
-code was described in the
-[7th International Python Conference](http://legacy.python.org/workshops/1998-11/proceedings/papers/aycock-little/aycock-little.html). That
-paper doesn't talk about decompilation, nor did John have that in mind
-at that time. It does mention that a full parser for Python (rather
-than the simple languages in the paper) was being considered.
+For the earlier history up to 2006 and the code up until Python 2.4, which I find interesting, look at that link.
 
-[This](http://pages.cpsc.ucalgary.ca/~aycock/spark/content.html#contributors)
-contains a of people acknowledged in developing SPARK. What's amazing
-about this code is that it is reasonably fast and has survived up to
-Python 3 with relatively little change. This work was done in
-conjunction with his Ph.D Thesis. This was finished around 2001. In
-working on his thesis, John realized SPARK could be used to deparse
-Python bytecode. In the fall of 1999, he started writing the Python
-program, "decompyle", to do this.
+Sometime around 2014 was the dawn of ["uncompyle" and PyPI](https://pypi.python.org/pypi/uncompyle/1.1) — the era of
+public version control. Dan Pascu's code although not public used [darcs](http://darcs.net/) for version control. I converted the darcs repository to git and put this at [decompyle-2.4](https://github.com/rocky/decompile-2.4).
 
-To help with control structure deparsing the instruction sequence was
-augmented with pseudo instruction COME_FROM. This code introduced
-another clever idea: using table-driven semantics routines, using
-format specifiers.
+# uncompyle, unpyc
 
-The last mention of a release of SPARK from John is around 2002. As
-released, although the Earley Algorithm parser was in good shape, this
-code was woefully lacking as serious Python deparser.
+In contrast to _decompyle_ that went up to Python 2.4, _uncompyle_, at least in its final versions, runs only on Python 2.7. However it accepts bytecode back to Python 2.5. Thomas Grainger is the package owner of this, although Hartmut is still listed as the author.
 
-In the fall of 2000, Hartmut Goebel
-[took over maintaining the code](https://groups.google.com/forum/#!searchin/comp.lang.python/hartmut$20goebel/comp.lang.python/35s3mp4-nuY/UZALti6ujnQJ). The
-first subsequent public release announcement that I can find is
-["decompyle - A byte-code-decompiler version 2.2 beta 1"](https://mail.python.org/pipermail/python-announce-list/2002-February/001272.html).
+The project exists not only on [github](https://github.com/gstarnberger/uncompyle) but also on
+[bitbucket](https://bitbucket.org/gstarnberger/uncompyle) and later the defunct [google
+code](https://code.google.com/archive/p/unpyc/) under the name _unpyc_. The git/svn history goes back to 2009. Somewhere in there the name was changed from "decompyle" to "unpyc" by Keknehv, and then to "uncompyle" by Guenther Starnberger.
 
-From the CHANGES file found in
-[the tarball for that release](http://old-releases.ubuntu.com/ubuntu/pool/universe/d/decompyle2.2/decompyle2.2_2.2beta1.orig.tar.gz),
-it appears that Hartmut did most of the work to get this code to
-accept the full Python language. He added precedence to the table
-specifiers, support for multiple versions of Python, the
-pretty-printing of docstrings, lists, and hashes. He also wrote test and verification routines of
-deparsed bytecode, and used this in an extensive set of tests that he also wrote. He says he could verify against the
-entire Python library. However I have subsequently found small and relatively obscure bugs in the decompilation code.
+The name Thomas Grainger isn't found in (m)any of the commits in the several years of active development. First Keknehv worked on this up to Python 2.5 or so while accepting Python bytecode back to 2.0 or so. Then "hamled" made a few commits earlier on, while Eike Siewertsen made a few commits later on. But mostly "wibiti", and Guenther Starnberger got the code to where uncompyle2 was around 2012.
 
-decompyle2.2 was packaged for Debian (sarge) by
-[Ben Burton around 2002](https://packages.qa.debian.org/d/decompyle.html). As
-it worked on Python 2.2 only long after Python 2.3 and 2.4 were in
-widespread use, it was removed.
+While John Aycock and Hartmut Goebel were well versed in compiler technology, those that have come afterwards don't seem to have been as facile in it.  Furthermore, documentation or guidance on how the decompiler code worked, comparison to a conventional compiler pipeline, how to add new constructs, or debug grammars was weak. Some of the grammar tracing and error reporting was a bit weak as well.
 
-[Crazy Compilers](http://www.crazy-compilers.com/decompyle/) offers a
-byte-code decompiler service for versions of Python up to 2.6. As
-someone who worked in compilers, it is tough to make a living by
-working on compilers. (For example, based on
-[John Aycock's recent papers](http://pages.cpsc.ucalgary.ca/~aycock/)
-it doesn't look like he's done anything compiler-wise since SPARK). So
-I hope people will use the crazy-compilers service. I wish them the
-success that his good work deserves.
+Given this, perhaps it is not surprising that subsequent changes tended to shy away from using the built-in compiler technology mechanisms and addressed problems and extensions by some other means.
 
-Dan Pascu did a bit of work from late 2004 to early 2006 to get this
-code to handle first Python 2.3 and then 2.4 bytecodes. Because of
-jump optimization introduced in the CPython bytecode compiler at that
-time, various JUMP instructions were classified to assist parsing For
-example, due to the way that code generation and line number table
-work, jump instructions to an earlier offset must be looping jumps,
-such as those found in a "continue" statement; "COME FROM"
-instructions were reintroduced.  See
-[RELEASE-2.4-CHANGELOG.txt](https://github.com/rocky/python-uncompyle6/blob/master/DECOMPYLE-2.4-CHANGELOG.txt)
-for more details here. There wasn't a public release of RELEASE-2.4
-and bytecodes other than Python 2.4 weren't supported. Dan says the
-Python 2.3 version could verify the entire Python library. But given
-subsequent bugs found like simply recognizing complex-number constants
-in bytecode, decompilation wasn't perfect.
+Specifically, in `uncompyle`, decompilation of python bytecode 2.5 & 2.6 is done by transforming the byte code into a pseudo-2.7 Python bytecode and is based on code from Eloi Vanderbeken. A bit of this could have been easily added by modifying grammar rules.
 
-Next we get to ["uncompyle" and
-PyPI](https://pypi.python.org/pypi/uncompyle/1.1) and the era of
-public version control. (Dan's code although not public used
-[darcs](http://darcs.net/) for version control.)
 
-In contrast to _decompyle_, _uncompyle_ at least in its final versions,
-runs only on Python 2.7. However it accepts bytecode back to Python
-2.5. Thomas Grainger is the package owner of this, although Hartmut is
-still listed as the author.
+# uncompyle2, uncompyle3, uncompyle6
 
-The project exists not only on
-[github](https://github.com/gstarnberger/uncompyle) but also on
-[bitbucket](https://bitbucket.org/gstarnberger/uncompyle) and later
-the defunct [google
-code](https://code.google.com/archive/p/unpyc/). The git/svn history
-goes back to 2009. Somewhere in there the name was changed from
-"decompyle" to "unpyc" by Keknehv, and then to "uncompyle" by Guenther Starnberger.
+`Uncompyle6`, which I started in 2015, owes its existence to the fork of [uncompyle2](https://github.com/Mysterie/uncompyle2) by Myst herie (Mysterie) whose first commit picks up at 2012. I chose this since it seemed to have been at that time the most actively, if briefly, worked on. Also starting around 2012 is Dark Fenx's [uncompyle3](https://github.com/DarkFenX/uncompyle3) which I used for inspiration for Python3 support.
 
-The name Thomas Grainger isn't found in (m)any of the commits in the
-several years of active development. First Keknehv worked on this up
-to Python 2.5 or so while acceping Python bytecode back to 2.0 or
-so. Then hamled made a few commits earler on, while Eike Siewertsen
-made a few commits later on. But mostly wibiti, and Guenther
-Starnberger got the code to where uncompyle2 was around 2012.
-
-While John Aycock and Hartmut Goebel were well versed in compiler
-technology, those that have come afterwards don't seem to have been as
-facile in it.  Furthermore, documentation or guidance on how the
-decompiler code worked, comparison to a conventional compiler
-pipeline, how to add new constructs, or debug grammars was weak. Some
-of the grammar tracing and error reporting was a bit weak as well.
-
-Given this, perhaps it is not surprising that subsequent changes
-tended to shy away from using the built-in compiler technology
-mechanisms and addressed problems and extensions by some other means.
-
-Specifically, in `uncompyle`, decompilation of python bytecode 2.5 &
-2.6 is done by transforming the byte code into a pseudo-2.7 Python
-bytecode and is based on code from Eloi Vanderbeken. A bit of this
-could have bene easily added by modifying grammar rules.
-
-This project, `uncompyle6`, abandons that approach for various
-reasons. Having a grammar per Python version is much cleaner and it
-scales indefinitely. That said, we don't have entire copies of the
-grammar, but work off of differences from some neighboring version.
-
-Should there be a desire to rebase or start a new base version to work
-off of, say for some future Python version, that can be done by
-dumping a grammar for a specific version after it has been loaded
-incrementally. You can get a full dump of the grammar by profiling the
-grammar on a large body of Python source code.
-
-Another problem with pseudo-2.7 bytecode is that that we need offsets
-in fragment deparsing to be exactly the same as the bytecode; the
-transformation process can remove instructions.  _Adding_ instructions
-with psuedo offsets is however okay.
-
-`Uncompyle6` however owes its existence to the fork of `uncompyle2` by
-Myst herie (Mysterie) whose first commit picks up at
-2012. I chose this since it seemed to have been at that time the most
-actively, if briefly, worked on. Also starting around 2012 is Dark
-Fenx's uncompyle3 which I used for inspiration for Python3 support.
-
-I started working on this late 2015, mostly to add fragment support.
-In that, I decided to make this runnable on Python 3.2+ and Python 2.6+
-while, handling Python bytecodes from Python versions 2.5+ and
-3.2+. In doing so, it has been expedient to separate this into three
-projects:
+I started working on this late 2015, mostly to add fragment support. In that, I decided to make this runnable on Python 3.2+ and Python 2.6+ while handling Python bytecodes from Python versions 2.5+ and
+3.2+. In doing so, it was expedient to separate this into three projects:
 
 * marshaling/unmarshaling, bytecode loading and disassembly ([xdis](https://pypi.python.org/pypi/xdis)),
 * parsing and tree building ([spark_parser](https://pypi.python.org/pypi/spark_parser)),
 * this project - grammar and semantic actions for decompiling
   ([uncompyle6](https://pypi.python.org/pypi/uncompyle6)).
 
+ `uncompyle6`, abandons the idea found in some 2.7 version of `uncompyle` that support Python 2.6 and 2.5 by trying to rewrite opcodes at the bytecode level.
 
-Over the many years, code styles and Python features have
-changed. However brilliant the code was and still is, it hasn't really
-had a single public active maintainer. And there have been many forks
-of the code.  I have spent a great deal of time trying to organize and
-modularize the code so that it can handle more Python versions more
-gracefully (with still only moderate success).
+Having a grammar per Python version is simpler to maintain, cleaner and it scales indefinitely.
 
-That it has been in need of an overhaul has been recognized by the
-Hartmut a decade an a half ago:
+Over the many years, code styles and Python features have changed. However brilliant the code was and still is, it hasn't really had a single public active maintainer. And there have been many forks of the code.
+
+That this code has been in need of an overhaul has been recognized by the Hartmut more than two decades ago.
 
 [decompyle/uncompile__init__.py](https://github.com/gstarnberger/uncompyle/blob/master/uncompyle/__init__.py#L25-L26)
 
     NB. This is not a masterpiece of software, but became more like a hack.
     Probably a complete rewrite would be sensefull. hG/2000-12-27
 
-This project deparses using an Earley-algorithm parse with lots of
-massaging of tokens and the grammar in the scanner
-phase. Earley-algorithm parsers are context free and tend to be linear
-if the grammar is LR or left recursive. There is a technique for
-improving LL right recursion, but our parser doesn't have that yet.
+In 2021, I created three git branches in order to allow the decompiler to run on a wide variety of Python versions from 2.4 up to 3.10. (Note this doesn't mean we decompile these versions. In fact we decompile starting from Python 1.0 up to Python 3.8 and no later than that.)
 
-Another approach to decompiling, and one that doesn't use grammars is
-to do something like simulate execution symbolically and build
-expression trees off of stack results. Control flow in that approach
-still needs to be handled somewhat ad hoc.  The two important projects
-that work this way are [unpyc3](https://code.google.com/p/unpyc3/) and
-most especially [pycdc](https://github.com/zrax/pycdc) The latter
-project is largely by Michael Hansen and Darryl Pogue. If they
-supported getting source-code fragments, did a better job in
-supporting Python more fully, and had a way I could call it from
-Python, I'd probably would have ditched this and used that. The code
-runs blindingly fast and spans all versions of Python, although more
-recently Python 3 support has been lagging. The code is impressive for
-its smallness given that it covers many versions of Python. However, I
-think it has reached a scalability issue, same as all the other
-efforts. To handle Python versions more accurately, I think that code
-base will need to have a lot more code specially which specializes for
-Python versions. And then it will run into a modularity problem.
+Using the separate git branches allows me to continually improve the coding style and add feature support while still supporting older Pythons. Supporting older Pythons is nice (but not strictly necessary) when you want to debug decompilation on older Pythons.
 
-Tests for the project have been, or are being, culled from all of the
-projects mentioned. Quite a few have been added to improve grammar
-coverage and to address the numerous bugs that have been encountered.
+I have spent a great deal of time trying to organize, modularize and even modernize the code so that it can handle more Python versions more gracefully (with still only moderate success).
 
-If you think, as I am sure will happen in the future, "hey, I can just
-write a decompiler from scratch and not have to deal with all all of
-the complexity here", think again. What is likely to happen is that
-you'll get at best a 90% solution working for a single Python release
-that will be obsolete in about a year, and more obsolete each
-subsequent year.  Writing a decompiler for Python gets harder as it
-Python progresses, so writing one for Python 3.7 isn't as easy as it
-was for Python 2.2.  That said, if you still feel you want to write a
-single version decompiler, look at the test cases in this project and
-talk to me. I may have some ideas.
+Tests for the project have been, or are being, culled from all of the projects mentioned above or below. Quite a few have been added to improve grammar coverage and to address the numerous bugs that have been encountered.
 
 
-For a little bit of the history of changes to the Earley-algorithm parser,
-see the file [NEW-FEATURES.rst](https://github.com/rocky/python-spark/blob/master/NEW-FEATURES.rst) in the [python-spark github repository](https://github.com/rocky/python-spark).
+# unpyc3 and pydc
 
-NB. If you find mistakes, want corrections, or want your name added
-(or removed), please contact me.
+Another approach to decompiling, and one that doesn't use grammars is to do something like simulate execution symbolically and build expression trees off of stack results. Control flow in that approach
+still needs to be handled somewhat ad hoc.  The two important projects that work this way are [unpyc3](https://code.google.com/p/unpyc3/) and most especially [pycdc](https://github.com/zrax/pycdc) The latter
+project is largely by Michael Hansen and Darryl Pogue. If they supported getting source-code fragments, did a better job in supporting Python more fully, and had a way I could call it from Python, I'd probably would have ditched this and used that. The code runs blindingly fast and spans all versions of Python, although more recently Python 3 support has been lagging. The code is impressive for its smallness given that it covers many versions of Python. However, I think it has reached a scalability issue, same as all the other efforts. To handle Python versions more accurately, I think that code base will need to have a lot more code specially which specializes for Python versions. And then it will run into a modularity problem.
+
+# So you want to write a decompiler for Python?
+
+If you think, as I am sure will happen in the future, "hey, I can just write a decompiler from scratch and not have to deal with all of the complexity in uncompyle6", think again. What is likely to happen is that you'll get at best a 90% solution working for a single Python release that will be obsolete in about a year, and more obsolete each subsequent year.
+
+Writing a decompiler for Python gets harder as it Python progresses. Writing decompiler for Python 3.7 isn't as easy as it was for Python 2.2. For one thing, now that Python has a well-established AST, that opens another interface by which code can be improved.
+
+In Python 3.10 I am seeing (for the first time?) bytecode getting moved around so that it is no longer the case that line numbers have to be strictly increasing as bytecode offsets increase. And I am seeing dead code appear as well.
+
+That said, if you still feel you want to write a single version decompiler, look at the test cases in this project and talk to me. I may have some ideas that I haven't made public yet. See also what I've written about the on how this code works and on [decompilation in dynamic runtime languages](http://rocky.github.io/Deparsing-Paper.pdf) in general.
+
+
+
+# Earley Algorithm Parser
+
+This project deparses using an Earley-algorithm parse. But in order to do this accurately, the process of tokenization is a bit more involved in the scanner. We don't just disassemble bytecode and use the opcode name. That aspect hasn't changed from the very first decompilers. However understanding _what_ information needs to be made explicit and what pseudo instructions to add that accomplish this has taken some time to understand.
+
+Earley-algorithm parsers have gotten negative press, most notably by the dragon book. Having used this a bit, I am convinced having a system that handles ambiguous grammars is the right thing to do and matches the problem well. In practice the speed of the parser isn't a problem when one understand what's up. And this has taken a little while to understand.
+Earley-algorithm parsers for context free languages or languages that are to a large extent context free and tend to be linear and the grammar steers towards left recursive rules. There is a technique for improving LL right recursion, but our parser doesn't have that yet.
+
+The [decompiling paper](http://rocky.github.io/Deparsing-Paper.pdf) discusses these aspects in a more detail.
+
+
+For a little bit of the history of changes to the Earley-algorithm parser, see the file [NEW-FEATURES.rst](https://github.com/rocky/python-spark/blob/master/NEW-FEATURES.rst) in the [python-spark github repository](https://github.com/rocky/python-spark).
+
+NB. If you find mistakes, want corrections, or want your name added (or removed), please contact me.

HOW-TO-REPORT-A-BUG.md

@@ -1,60 +1,66 @@
 <!-- markdown-toc start - Don't edit this section. Run M-x markdown-toc-refresh-toc -->
 **Table of Contents**
 
-- [The difficulty of the problem](#the-difficulty-of-the-problem)
+- [Ethics](#ethics)
+- [The importance of your bug report](#the-importance-of-your-bug-report)
+- [The difficulty of the problem and your bug](#the-difficulty-of-the-problem-and-your-bug)
 - [Is it really a bug?](#is-it-really-a-bug)
     - [Do you have valid bytecode?](#do-you-have-valid-bytecode)
     - [Semantic equivalence vs. exact source code](#semantic-equivalence-vs-exact-source-code)
 - [What to send (minimum requirements)](#what-to-send-minimum-requirements)
 - [What to send (additional helpful information)](#what-to-send-additional-helpful-information)
     - [But I don't *have* the source code!](#but-i-dont-have-the-source-code)
-    - [But I don't *have* the source code and am incapable of figuring how how to do a hand disassembly!](#but-i-dont-have-the-source-code-and-am-incapable-of-figuring-how-how-to-do-a-hand-disassembly)
+        - [But I don't *have* the source code and am incapable of figuring how to do a hand disassembly!](#but-i-dont-have-the-source-code-and-am-incapable-of-figuring-how-to-do-a-hand-disassembly)
 - [Narrowing the problem](#narrowing-the-problem)
 - [Karma](#karma)
 - [Confidentiality of Bug Reports](#confidentiality-of-bug-reports)
-- [Ethics](#ethics)
 
 <!-- markdown-toc end -->
-# The difficulty of the problem
+
+TL;DR (too long; didn't read)
+
+* Don't do something illegal. And don't ask me to do something illegal or help you do something illegal.
+* We already have an infinite supply of decompilation bugs that need fixing, and an automated mechanism for finding more. Decompilation bugs get addressed by easiness to fix and by whim. If you expect yours to be fixed ahead of those, you need to justify why. You can ask for a hand-assisted decompilation, but that is expensive and beyond what most are willing to spend. A $100 fee is needed just to look at the bytecode.
+* When asking for help, you may be asked for what you've tried on your own first. There are plenty of sources of information about this code.
+* Bugs get fixed, slowly. Sometimes on the order of months or years. If you are looking for *timely* help or support, that is typically known as a _paid_ service.
+* Submitting a bug or issue report that is likely to get acted upon may require a bit of effort on your part to make it easy for the problem solver. If you are not willing to do that, please don't waste your or our time. Bug report may be closed with about as much thought and care as apparent in the effort to create the bug. Supporting the project however, does increase the likelihood of your issue getting noticed and acted upon.
+
+# Ethics
+
+Do not use this program for unethical or illegal purposes. More detestable, at least to me, is asking for help to assist you in something that might not legitimate.
+
+Don't use the issue tracker for such unethical or illegal solicitations. To try to stave off illegitimate behavior, you should note that the issue tracker, the code, and bugs mentioned in that are in the open: there is no
+confidentiality. You may be asked about the authorship or claimed ownership of the bytecode. If I think something is not quite right, I may label the issue questionable which may make the it easier those who are looking for illegal activity.
+
+
+# The importance of your bug report
+
+For many open-source projects bugs where the expectation is that bugs are rare, reporting bugs in a *thoughtful* way can be helpful. See also [How to Ask Questions the Smart Way](http://www.catb.org/~esr/faqs/smart-questions.html).
+
+In this project though, most of the bug reports boil down to the something like: I am trying to reverse engineer some code that I am not the author/owner and that person doesn't want me to have access to. I am hitting a problem somewhere along the line which might have to do with decompilation. But it could be something else like how the bytecode was extracted, some problem in deliberately obfuscated code, or the use some kind of Python bytecode version that isn't supported by the decompiler. Gee this stuff is complicated, here's an open source project, so maybe someone there will help me figure stuff out.
+
+While you are free to report bugs, unless you sponsor the project, I may close them with about the same amount of effort spent that I think was used to open the report for them. And if you spent a considerable amount of time to create the bug report but didn't follow instructions given here and in the issue template, I am sorry in advance. Just go back, read, and follow instructions.
+
+This project already has an infinite supply of bugs that have been narrowed to the most minimal form and where I have source code to compare against. And in the unlikely event this supply runs out, I have automated means for generating *another* infinite supply.
+
+The task of justifying why addressing your bug is of use to the community, and why it should be prioritized over the others, is the bug reporter's responsibility.
+
+While in the abstract, I have no problem answering questions about how to read a Python traceback or install Python software, or trying to understand what is going wrong in your particular setup, I am not a paid support person and there other things I'd rather be doing with my limited volunteer time. So save us both time, effort, and aggravation: use other avenues like StackOverflow. Again, justifying why you should receive unpaid help is the help requester's responsibility.
+
+
+# The difficulty of the problem and your bug
 
 This decompiler is a constant work in progress: Python keeps
 changing, and so does its code generation.
 
-There is no Python decompiler yet that I know about that will
-decompile everything. Overall, I think this one probably does the best
-job of *any* Python decompiler that handles such a wide range of
-versions.
+There is no Python decompiler yet that I know about that will decompile everything. Overall, I think this one probably does the best job of *any* Python decompiler that handles such a wide range of versions.
 
-But at any given time, there are a number of valid Python bytecode
-files that I know of that will cause problems. See, for example, the
-list in
+But at any given time, there are a number of valid Python bytecode files that I know of that will cause problems. See, for example, the list in
 [`test/stdlib/runtests.sh`](https://github.com/rocky/python-uncompyle6/blob/master/test/stdlib/runtests.sh).
 
-But I understand: you would the bugs _you_ encounter addressed before
-all the other known bugs.
+There are far more bug reporters than there are bug fixers.
 
-From my standpoint, the good thing about the bugs listed in
-`runtests.sh` is that each test case is small and isolated to a single
-kind of problem. And I'll tend to fix easier, more isolated cases than
-generic "something's wrong" kinds of bugs where I'd have to do a bit
-of work to figure out what's up, if not use some sort of mind reading,
-make some guesses, and perform some experiments to see if the guesses
-are correct. I can't read minds, nor am I into guessing games; I'd
-rather devote the effort spent instead towards fixing bugs that are
-precisely defined.
-
-And it often turns out that by just fixing the well-defined and
-prescribed cases, the ill-defined amorphous cases as well will get
-handled as well.
-
-In sum, you may need to do some work to have the bug you have found
-handled before the hundreds of other bugs, and other things I could be
-doing.
-
-No one is getting paid to work to work on this project, let alone the
-bugs you may have an interest in. If you require decompiling bytecode
-immediately, consider using a decompilation service, listed further
-down in this document.
+Unless you are a sponsor of this project, it may take a while, maybe a week or so, before the bug report is noticed, let alone acted upon. Things eventually get fixed, but it may take years. And if your bug hasn't been narrowed, it might happen as a result of some other bug fix.
 
 # Is it really a bug?
 
@@ -62,28 +68,17 @@ down in this document.
 ## Do you have valid bytecode?
 
 As mentioned in README.rst, this project doesn't handle obfuscated
-code. See README.rst for suggestions for how to remove some kinds of
+code, release candidates, and the most recent versions of Python: version 3.9 and up. See README.rst for suggestions for how to remove some kinds of
 obfuscation.
 
 Checking if bytecode is valid is pretty simple: disassemble the code.
 Python comes with a disassembly module called `dis`. A prerequisite
 module for this package, `xdis` has a cross-python version
-disassembler called `pydisasm`.
+disassembler called `pydisasm`. Using that with the `-F extended` option, generally provides a more comprehensive disassembly than is provided by other disassemblers.
 
 ## Semantic equivalence vs. exact source code
 
-Consider how Python compiles something like "(x*y) + 5". Early on
-Python creates an "abstract syntax tree" (AST) for this. And this is
-"abstract" in the sense that unimportant, redundant or unnecessary
-items have been removed. Here, this means that any notion that you
-wrote "x+y" in parenthesis is lost, since in this context they are
-unneeded. Also lost is the fact that the multiplication didn't have
-spaces around it while the addition did. It should not come as a
-surprise then that the bytecode which is derived from the AST also has
-no notion of such possible variation. Generally this kind of thing
-isn't noticed since the Python community has laid out a very rigid set
-of formatting guidelines; and it has largely beaten the community into
-compliance.
+Consider how Python compiles something like "(x*y) + 5". Early on Python creates an "abstract syntax tree" (AST) for this. And this is "abstract" in the sense that unimportant, redundant or unnecessary items have been removed. Here, this means that any notion that you wrote "x+y" in parenthesis is lost, since in this context they are unneeded. Also lost is the fact that the multiplication didn't have spaces around it while the addition did. It should not come as a surprise then that the bytecode which is derived from the AST also has no notion of such possible variation. Generally this kind of thing isn't noticed since the Python community has laid out a very rigid set of formatting guidelines; and it has largely beaten the community into compliance.
 
 Almost all versions of Python can perform some sort of code
 improvement that can't be undone. In earlier versions of Python it is
@@ -144,8 +139,7 @@ if False:
 
 Python will eliminate the entire "if" statement.
 
-So just because the text isn't the same, does not
-necessarily mean there's a bug.
+So just because the text isn't the same, this does not necessarily mean there's a bug.
 
 # What to send (minimum requirements)
 
@@ -166,7 +160,7 @@ Also try to narrow the bug. See below.
 Some kind folks also give the invocation they used and the output
 which usually includes an error message produced. This is
 helpful. From this, I can figure out what OS you are running this on
-and what version of *uncomplye6* was used. Therefore, if you _don't_
+and what version of *uncompyle6* was used. Therefore, if you _don't_
 provide the input command and the output from that, please give:
 
 * _uncompyle6_ version used
@@ -176,30 +170,18 @@ provide the input command and the output from that, please give:
 
 ## But I don't *have* the source code!
 
-Sure, I get it. No problem. There is Python assembly code on parse
-errors, so simply by hand decompile that. To get a full disassembly,
-use `pydisasm` from the [xdis](https://pypi.python.org/pypi/xdis)
-package. Opcodes are described in the documentation for
-the [dis](https://docs.python.org/3.6/library/dis.html) module.
+There is Python assembly code on parse errors, so simply by hand decompile that. To get a full disassembly, use `pydisasm` from the [xdis](https://pypi.python.org/pypi/xdis) package. Opcodes are described in the documentation for the [dis](https://docs.python.org/3.6/library/dis.html) module.
 
 ### But I don't *have* the source code and am incapable of figuring how to do a hand disassembly!
 
-Well, you could learn. No one is born into this world knowing how to
-disassemble Python bytecode. And as Richard Feynman once said, "What
-one fool can learn, so can another."
+Well, you could learn. No one is born into this world knowing how to disassemble Python bytecode. And as Richard Feynman once said, "What one fool can learn, so can another."
 
-If this is too difficult, or too time consuming, or not of interest to
-you, then perhaps what require is a decompilation service. [Crazy
-Compilers](http://www.crazy-compilers.com/decompyle/) offers a
-byte-code decompiler service for versions of Python up to 2.6. (If
-there are others around let me know and I'll list them here.)
+If this is too difficult, or too time consuming, or not of interest to you, then you might consider [sponsoring](https://github.com/sponsors/rocky) the project. [Crazy
+Compilers](http://www.crazy-compilers.com/decompyle/) offers a byte-code decompiler service for versions of Python up to 2.6. (If there are others around let me know and I'll list them here.) Don't be surprised if I ask you to pay for work (if I think the work is ethical) when you want me to work on your problem that I think isn't of interest or benefit to anyone but yourself or a small limited number of people, or I think the need is questionable.
 
 # Narrowing the problem
 
-I don't need or want the entire source code base for the file(s) or
-module(s) can't be decompiled. I just need those file(s) or module(s).
-If there are problems in several files, file a bug report for each
-file.
+I don't need or want the entire source code base for the file(s) or module(s) can't be decompiled. I just need those file(s) or module(s). If there are problems in several files, file a bug report for each file.
 
 Python modules can get quite large, and usually decompilation problems
 occur in a single function or maybe the main-line code but not any of
@@ -217,33 +199,29 @@ likely the problem will be fixed and fixed sooner.
 # Karma
 
 I realize that following the instructions given herein puts a bit of
-burden on the bug reporter. In my opinion, this is justified as
-attempts to balance somewhat the burden and effort needed to fix the
-bug and the attempts to balance number of would-be bug reporters with
-the number of bug fixers. Better bug reporters are more likely to move
-in the category of bug fixers.
+burden on the bug reporter. This is justified since it attempts to balance
+the burden and effort needed to fix the bug with the amount of effort to report the problem. And it attempts
+to balance number of would-be bug reporters with the number of bug
+fixers. Better bug reporters are more likely to move in the category
+of bug fixers.
 
 The barrier to reporting a big is pretty small: all you really need is
 a github account, and the ability to type something after clicking
 some buttons. So the reality is that many people just don't bother to
 read these instructions, let alone follow it to any simulacrum.
 
-And the reality is also that bugs sometimes get fixed even though
-these instructions are not followed.
+That said, bugs sometimes get fixed even though these instructions are not followed.
 
-So one factors I may take into consideration is the bug reporter's karma.
+I may take into consideration is the bug reporter's karma.
 
-* Have you demonstrably contributed to open source? I may look at your
-  github profile to see what contributions you have made, how popular
-  those contributions are, or how popular you are.
-* How appreciative are you? Have you starred this project that you are
-  seeking help from? Have you starred _any_ github project? And the above
-  two kind of feed into ...
+* Have you demonstrably contributed to open source? I may look at your github profile to see what contributions you have made, how popular those contributions are, or how popular you are.
+* How appreciative are you? Have you starred this project that you are seeking help from? Have you starred _any_ github project? And the above two kind of feed into ...
 * Attitude. Some people feel that they are doing me and the world a
-  great favor by just pointing out that there is a problem whose solution
-  would greatly benefit them. Perhaps this is why they feel that
-  instructions are not to be followed by them, nor any need for
-  showing evidence gratitude when help is offered them.
+  great favor by just pointing out that there is a problem whose
+  solution would greatly benefit them. (This might account partially
+  for the fact that those that have this attitude often don't read or
+  follow instructions such as those given here.)
+
 
 # Confidentiality of Bug Reports
 
@@ -255,16 +233,6 @@ remains would not be an issue.
 However feel free to remove any comments, and modify variable names
 or constants in the source code.
 
-# Ethics
+If there is some legitimate reason to keep confidentiality, you can contact me by email to explain the extenuating circumstances. However I tend to discard without reading anonymous email.
 
-I do not condone using this program for unethical or illegal purposes.
-More detestable, at least to me, is asking for help to assist you in
-something that might not legitimate.
-
-Don't use the issue tracker for such solicitations. To try to stave
-off illegitimate behavior, you should note that the issue tracker, the
-code, and bugs mentioned in that are in the open: there is no
-confidentiality. You may be asked about the authorship or claimed
-ownership of the bytecode. If I think something is not quite right, I
-may label the issue questionable which may make the it easier those
-who are looking for illegal activity.
+Private consulting available via https://calendly.com/rb3216 rates: $150 for 30 minutes; $250 for 60 minutes.

Makefile

@@ -11,7 +11,10 @@ RM      ?= rm
 LINT    = flake8
 
 #EXTRA_DIST=ipython/ipy_trepan.py trepan
-PHONY=all check clean distcheck pytest check-long dist distclean lint flake8 test rmChangeLog clean_pyc
+PHONY=all check check-2.7 check-3.4 \
+     clean distcheck pytest check-long check-short \
+     dist distclean lint flake8 test rmChangeLog clean_pyc \
+     2.6 5.0 5.3 5.6 5.8 7.2 7.3 check-short
 
 TEST_TYPES=check-long check-short check-2.7 check-3.4
 
@@ -47,8 +50,16 @@ check-3.8:
 # Skip for now
 2.6 5.0 5.3 5.6 5.8:
 
-#:PyPy pypy3-2.4.0 Python 3:
-pypy-3.2 2.4:
+#:PyPy pypy3-2.4.0 Python 3.6.1:
+7.1 pypy-3.2 2.4:
+	$(MAKE) -C test $@
+
+#:PyPy versions
+7.2 7.3:
+	$(MAKE) -C test $@
+
+#:pyston versions
+2.3:
 	$(MAKE) -C test $@
 
 #: Run py.test tests

NEWS.md

@@ -1,8 +1,447 @@
-3.3.0 2019-03-23 Holy Week
+3.9.2: 2024-07-21
+=================
+
+- track xdis API changes
+- Bug fixes and lint
+
+3.9.1: 2024-05-15
+=================
+
+Lots of changes major changes. track xdis API has changes.
+
+Separate Phases more clearly:
+* disassembly
+* tokenization
+* parsing
+* abstracting to AST (more is done in newer projects)
+* printing
+
+Although we do not decompile bytecode greater than 3.8, code supports running from up to 3.12.
+
+Many bugs fixed.
+
+A lot of Linting and coding style modernization.
+
+Work done in preparation for Blackhat Asia 2024
+
+3.9.0: 2022-12-22
+=================
+
+* deparse generator expressions for Python 3.0 .. 3.2
+* Python 3.0 list comprehension.
+* Fix Issues #310, #344, #377, #391, #409, #414
+* Limited support for 3.8+ f-string "=" specifier
+* Correct 2.5-7 relative import formatting
+* Miscellaneous bug fixing
+* remove \n in lambda
+* Python 2.6 grammar cleanup
+* Correct some Python 2.6 chain compare decompilation
+* Ensure no parenthesis subscript slices
+* Correct 2.x formatting "slice2" nonterminal
+* Correct 3.7 imports
+* Improve "async for" parsing
+* Handle BUILD_MAP opcode
+* match Python AT better
+* Correct 3.7 positional args
+* PyPy 3.7 and PyPy 3.8 support
+* Miscellaneous linting, isorting, blacking
+
+3.8.0: 2021-10-29
+=================
+
+* Better handling of invalid bytecode magic
+* Support running from 3.9 and 3.10 although we do not support those bytecodes
+* Redo version comparisons using tuples instead of floats. This is needed for Python 3.10
+* Split out into 3 branches so that the master branch can assume Python 3.6+ conventions, especially type annotations
+* Source Fragment fixes
+* Lambda-bug fixes #360
+* Bug fixes
+
+3.7.4: 2020-8-05
+================
+
+* Fragment parsing was borked. This means deparsing in trepan2/trepan3k was broken
+* 3.7+: narrow precedence for call statement
+* del_stmt -> delete to better match Python AST
+* 3.8+ Add another `forelsestmt` (found only in a loop)
+* 3.8+ Add precedence on walrus operator
+* More files blackened
+* bump min xdis version
+
+3.7.3: 2020-7-25
+================
+
+Mostly small miscellaneous bug fixes
+
+* `__doc__ = DocDescr()` from `test_descr.py` was getting confused as a docstring.
+* detect 2.7 exchandler range better
+* Add for .. else reduction checks on 2.6 and before
+* Add reduce check for 2.7 augmented assign
+* Add VERSION in a pydoc-friendly way
+
+
+3.7.2: 2020-6-27
+================
+
+* Use newer xdis
+* Docstrings (again) which were broken again on earlier Python
+* Fix 2.6 and 2.7 decompilation bug in handling "list if" comprehensions
+
+
+
+3.7.1: 2020-6-12 Fleetwood66
+====================================================
+
+Released to pick up new xdis version which has fixes to read bytestrings better on 3.x
+
+* Handle 3.7+ "else" branch removal adAs seen in `_cmp()` of `python3.8/distutils/version.py` with optimization `-O2`
+* 3.6+ "with" and "with .. as" grammar improvements
+* ast-check for "for" loop was missing some grammar rules
+
+3.7.0: 2020-5-19 Primidi 1st Prairial - Alfalfa - HF
+====================================================
+
+The main impetus for this release is to pull in the recent changes from xdis.
+We simplify imports using xdis 4.6.0.
+
+There were some bugfixes to Python 3.4-3.8. See the ChangeLog for details
+
+
+3.6.7: 2020-4-27 xdis again
+===========================
+
+More upheaval in xdis which we need to track here.
+
+3.6.6: 2020-4-20 Love in the time of Cholera
+============================================
+
+The main reason for this release is an incompatibility bump in xdis which handles
+3.7 SipHash better.
+
+* Go over "yield" as an expression precedence
+* Some small alignment with code in decompyle3 for "or" and "and" was done
+
+
+3.6.5: 2020-4-1 April Fool
+==========================
+
+Back port some of the changes in decompile3 here which mostly helps 3.7 and 3.8 decompilation, although this may also help 3.6ish versions too.
+
+- Handle nested `async for in for...`  and better async comprehension detection via `xdis`.  Still more work is needed.
+- include token number in listings when `-g` and there is a parser error
+- remove unneeded `Makefile`s now that remake 4.3+1.5dbg is a thing that has `-c`
+- Bug in finding annotations in functions with docstrings
+- Fix bug found by 2.4 sre_parse.py testing
+- Fix `transform` module's  `ifelseif` bugs
+- Fix bug in 3.0 name module detection
+- Fix docstring detection
+
+3.6.4: 2020-2-9 Plateau
+=======================
+
+The main focus in this release was fix some of the more glaring problems creapt in from the last release due to that refactor.
+
+`uncompyle6` code is at a plateau where what is most needed is a code refactoring. In doing this, until everything refactored and replaced, decomplation may get worse.
+Therefore, this release largely serves as a checkpoint before more major upheaval.
+
+The upheaval, in  started last release, I believe the pinnacle was around c90ff51 which wasn't a release. I suppose I should tag that.
+
+After c90ff5, I started down the road of redoing control flow in a more comprehensible, debuggable, and scalable way. See [The Control Flow Mess](https://github.com/rocky/python-uncompyle6/wiki/The-Control-Flow-Mess)
+
+The bulk of the refactoring going on in the [decompyle3](https://github.com/rocky/python-decompil3) project, but I try to trickle down the changes.
+
+It is tricky because the changes are large and I have to figure decompose things so that little testable pieces can be done. And there is also the problem that what is in decompyle3 is incomplete as well.
+
+Other than control flow, another change that will probably happen in the next release is to redo the grammar for lambda expressions. Right now, we treat them as Python statements, you know, things with compound statements in them. But lambda aren't that. And so there is hackery to paper over difference making a statement out of an expression the wrong thing to do. For example, a return of an "and" expression can be expressed as nested "if" statements with return inside them, but the "if" variant of the bytecode is not valid in a lambda.
+
+In the decompyle3 code, I've gone down the road making the grammar goal symbol be an expression. This also offers the opportunity to split the grammar making parsing inside lambda not only more reliable because the wrong choices don't exist, but also simpler and faster because all those rules just need don't need to exist in parsing.
+
+I cringe in thinking about how the code has lived for so long without noticing such a simple stupidity, and lapse of sufficient thought.
+
+Some stats from testing. The below give numbers of decompiled tests from Python's test suite which successfully ran
+
+```
+   Version  test-suites passing
+   -------  -------------------
+   2.4.6     243
+   2.5.6     265
+   2.6.9     305
+   3.3.7     300
+   3.4.10    304
+   3.5.9     260
+   3.6.10    236
+   3.7.6     306
+   3.8.1     114
+```
+
+Decompiled bytecode files distributed with Python (syntax check only):
+
+```
+2.7.17  647 files:   0 failed
+3.2.6   900 files:   0 failed
+3.3.7  1256 files:   0 failed
+3.4.10  800 files:   0 failed
+3.5.9   900 files:   0 failed
+3.6.10 1300 files:  28 failed
+```
+
+
+3.6.3: 2020-1-26 Martin and Susanne
+===================================
+
+Of late, every release fixes major gaps and embarrassments of the last release....
+
+And in some cases, like this one, exposes lacuna and rot.
+
+I now have [control] flow under control, even if it isn't the most optimal way.
+
+I now have greatly expanded automated testing.
+
+On the most recent Python versions I regularly decompile thousands of Python programs that are distributed with Python.  when it is possible, I then decompile Python's standard test suite distributed with Python and run the decompiled source code which basically checks itself. This amounts to about 250 test programs per version. This is in addition to the 3 CI testing services which do different things.
+
+Does this mean the decompiler works perfectly? No. There are still a dozen or so failing programs, although the actual number of bugs is probably smaller though.
+
+However, in preparation of a more major refactoring of the parser grammar, this release was born.
+
+In many cases, decompilation is better. But there are some cases where decompilation has gotten worse. For lack of time (and interest) 3.0 bytecode suffered a hit. Possibly some code in the 3.x range did too. In time and with cleaner refactored code, this will come back.
+
+Commit c90ff51 was a local maximum before, I started reworking the grammar to separate productions that were specific to loops versus those that are not in loops.
+In the middle of that I added another grammar simplification to remove singleton productions of the form `sstmts-> stmts`. These were always was a bit ugly, and complicated output.
+
+At any rate if decompilation fails, you can try c90ff51. Or another decompiler. `unpyc37` is pretty good for 3.7. wibiti `uncompyle2` is great for 2.7. `pycdc` is mediocre for Python before 3.5 or so, and not that good for the most recent Python. Generally these programs will give some sort of answer even if it isn't correct.
+
+decompyle3 isn't that good for 3.7 and worse for 3.8, but right now it does things no other Python decompiler like `unpyc37` or `pycdc` does. For example, `decompyle3` handles variable annotations. As always, the issue trackers for the various programs will give you a sense for what needs to be done. For now, I've given up on reporting issues in the other decompilers because there are already enough issues reported, and they are just not getting fixed anyway.
+
+
+3.6.2: 2020-1-5 Samish
+======================
+
+Yet again the focus has been on just fixing bugs, mostly geared in the
+later 3.x range. To get some sense what sill needs fixing, consult
+test/stdlib/runtests.sh. And that only has a portion of what's known.
+
+`make_function.py` has gotten so complex that it was split out into 3 parts
+to handle different version ranges: Python <3, Python 3.0..3.6 and Python 3.7+.
+
+An important fix is that we had been dropping docstrings in Python 3 code as a result
+of a incomplete merge from the decompile3 base with respect to the transform phase.
+
+Also important (at least to me) is that we can now handle 3.6+
+variable type annotations.  Some of the decompile3 code uses that in
+its source code, and I now use variable annotations in conjunction
+with mypy in some of my other Python projects
+
+Code generation for imports, especially where the import is dotted
+changed a bit in 3.7; with this release are just now tracking that
+change better. For this I've added pseudo instruction
+`IMPORT_NAME_ATTR`, derived from the `IMPORT_NAME` instruction, to
+indicate when an import contains a dotted import. Similarly, code for
+3.7 `import .. as ` is basically the same as `from .. import`, the
+only difference is the target of the name changes to an "alias" in the
+former. As a result, the disambiguation is now done on the semantic
+action side, rather than in parsing grammar rules.
+
+Some small specific fixes:
+
+* 3.7+ some chained compare parsing has been fixed. Other remain.
+* better if/else rule checking in the 3.4 and below range.
+* 3.4+ keyword-only parameter handling was fixed more generally
+* 3.3 .. 3.5 keyword-only parameter args in lambda was fixed
+
+
+3.6.1: 2019-12-10 Christmas Hannukah
+====================================
+
+Overall, as in the past, the focus has been on just fixing bugs, more geared
+in the later 3.x range. Handling "async for/with" in 3.8+ works better.
+
+Numerous bugs around handling `lambda` with keyword-only and `*` args in the
+3.0-3.8 have been fixed. However many still remain.
+
+`binary_expr` and `unary_expr` have been renamed to `bin_op` and
+`unary_op` to better correspond the Python AST names.
+
+Some work was done Python 3.7+ to handle `and` better; less was done
+along the lines of handling `or`. Much more is needed to improve
+parsing stability of 3.7+. More of what was done with `and` needs to
+be done with `or` and this will happen first in the "decompyle3"
+project.
+
+Later this will probably be extended backwards to handle the 3.6-
+versions better. This however comes with a big decompilation speed
+penalty. When we redo control flow this should go back to normal, but
+for now, accuracy is more important than speed.
+
+Another `assert` transform rule was added. Parser rules to distinguish
+`try/finally` in 3.8 were added and we are more stringent about what
+can be turned into an `assert`. There was some grammar cleanup here
+too.
+
+A number of small bugs were fixed, and some administrative changes to
+make `make check-short` really be short, but check more thoroughly what
+it checks. minimum xdis version needed was bumped to include in the
+newer 3.6-3.9 releases. See the `ChangeLog` for details.
+
+
+3.6.0: 2019-12-10 gecko gecko
+=============================
+
+The main focus in this release was more accurate decompilation especially
+for 3.7 and 3.8. However there are some improvements to Python 2.x as well,
+including one of the long-standing problems of detecting the difference between
+`try ... ` and `try else ...`.
+
+With this release we now rebase Python 3.7 on off of a 3.7 base; This
+is also as it is (now) in decompyle3.  This facilitates removing some of the
+cruft in control-flow detection in the 2.7 uncompyle2 base.
+
+Alas, decompilation speed for 3.7 on is greatly increased. Hopefully
+this is temporary (cough, cough) until we can do a static control flow
+pass.
+
+Finally, running in 3.9-dev is tolerated. We can disassemble, but no parse tables yet.
+
+
+3.5.1 2019-11-17 JNC
+====================
+
+- Pypy 3.3, 3.5, 3.6, and 3.6.9 support
+- bump xdis version to handle newer Python releases, e.g. 2.7.17, 3.5.8, and 3.5.9
+- Improve 3.0 decompilation
+    - no parse errors on stlib bytecode. However accurate translation in
+	  control-flow and and/or detection needs work
+- Remove extraneous iter() in "for" of list comprehension  Fixes #272
+- "for" block without a `POP_BLOCK `and confusing `JUMP_BACK` for `CONTINUE`. Fixes #293
+- Fix unmarshal incompletness detected in Pypy 3.6
+- Miscellaneous bugs fixed
+
+3.5.0 2019-10-12 Stony Brook Ride
+=================================
+
+- Fix fragment bugs
+   * missing `RETURN_LAST` introduced when adding transformation layer
+   * more parent entries on tokens
+- Preliminary support for decompiling Python 1.0, 1.1, 1.2, and 1.6
+   * Newer _xdis_ version needed
+
+3.4.1 2019-10-02
+================
+
+- Correct assert{,2} transforms Fixes #289
+- Fragment parsing fixes:
+     * Wasn't handling 3-arg `%p`
+   	 * fielding error in `code_deparse()`
+- Use newer _xdis_ to better track Python 3.8.0
+
+
+3.4.0 2019-08-24 Totoro
+=======================
+
+The main change is to add a tree-transformation phase. This simplifies the
+code a little and allows us to turn `if ...: raise AssertionError` into
+`assert`, and many `if ..: else if ...` into `if ... elif ..`
+
+Use options `--show=before` and `--show=after` to see the before the tree transformation phase and after the tree transformation phase.
+
+Most of the heavy lifting for this was done by x0ret.
+
+Other changes:
+
+- Fix issue #275, #283 (process to fix this bug is documented on wiki), #284
+- blacken more code
+- CircleCI adjustments for a changing CircleCi
+- Require more recent `xdis` for Python 3.8
+- Fix bugs in code using `BUILD_LIST_UNPACK` and variants
+
+3.3.5 2019-07-03 Pre Independence Day
+=====================================
+
+Again, most of the work in this is release is thanks to x0ret.
+
+- Handle annotation arguments in Python 3.x
+- Fix _vararg_ and function signatures in 3.x
+- Some 3.x < 3.6 `while` (1)/`if` fixes &mdash; others remain
+- Start reinstating `else if` -> `elif`
+- `LOAD_CONST` -> `LOAD_CODE` where appropriate
+- option `--weak-verify` is now `--syntax-verify`
+- code cleanups, start using [black](https://github.com/python/black) to reformat text
+
+
+3.3.4 2019-06-19 Fleetwood at 65
+================================
+
+Most of the work in this is release is thanks to x0ret.
+
+- Major work was done by x0ret to correct function signatures and include annotation types
+- Handle Python 3.6 `STORE_ANNOTATION` [#58](https://github.com/rocky/python-uncompyle6/issues/58)
+- Friendlier assembly output
+- `LOAD_CONST` replaced by `LOAD_STR` where appropriate to simplify parsing and improve clarity
+- remove unneeded parenthesis in a generator expression when it is the single argument to the function [#247](https://github.com/rocky/python-uncompyle6/issues/246)
+- Bug in noting an async function [#246](https://github.com/rocky/python-uncompyle6/issues/246)
+- Handle Unicode docstrings and fix docstring bugs [#241](https://github.com/rocky/python-uncompyle6/issues/241)
+- Add short option -T as an alternate for --tree+
+- Some grammar cleanup
+
+3.3.3 2019-05-19 Henry and Lewis
+================================
+
+As before, decomplation bugs fixed. The focus has primarily been on
+Python 3.7. But with this release, releases will be put on hold,as a
+better control-flow detection is worked on . This has been needed for a
+while, and is long overdue. It will probably also take a while to get
+done as good as what we have now.
+
+However this work will be done in a new project
+[decompyle3](https://github.com/rocky/python-decompile3).  In contrast
+to _uncompyle6_ the code will be written assuming a modern Python 3,
+e.g. 3.7. It is originally intended to decompile Python version 3.7
+and greater.
+
+* A number of Python 3.7+ chained comparisons were fixed
+* Revise Python 3.6ish format string handling
+* Go over operator precedence, e.g. for AST `IfExp`
+
+Reported Bug Fixes
+------------------
+
+* [#239: 3.7 handling of 4-level attribute import](https://github.com/rocky/python-uncompyle6/issues/239),
+* [#229: Inconsistent if block in python3.6](https://github.com/rocky/python-uncompyle6/issues/229),
+* [#227: Args not appearing in decompiled src when kwargs is specified explicitly (call_ex_kw)](https://github.com/rocky/python-uncompyle6/issues/227)
+2.7 confusion around "and" versus comprehension "if"
+* [#225: 2.7 confusion around "and" vs comprehension "if"](https://github.com/rocky/python-uncompyle6/issues/225)
+
+3.3.2 2019-05-03 Better Friday
+==============================
+
+As before, lots of decomplation bugs fixed. The focus has primarily
+been on Python 3.6. We can now parse the entire 3.6.8 Python library
+and verify that without an error. The same is true for 3.5.8. A number
+of the bugs fixed though are not contained to these versions. In fact
+some span back as far as 2.x
+
+But as before, many more remain in the 3.7 and 3.8 range which will
+get addressed in future releases
+
+Pypy 3.6 support was started. Pypy 3.x detection fixed (via `xdis`)
+
+3.3.1 2019-04-19 Good Friday
+==========================
+
+Lots of decomplation bugs, especially in the 3.x series fixed. Don't worry though, many more remain.
+
+* Add annotation return values in 3.6+
+* Fix 3.6+ lambda parameter handling decompilation
+* Fix 3.7+ chained comparison decompilation
+* split out semantic-action customization into more separate files
+* Add 3.8 try/else
+* Fix 2.7 generator decompilation
+* Fix some parser failures fixes in 3.4+ using test_pyenvlib
+* Add more run tests
+
+3.3.0 2019-04-14 Holy Week
 ==========================
 
 * First cut at Python 3.8 (many bug remain)
-* Reinstate -c | --compile (compile before disassembly) option
 * The usual smattering of bug and doc fixes
 
 3.2.6 2019-03-23 Mueller Report
@@ -13,23 +452,25 @@ Mostly more of the same: bug fixes and pull requests.
 Bug Fixes
 -----------
 
-* [#155: Python 3.x bytecode confusing "try/else" with "try" in a loop](https://github.com/rocky/python-uncompyle6/issues/155),
-* [#200: Python 3 bug in not detecting end bounds of an "if" ... "elif"](https://github.com/rocky/python-uncompyle6/issues/200),
-* [#208: Comma placement in 3.6 and 3.7 **kwargs](https://github.com/rocky/python-uncompyle6/issues/208),
-* [#209: Fix "if" return boundary in 3.6+](https://github.com/rocky/python-uncompyle6/issues/209),
+* [#221: Wrong grammar for nested ifelsestmt (in Python 3.7 at least)](https://github.com/rocky/python-uncompyle6/issues/221)
 * [#215: 2.7 can have two JUMP_BACKs at the end of a while loop](https://github.com/rocky/python-uncompyle6/issues/215)
+* [#209: Fix "if" return boundary in 3.6+](https://github.com/rocky/python-uncompyle6/issues/209),
+* [#208: Comma placement in 3.6 and 3.7 **kwargs](https://github.com/rocky/python-uncompyle6/issues/208),
+* [#200: Python 3 bug in not detecting end bounds of an "if" ... "elif"](https://github.com/rocky/python-uncompyle6/issues/200),
+* [#155: Python 3.x bytecode confusing "try/else" with "try" in a loop](https://github.com/rocky/python-uncompyle6/issues/155),
+
 
 Pull Requests
 ----------------
 
-* [#202: Better "assert" statement detemination in Python 2.7](https://github.com/rocky/python-uncompyle6/pull/211)
+* [#202: Better "assert" statement determination in Python 2.7](https://github.com/rocky/python-uncompyle6/pull/211)
 * [#204: Python 3.7 testing](https://github.com/rocky/python-uncompyle6/pull/204)
 * [#205: Run more f-string tests on Python 3.7](https://github.com/rocky/python-uncompyle6/pull/205)
 * [#211: support utf-8 chars in Python 3 sourcecode](https://github.com/rocky/python-uncompyle6/pull/202)
 
 
 
-3.2.5 2018-12-30 Clearout sale
+3.2.5 2018-12-30 Clear-out sale
 ======================================
 
 - 3.7.2 Remove deprecation warning on regexp string that isn't raw
@@ -64,7 +505,7 @@ Pull Requests
 
 - Add rudimentary 1.4 support (still a bit buggy)
 - add --tree+ option to show formatting rule, when it is constant
-- Python 2.7.15candidate1 support (via xdis)
+- Python 2.7.15candidate1 support (via `xdis`)
 - bug fixes, especially for 3.7 (but 2.7 and 3.6 and others as well)
 
 3.1.3 2018-04-16
@@ -94,14 +535,14 @@ Jesus on Friday's New York Times puzzle: "I'm stuck on 2A"
 - reduce 3.5, 3.6 control-flow bugs
 - reduce ambiguity in rules that lead to long (exponential?) parses
 - limit/isolate some 2.6/2.7,3.x grammar rules
-- more runtime testing of decompiled code
-- more removal of parenthesis around calls via setting precidence
+- more run-time testing of decompiled code
+- more removal of parenthesis around calls via setting precedence
 
 3.1.0 2018-03-21 Equinox
 ==============================
 
 - Add code_deparse_with_offset() fragment function.
-- Correct paramenter call fragment deparse_code()
+- Correct parameter call fragment deparse_code()
 - Lots of 3.6, 3.x, and 2.7 bug fixes
   About 5% of 3.6 fail parsing now. But
   semantics still needs much to be desired.
@@ -151,7 +592,7 @@ function calls and definitions.
 - Misc pydisasm fixes
 - Weird comprehension bug seen via new loctraceback
 - Fix Python 3.5+ CALL_FUNCTION_VAR and BUILD_LIST_UNPACK in call; with this
-  we can can handle 3.5+ f(a, b, *c, *d, *e) now
+  we can handle 3.5+ f(a, b, *c, *d, *e) now
 
 2.15.1 2018-02-05
 =====================
@@ -169,24 +610,24 @@ function calls and definitions.
 2.15.1 2018-01-27
 =====================
 
-- Add --linemap option to give line correspondences
+- Add `--linemap` option to give line correspondences
   between original source lines and reconstructed line sources.
   It is far from perfect, but it is a start
 - Add a new class of tests: tests which when decompiled check themselves
 - Split off Python version semantic action customizations into its own file
-- Fix 2.7 bug in ifelse loop statement
-- Handle 3.6+ EXTENDED_ARGs for POP_JUMP_IF... instructions
-- Correct 3.6+ calls with kwargs
+- Fix 2.7 bug in `if`/`else` loop statement
+- Handle 3.6+ `EXTENDED_ARG`s for `POP_JUMP_IF..` instructions
+- Correct 3.6+ calls with `kwargs`
 - Describe the difficulty of 3.6 in README
 
 2.14.3 2018-01-19
 =====================
 
-- Fix bug in 3.5+ await stmt
+- Fix bug in 3.5+ `await` statement
 - Better version to magic handling; handle 3.5.2 .. 3.5.4 versions
 - Improve/correct test_pyenvlib.py status messages
 - Fix some 2.7 and 2.6 parser bugs
-- Fix whilelse parsing bugs
+- Fix `whilelse` parsing bugs
 - Correct 2.5- decorator parsing
 - grammar for decorators matches AST a little more
 - better tests in setup.py for running the right version of Python
@@ -197,15 +638,15 @@ function calls and definitions.
 
 Decompilation bug fixes, mostly 3.6 and pre 2.7
 
-- 3.6 FUNCTION_EX (somewhat)
-- 3.6 FUNCTION_EX_KW fixes
-- 3.6 MAKE_FUNCTION fixes
-- correct 3.5 CALL_FUNCTION_VAR
+- 3.6 `FUNCTION_EX` (somewhat)
+- 3.6 `FUNCTION_EX_KW` fixes
+- 3.6 `MAKE_FUNCTION` fixes
+- correct 3.5 `CALL_FUNCTION_VAR`
 - stronger 3.x "while 1" testing
 - Fix bug in if's with "pass" bodies. Fixes #104
 - try/else and try/finally fixes on 2.6-
 - limit pypy customization to pypy
-- Add addr fields in COME_FROMS
+- Add addr fields in `COME_FROM`S
 - Allow use of full instructions in parser reduction routines
 - Reduce grammar in Python 3 by specialization more to specific
   Python versions
@@ -214,11 +655,11 @@ Decompilation bug fixes, mostly 3.6 and pre 2.7
 2.14.1 2017-12-10 Dr. Gecko
 ===================================
 
-- Many decompilation bugfixes
+- Many decompilation bug fixes
 - Grammar rule reduction and version isolation
 - Match higher-level nonterminal names more closely
   with Python AST
-- Start automated Python stdlib testing - full round trip
+- Start automated Python _stdlib_ testing &mdash; full round trip
 
 2.14.0 2017-11-26 johnnybamazing
 =========================================
@@ -227,7 +668,7 @@ Decompilation bug fixes, mostly 3.6 and pre 2.7
   and remove used grammar rules
 - Fix a number of bytecode decompile problems
   (many more remain)
-- Add stdlib/runtests.sh for even more rigorous testing
+- Add `stdlib/runtests.sh` for even more rigorous testing
 
 2.13.3 2017-11-13
 =====================
@@ -243,16 +684,16 @@ Overall: better 3.6 decompiling and some much needed code refactoring and cleanu
   rather trying to parse the bytecode array. This largely been done in for versions 3.x;
   3.0 custom mangling code has been reduced;
   some 2.x conversion has been done, but more is desired. This make it possible to...
-- Handle EXTENDED_ARGS better. While relevant to all Python versions it is most noticeable in
-  version 3.6+ where in switching to wordcodes the size of operands has been reduced from 2**16
-  to 2**8. JUMP instruction then often need EXTENDED_ARGS.
-- Refactor find_jump_targets() with via working of of instructions rather the bytecode array.
-- use --weak-verify more and additional fuzzing on verify()
+- Handle `EXTENDED_ARGS` better. While relevant to all Python versions it is most noticeable in
+  version 3.6+ where in switching to wordcodes the size of operands has been reduced from 2^16
+  to 2^8. `JUMP` instruction then often need EXTENDED_ARGS.
+- Refactor find_jump_targets() with via working of instructions rather the bytecode array.
+- use `--weak-verify` more and additional fuzzing on verify()
 - fragment parser now ignores errors in nested function definitions; an parameter was
   added to assist here. Ignoring errors may be okay because the fragment parser often just needs,
   well, *fragments*.
-- Distinguish RETURN_VALUE from RETURN_END_IF in exception bodies better in 3.6
-- bug in 3.x language changes: import queue via import Queue
+- Distinguish `RETURN_VALUE` from `RETURN_END_IF` in exception bodies better in 3.6
+- bug in 3.x language changes: import queue via `import Queue`
 - reinstate some bytecode tests since decompiling has gotten better
 - Revise how to report a bug
 
@@ -272,12 +713,12 @@ Overall: better 3.6 decompiling and some much needed code refactoring and cleanu
 - Fixes in deparsing lambda expressions
 - Improve table-semantics descriptions
 - Document hacky customize arg count better (until we can remove it)
-- Update to use xdis 3.7.0 or greater
+- Update to use `xdis` 3.7.0 or greater
 
 2.12.0 2017-09-26
 =====================
 
-- Use xdis 3.6.0 or greater now
+- Use `xdis` 3.6.0 or greater now
 - Small semantic table cleanups
 - Python 3.4's terms a little names better
 - Slightly more Python 3.7, but still failing a lot
@@ -291,13 +732,13 @@ Overall: better 3.6 decompiling and some much needed code refactoring and cleanu
 2.11.4 2017-08-15
 =====================
 
-* scanner and parser now allow 3-part version string lookups,
+* scanner and parser now allow 3-part version string look ups,
   e.g. 2.7.1 We allow a float here, but if passed a string like '2.7'. or
-* unpin 3.5.1. xdis 3.5.4 has been release and fixes the problems we had. Use that.
-* some routines here moved to xdis. Use the xdis version
-* README.rst: Link typo Name is trepan2 now not trepan
-* xdis-forced change adjust for COMPARE_OP "is-not" in
-  semanatic routines. We need "is not".
+* unpin 3.5.1. `xdis` 3.5.4 has been release and fixes the problems we had. Use that.
+* some routines here moved to `xdis`. Use the `xdis` version
+* `README.rst`: Link typo Name is _trepan2_ now not _trepan_
+* xdis-forced change adjust for `COMPARE_OP` "is-not" in
+  semantic routines. We need "is not".
 * Some PyPy tolerance in validate testing.
 * Some pyston tolerance
 
@@ -307,15 +748,15 @@ Overall: better 3.6 decompiling and some much needed code refactoring and cleanu
 Very minor changes
 
 - RsT doc fixes and updates
-- use newer xdis, but not too new; 3.5.2 breaks uncompyle6
-- use xdis opcode sets
-- xdis "exception match" is now "exception-match"
+- use newer `xdis`, but not too new; 3.5.2 breaks uncompyle6
+- use `xdis` opcode sets
+- `xdis` "exception match" is now "exception-match"
 
 2.11.2 2017-07-09
 =====================
 
 - Start supporting Pypy 3.5 (5.7.1-beta)
-- use xdis 3.5.0's opcode sets and require xdis 3.5.0
+- use `xdis` 3.5.0's opcode sets and require `xdis` 3.5.0
 - Correct some Python 2.4-2.6 loop detection
 - guard against badly formatted bytecode
 
@@ -323,55 +764,55 @@ Very minor changes
 =====================
 
 - Python 3.x annotation and function signature fixes
-- Bump xdis version
-- Small pysource bug fixes
+- Bump `xdis` version
+- Small `pysource.py` bug fixes
 
 2.11.0 2017-06-18 Fleetwood
 ==================================
 
 - Major improvements in fragment tracking
-  * Add nonterminal node in extractInfo
+  * Add nonterminal node in `extractInfo()`
   * tag more offsets in expressions
   * tag array subscripts
-  * set YIELD value offset in a <yield> expr
+  * set `YIELD` value offset in a _yield expr_
   * fix a long-standing bug in not adjusting final AST when melding other deparse ASTs
 - Fixes yet again for make_function node handling; document what's up here
-- Fix bug in snowflake Python 3.5 *args kwargs
+- Fix bug in snowflake Python 3.5 `*args`, `kwargs`
 
 2.10.1 2017-06-3 Marylin Frankel
 ========================================
 
 - fix some fragments parsing bugs
-  - was returning the wrong type sometimes in deparse_code_around_offset()
+  - was returning the wrong type sometimes in `deparse_code_around_offset()`
   - capture function name in offsets
-  - track changes to ifelstrmtr node from pysource into fragments
+  - track changes to `ifelstrmtr` node from `pysource.py` into fragments
 
 2.10.0 2017-05-30 Elaine Gordon
 =======================================
 
 - Add fuzzy offset deparse look up
 - 3.6 bug fixes
-   - fix EXTENDED_ARGS handling (and in 2.6 and others)
+   - fix `EXTENDED_ARGS` handling (and in 2.6 and others)
    - semantic routine make_function fragments.py
-   - MAKE_FUNCTION handling
-   - CALL_FUNCTION_EX handling
-   - async property on defs
-   - support for CALL_FUNCTION_KW (moagstar)
-- 3.5+ UNMAP_PACK and BUILD_UNMAP_PACK handling
+   - `MAKE_FUNCTION` handling
+   - `CALL_FUNCTION_EX` handling
+   - `async` property on `defs`
+   - support for `CALL_FUNCTION_KW` (moagstar)
+- 3.5+ `UNMAP_PACK` and` BUILD_UNMAP_PACK` handling
 - 3.5 FUNCTION_VAR bug
-- 3.x pass statement insdie while True
+- 3.x pass statement inside `while True`
 - Improve 3.2 decompilation
-- Fixed -o argument processing (grkov90)
+- Fixed `-o` argument processing (grkov90)
 - Reduce scope of LOAD_ASSERT as expr to 3.4+
-- "await" statement fixes
+- `await` statement fixes
 - 2.3, 2.4 "if 1 .." fixes
 - 3.x annotation fixes
 
 2.9.11 2017-04-06
 =====================
 
-- Better support for Python 3.5+ BUILD_MAP_UNPACK
-- Start 3.6 CALL_FUNCTION_EX support
+- Better support for Python 3.5+ `BUILD_MAP_UNPACK`
+- Start 3.6 `CALL_FUNCTION_EX` support
 - Many decompilation bug fixes. (Many more remain). See ChangeLog
 
 2.9.10 2017-02-25
@@ -379,7 +820,7 @@ Very minor changes
 
 - Python grammar rule fixes
 - Add ability to get grammar coverage on runs
-- Handle Python 3.6 opcode BUILD_CONST_KEYMAP
+- Handle Python 3.6 opcode `BUILD_CONST_KEYMAP`
 
 2.9.9 2016-12-16
 
@@ -395,13 +836,13 @@ Very minor changes
 ====================
 
 - Better control-flow detection
-- pseudo instruction THEN in 2.x
+- pseudo instruction `THEN` in 2.x
   to disambiguate if from and
-- fix bug in --verify option
+- fix bug in `--verify` option
 - DRY (a little) control-flow detection
 - fix syntax in tuples with one element
 - if AST rule inheritance in Python 2.5
-- NAME_MODULE removal for Python <= 2.4
+- `NAME_MODULE` removal for Python <= 2.4
 - verify call fixes for Python <= 2.4
 - more Python lint
 
@@ -412,9 +853,9 @@ Very minor changes
 - Some Python 3.6 bytecode to wordcode conversion fixes
 - option -g: show start-end range when possible
 - track print_docstring move to help (used in python 3.1)
-- verify: allow RETURN_VALUE to match RETURN_END_IF
+- verify: allow `RETURN_VALUE` to match `RETURN_END_IF`
 - some 3.2 compatibility
-- Better Python 3 control flow detection by adding Pseudo ELSE opcodes
+- Better Python 3 control flow detection by adding Pseudo `ELSE` opcodes
 
 2.9.6 2016-12-04
 ====================
@@ -430,7 +871,7 @@ Very minor changes
 
 - Correct MANIFEST.in
 - More AST grammar checking
-- --linemapping option or linenumbers.line_number_mapping()
+- `--linemapping` option or _linenumbers.line_number_mapping()_
   Shows correspondence of lines between source
   and decompiled source
 - Some control flow adjustments in code for 2.x.
@@ -450,7 +891,7 @@ Very minor changes
   * improper while 1 else
   * docstring indent
   * 3.3 default values in lambda expressions
-  * start 3.0 decompilation (needs newer xdis)
+  * start 3.0 decompilation (needs newer `xdis`)
 - Start grammar misparse checking
 
 
@@ -464,12 +905,12 @@ Very minor changes
 2.9.3 2016-10-26
 ====================
 
-Release forced by incompatibility change in xdis 3.2.0.
+Release forced by incompatibility change in` xdis` 3.2.0.
 
 - Python 3.1 bugs:
-  * handle "with ... as"
-  * handle "with"
-  * Start handling def (...) -> yy (has bugs still)
+  * handle `with`... `as`
+  * handle `with`
+  * Start handling `def` (...) -> _yy_ (has bugs still)
 
 - DRY Python 3.x via inheritance
 - Python 3.6 work (from Daniel Bradburn)
@@ -495,12 +936,12 @@ Release forced by incompatibility change in xdis 3.2.0.
 2.9.0 2016-10-09
 ====================
 
-- Use xdis 3.0.0 protocol load_module.
+- Use `xdis` 3.0.0 protocol `load_module`.
   this Forces change in requirements.txt and _pkg_info_.py
 - Start Python 1.5 decompiling; another round of work is needed to
   remove bugs
 - Simplify python 2.1 grammar
-- Fix bug with -t ...  Wasn't showing source text when -t option was given
+- Fix bug with `-t` ...  Wasn't showing source text when `-t` option was given
 - Fix 2.1-2.6 bug in list comprehension
 
 2.8.4 2016-10-08
@@ -509,8 +950,8 @@ Release forced by incompatibility change in xdis 3.2.0.
 - Python 3 disassembly bug fixes
 - Python 3.6 fstring bug fixes (from moagstar)
 - Python 2.1 disassembly
-- COME_FROM suffixes added in Python3
-- use .py extension in verification disassembly
+- `COME_FROM` suffixes added in Python3
+- use `.py` extension in verification disassembly
 
 2.8.3 2016-09-11 live from NYC!
 =======================================
@@ -555,8 +996,8 @@ control-flow structure detection is done.
 - Add Python 2.2 decompilation
 
 - Fix bugs
- * PyPy LOOKUP_METHOD bug
- * Python 3.6 FORMAT_VALUE handles expressions now
+ * PyPy `LOOKUP_METHOD` bug
+ * Python 3.6 `FORMAT_VALUE` handles expressions now
 
 2.8.0 2016-08-03
 ====================
@@ -606,7 +1047,7 @@ control-flow structure detection is done.
 ====================
 
 - Improve Python 2.6 bytecode deparsing:
-  stdlib now will deparse something
+  _stdlib_ now will deparse something
 - Better <2.6 vs. 2.7 grammar separation
 - Fix some 2.7 deparsing bugs
 - Fix bug in installing uncompyle6 script
@@ -669,9 +1110,9 @@ control-flow structure detection is done.
 2.3.2 2016-05-1
 ===================
 
-- Add --version option standalone scripts
+- Add `--version` option standalone scripts
 - Correct License information in package
-- expose fns uncompyle_file, load_file, and load_module
+- expose functions `uncompyle_file()`, `load_file()`, and `load_module()`
 - Start to DRY Python2 and Python3 grammars Separate out 3.2, and 3.5+
   specific grammar code
 - Fix bug in 3.5+ constant map parsing
@@ -679,12 +1120,12 @@ control-flow structure detection is done.
 2.3.0, 2.3.1 2016-04-30
 =============================
 
-- Require spark_parser  >= 1.1.0
+- Require `spark_parser` >= 1.1.0
 
 2.2.0 2016-04-30
 ====================
 
-- Spark is no longer here but pulled separate package spark_parse
+- Spark is no longer here but pulled separate package [spark_parser](https://pypi.org/project/spark_parser/)
 - Python 3 parsing fixes
 - More tests
 
@@ -694,7 +1135,7 @@ control-flow structure detection is done.
 - Support single-mode (in addition to exec-mode) compilation
 - Start to DRY Python 2 and Python 3 grammars
 - Fix bug in if else ternary  construct
-- Fix bug in uncomplye6 -d and -r options (via lelicopter)
+- Fix bug in uncomplye6 `-d` and `-r` options (via lelicopter)
 
 
 2.1.3 2016-01-02
@@ -702,7 +1143,7 @@ control-flow structure detection is done.
 
 - Limited support for decompiling Python 3.5
 - Improve Python 3 class deparsing
-- Handle MAKE_CLOSURE opcode
+- Handle `MAKE_CLOSURE` opcode
 - Start to DRY opcode code.
 - increase test coverage
 - fix misc small bugs and some improvements
@@ -744,5 +1185,5 @@ Changes from uncompyle2
 
 SPARK:
   add option to show grammar rules applied
-  allow Python-style # comments in grammar
+  allow Python-style `#` comments in grammar
   Runs on Python 3 and Python 2

NEW_FEATURES.rst

@@ -0,0 +1,221 @@
+Introduction
+============
+
+The original versions of this code up until the time I started were
+pretty awesome.  You can get a sense of this by running it.  For the
+most part it was remarkably fast, and a single module with few dependencies.
+
+Here I will largely give what are the major improvements over old code.
+
+This also serves to outline a little bit about what is in this code.
+
+See also `How does this code work? <https://github.com/rocky/python-uncompyle6/wiki/How-does-this-code-work%3F>`_.
+
+Old Cool Features
+==================
+
+Before getting to the new stuff, I'll describe cool things that was there before.
+
+I particularly liked the ability to show the assembly, grammar
+reduction rules as they occurred, and the resulting parse tree. It is
+neat that you could follow the process and steps that deparser takes,
+and in this not only see the result how the bytecode corresponds to
+the resulting source. Compare this with other Python decompilers.
+
+And of course also neat was that this used a grammar and table-driven
+approach to decompile.
+
+
+Expanding decompilation to multiple Python Versions
+==================================================
+
+Aside from ``pycdc``, most of the Python decompilers handle a small
+number of Python versions, if they supported more than one. And even
+when more than one version is supported if you have to be running the
+Python version that the bytecode was compiled for.
+
+There main reason that you have to be running the Python bytecode
+interpreter as the one you want to decompile largely stems from the
+fact that Python's ``dis`` module is often what is used and that has this limitation.
+
+``pycdc`` doesn't suffer this problem because it is written in C++,
+not Python.  Hartmut Goebel's code had provisions for multiple Python
+versions running from an interpreter different from the one that was
+running the decompiler. That however used compiled code in the process
+was tied a bit to the Python C headers for a particular version.
+
+You need to not only to account for different "marshal" and "unmarshal"
+routines for the different Python versions, but also, as the Python versions
+extend, you need a different code type as well.
+
+Enter ``xdis``
+--------------
+
+To handle all of these problems, I split off the marshal loading
+portion and disassembly routines into a separate module,
+`xdis <https://pypi.org/project/xdis/>`_. This also allows older Pythons to have access to features
+found in newer Pythons, such as parsing the bytecode, a uniform stream
+of bytes, into a list of structured bytecode instructions.
+
+Python 2.7's ``dis`` module doesn't has provide a instruction abstraction.
+Therefore in ``uncompyle2`` and other earlier decompilers you see code with magic numbers like 4 in::
+
+    if end > jump_back+4 and code[end] in (JF, JA):
+        if code[jump_back+4] in (JA, JF):
+            if self.get_target(jump_back+4) == self.get_target(end):
+                self.fixed_jumps[pos] = jump_back+4
+                end = jump_back+4
+    elif target < pos:
+        self.fixed_jumps[pos] = jump_back+4
+        end = jump_back+4
+
+and in other code -1 and 3 in::
+
+        if self.get_target(jmp) != start_else:
+            end_else = self.get_target(jmp)
+        if self.code[jmp] == JF:
+            self.fixed_jumps[jmp] = -1
+        self.structs.append({'type':  'except',
+                       'start': i,
+                       'end':   jmp})
+        i = jmp + 3
+
+All of that offset arithmetic is trying to find the next instruction
+offset or the previous offset. Using a list of instructions you simply
+take the ``offset`` field of the previous or next instruction.
+
+The above code appears in the ``uncompyle2`` "Scanner" class in
+service of trying to figure out control flow. Note also that there
+isn't a single comment in there about what specifically it is trying
+to do, the logic or that would lead one to be confident that this is
+correct, let alone assumptions that are needed for this to be true.
+
+While this might largely work for Python 2.7, and ``uncompyle2`` does
+get control flow wrong sometimes, it is impossible to adapt code for
+other versions of Python.
+
+In addition adding an instruction structure, ``xdis`` adds various
+flags and features that assist in working with instructions. In the
+example above this replaces code like ``... in (JF, JA)`` which is
+some sort of unconditional jump instruction.
+
+Although not needed in the decompiler, ``xdis`` also has nicer
+instruction print format. It can show you the bytes as well as the
+interpreted instructions. It will interpret flag bits and packed
+structures in operands so you don't have to. It can even do a limited
+form of inspection at previous instructions to give a more complete
+description of an operand. For example on ``LOAD_ATTR`` which loads
+the attribute of a variable, often the variable name can be found as
+the previous instruction. When that is the case the disassembler can
+include that in the disassembly display for the ``LOAD_ATTR`` operand.
+
+
+Python Grammar Isolation
+------------------------
+
+If you want to support multiple versions of Python in a manageable way
+you really need to provide different grammars for the different
+versions, in a grammar-based system. None of the published versions of
+this decompiler did this.
+
+If you look at the changes in this code, right now there are no
+grammar changes needed between 1.0 to 1.3. (Some of this may be wrong
+though since we haven't extensively tested these earliest Python versions
+
+For Python 1.4 which is based off of the grammar for 1.5 though there
+are number of changes, about 6 grammar rules. Later versions of though
+we start to see larger upheaval and at certain places, especially
+those where new opcodes are introduced, especially those that change
+the way calls or exceptions get handled, we have major upheaval in the
+grammar. It is not just that some rules get added, but we also need to
+*remove* some grammar rules as well.
+
+I have been largely managing this as incremental differences between versions.
+However in the future I am leaning more towards totally separate grammars.
+A well constructed grammar doesn't need to be that large.
+
+When starting out a new version, we can just copy the grammar from the
+prior version.  Within a Python version though, I am breaking these
+into composable pieces. In particular the grammar for handling what
+can appear as the body of a lambda, is a subset of the full Python
+language. The language allowed in an ``eval`` is also a subset of the
+full Python language, as are what can appear in the various
+compilation modes like "single" versus "exec".
+
+Another nice natural self-contain grammar section is what can appear
+in list comprehensions and generators. The bodies of these are
+generally represented in a self-contained code block.
+
+Often in decompilation you may be interested not just in decompiling
+the entire code but you may be interested in only focusing on a
+specific part of the code. And if there is a problem in decompiling
+the entire piece of code, having these smaller breaking points can be
+of assistance.
+
+Other Modularity
+----------------
+
+Above we have mentioned the need for separate grammars or to isolate
+these per versions. But there are other major pieces that make up this
+decompiler. In particular there is a scanner and the source code
+generation part.
+
+Even though differences in version that occur in disassembly are
+handled by ``xdis``, we still have to do conversion of that to a token
+stream for parsing. So the scanners are again broken out per version
+with various OO mechanisms for reusing code. The same is true for
+source code generation.
+
+
+Expanding decompiler availability to multiple Python Versions
+--------------------------------------------------------------
+
+Above we mention decompiling multiple versions of bytecode from a
+single Python interpreter. We talk about having the decompiler
+runnable from multiple versions of Python, independent of the set of
+bytecode that the decompiler supports.
+
+
+There are slight advantages in having a decompiler that runs the same
+version as the code you are decompiling. The most obvious one is that
+it makes it easy to test to see whether the decompilation correct
+because you can run the decompiled code. Python comes with a suite of
+Python programs that check themselves and that aspects of Python are
+implemented correctly. These also make excellent programs to check
+whether a program has decompiled correctly.
+
+Aside from this, debugging can be easier as well. To assist
+understanding bytecode and single stepping it see `x-python
+<https://pypi.org/project/x-python/>`_ and the debugger for it
+`trepan-xpy <https://pypi.org/project/trepanxpy/>`_.
+
+Handling Language Drift
+-----------------------
+
+Given the desirability of having this code running on logs of Python
+versions, how can we get this done?
+
+The solution used here is to have several git branches of the
+code. Right now there are 3 branches. Each branch handles works across
+3 or so different releases of Python. In particular one branch handles
+Python 2.4 to 2.7 Another handles Python 3.3 to 3.5, and the master
+branch handles 3.6 to 3.10. (Again note that the 3.9 and 3.10
+decompilers do not decompile Python 3.9 or 3.10, but they do handle
+bytecode for all earlier versions.)
+
+
+Cool features of the Parser
+===========================
+
+* reduction rule checking
+* numbering tokens
+* showing a stack of completions
+
+Cool features Semantic Analysis
+===============================
+
+* ``--tree++`` (``-T``) option
+* showing precedence
+* See `Adding a tree transformation phase to uncompyle6 <https://github.com/rocky/python-uncompyle6/wiki/Adding-a-tree-transformation-phase-to-uncompyle6>`_
+* following AST
+* Fragment deparsing

PKG-INFO

@@ -1,10 +1,355 @@
-Metadata-Version: 2.0
+Metadata-Version: 1.1
 Name: uncompyle6
-Version: 2.0.1
-Summary: Python byte-code to source-code converter
-Home-page: http://github.com/rocky/python-uncompyle6
-Author: Rocky
+Version: 3.9.1
+Summary: Python cross-version byte-code decompiler
+Home-page: https://github.com/rocky/python-uncompyle6/
+Author: Rocky Bernstein, Hartmut Goebel, John Aycock, and others
 Author-email: rb@dustyfeet.com
-License: MIT
-Description: UNKNOWN
+License: GPL3
+Description: |buildstatus|  |Pypi Installs| |Latest Version| |Supported Python Versions|
+        
+        |packagestatus|
+        
+        .. contents::
+        
+        uncompyle6
+        ==========
+        
+        A native Python cross-version decompiler and fragment decompiler.
+        The successor to decompyle, uncompyle, and uncompyle2.
+        
+        
+        Introduction
+        ------------
+        
+        *uncompyle6* translates Python bytecode back into equivalent Python
+        source code. It accepts bytecodes from Python version 1.0 to version
+        3.8, spanning over 24 years of Python releases. We include Dropbox's
+        Python 2.5 bytecode and some PyPy bytecodes.
+        
+        Why this?
+        ---------
+        
+        Ok, I'll say it: this software is amazing. It is more than your
+        normal hacky decompiler. Using compiler_ technology, the program
+        creates a parse tree of the program from the instructions; nodes at
+        the upper levels that look a little like what might come from a Python
+        AST. So we can really classify and understand what's going on in
+        sections of Python bytecode.
+        
+        Building on this, another thing that makes this different from other
+        CPython bytecode decompilers is the ability to deparse just
+        *fragments* of source code and give source-code information around a
+        given bytecode offset.
+        
+        I use the tree fragments to deparse fragments of code *at run time*
+        inside my trepan_ debuggers_. For that, bytecode offsets are recorded
+        and associated with fragments of the source code. This purpose,
+        although compatible with the original intention, is yet a little bit
+        different.  See this_ for more information.
+        
+        Python fragment deparsing given an instruction offset is useful in
+        showing stack traces and can be incorporated into any program that
+        wants to show a location in more detail than just a line number at
+        runtime.  This code can be also used when source-code information does
+        not exist and there is just bytecode. Again, my debuggers make use of
+        this.
+        
+        There were (and still are) a number of decompyle, uncompyle,
+        uncompyle2, uncompyle3 forks around. Many of them come basically from
+        the same code base, and (almost?) all of them are no longer actively
+        maintained. One was really good at decompiling Python 1.5-2.3, another
+        really good at Python 2.7, but that only. Another handles Python 3.2
+        only; another patched that and handled only 3.3.  You get the
+        idea. This code pulls all of these forks together and *moves
+        forward*. There is some serious refactoring and cleanup in this code
+        base over those old forks. Even more experimental refactoring is going
+        on in decompyle3_.
+        
+        This demonstrably does the best in decompiling Python across all
+        Python versions. And even when there is another project that only
+        provides decompilation for subset of Python versions, we generally do
+        demonstrably better for those as well.
+        
+        How can we tell? By taking Python bytecode that comes distributed with
+        that version of Python and decompiling these.  Among those that
+        successfully decompile, we can then make sure the resulting programs
+        are syntactically correct by running the Python interpreter for that
+        bytecode version.  Finally, in cases where the program has a test for
+        itself, we can run the check on the decompiled code.
+        
+        We use an automated processes to find bugs. In the issue trackers for
+        other decompilers, you will find a number of bugs we've found along
+        the way. Very few to none of them are fixed in the other decompilers.
+        
+        Requirements
+        ------------
+        
+        The code in the git repository can be run from Python 2.4 to the
+        latest Python version, with the exception of Python 3.0 through
+        3.2. Volunteers are welcome to address these deficiencies if there a
+        desire to do so.
+        
+        The way it does this though is by segregating consecutive Python versions into
+        git branches:
+        
+        master
+           Python 3.6 and up (uses type annotations)
+        python-3.3-to-3.5
+           Python 3.3 through 3.5 (Generic Python 3)
+        python-2.4
+           Python 2.4 through 2.7 (Generic Python 2)
+        
+        PyPy 3-2.4 and later works as well.
+        
+        The bytecode files it can read have been tested on Python
+        bytecodes from versions 1.4, 2.1-2.7, and 3.0-3.8 and later PyPy
+        versions.
+        
+        Installation
+        ------------
+        
+        You can install from PyPI using the name ``uncompyle6``::
+        
+            pip install uncompyle6
+        
+        
+        To install from source code, this project uses setup.py, so it follows the standard Python routine::
+        
+            $ pip install -e .  # set up to run from source tree
+        
+        or::
+        
+            $ python setup.py install # may need sudo
+        
+        A GNU Makefile is also provided so :code:`make install` (possibly as root or
+        sudo) will do the steps above.
+        
+        Running Tests
+        -------------
+        
+        ::
+        
+           make check
+        
+        A GNU makefile has been added to smooth over setting running the right
+        command, and running tests from fastest to slowest.
+        
+        If you have remake_ installed, you can see the list of all tasks
+        including tests via :code:`remake --tasks`
+        
+        
+        Usage
+        -----
+        
+        Run
+        
+        ::
+        
+        $ uncompyle6 *compiled-python-file-pyc-or-pyo*
+        
+        For usage help:
+        
+        ::
+        
+           $ uncompyle6 -h
+        
+        Verification
+        ------------
+        
+        In older versions of Python it was possible to verify bytecode by
+        decompiling bytecode, and then compiling using the Python interpreter
+        for that bytecode version. Having done this, the bytecode produced
+        could be compared with the original bytecode. However as Python's code
+        generation got better, this no longer was feasible.
+        
+        If you want Python syntax verification of the correctness of the
+        decompilation process, add the :code:`--syntax-verify` option. However since
+        Python syntax changes, you should use this option if the bytecode is
+        the right bytecode for the Python interpreter that will be checking
+        the syntax.
+        
+        You can also cross compare the results with another version of
+        `uncompyle6` since there are sometimes regressions in decompiling
+        specific bytecode as the overall quality improves.
+        
+        For Python 3.7 and 3.8, the code in decompyle3_ is generally
+        better.
+        
+        Or try specific another python decompiler like uncompyle2_, unpyc37_,
+        or pycdc_.  Since the later two work differently, bugs here often
+        aren't in that, and vice versa.
+        
+        There is an interesting class of these programs that is readily
+        available give stronger verification: those programs that when run
+        test themselves. Our test suite includes these.
+        
+        And Python comes with another a set of programs like this: its test
+        suite for the standard library. We have some code in :code:`test/stdlib` to
+        facilitate this kind of checking too.
+        
+        Known Bugs/Restrictions
+        -----------------------
+        
+        The biggest known and possibly fixable (but hard) problem has to do
+        with handling control flow. (Python has probably the most diverse and
+        screwy set of compound statements I've ever seen; there
+        are "else" clauses on loops and try blocks that I suspect many
+        programmers don't know about.)
+        
+        All of the Python decompilers that I have looked at have problems
+        decompiling Python's control flow. In some cases we can detect an
+        erroneous decompilation and report that.
+        
+        Python support is pretty good for Python 2
+        
+        On the lower end of Python versions, decompilation seems pretty good although
+        we don't have any automated testing in place for Python's distributed tests.
+        Also, we don't have a Python interpreter for versions 1.6, and 2.0.
+        
+        In the Python 3 series, Python support is strongest around 3.4 or
+        3.3 and drops off as you move further away from those versions. Python
+        3.0 is weird in that it in some ways resembles 2.6 more than it does
+        3.1 or 2.7. Python 3.6 changes things drastically by using word codes
+        rather than byte codes. As a result, the jump offset field in a jump
+        instruction argument has been reduced. This makes the :code:`EXTENDED_ARG`
+        instructions are now more prevalent in jump instruction; previously
+        they had been rare.  Perhaps to compensate for the additional
+        :code:`EXTENDED_ARG` instructions, additional jump optimization has been
+        added. So in sum handling control flow by ad hoc means as is currently
+        done is worse.
+        
+        Between Python 3.5, 3.6, 3.7 there have been major changes to the
+        :code:`MAKE_FUNCTION` and :code:`CALL_FUNCTION` instructions.
+        
+        Python 3.8 removes :code:`SETUP_LOOP`, :code:`SETUP_EXCEPT`,
+        :code:`BREAK_LOOP`, and :code:`CONTINUE_LOOP`, instructions which may
+        make control-flow detection harder, lacking the more sophisticated
+        control-flow analysis that is planned. We'll see.
+        
+        Currently not all Python magic numbers are supported. Specifically in
+        some versions of Python, notably Python 3.6, the magic number has
+        changes several times within a version.
+        
+        **We support only released versions, not candidate versions.** Note
+        however that the magic of a released version is usually the same as
+        the *last* candidate version prior to release.
+        
+        There are also customized Python interpreters, notably Dropbox,
+        which use their own magic and encrypt bytecode. With the exception of
+        the Dropbox's old Python 2.5 interpreter this kind of thing is not
+        handled.
+        
+        We also don't handle PJOrion_ or otherwise obfuscated code. For
+        PJOrion try: PJOrion Deobfuscator_ to unscramble the bytecode to get
+        valid bytecode before trying this tool; pydecipher_ might help with that.
+        
+        This program can't decompile Microsoft Windows EXE files created by
+        Py2EXE_, although we can probably decompile the code after you extract
+        the bytecode properly. `Pydeinstaller <https://github.com/charles-dyfis-net/pydeinstaller>`_ may help with unpacking Pyinstaller bundlers.
+        
+        Handling pathologically long lists of expressions or statements is
+        slow. We don't handle Cython_ or MicroPython which don't use bytecode.
+        
+        There are numerous bugs in decompilation. And that's true for every
+        other CPython decompiler I have encountered, even the ones that
+        claimed to be "perfect" on some particular version like 2.4.
+        
+        As Python progresses decompilation also gets harder because the
+        compilation is more sophisticated and the language itself is more
+        sophisticated. I suspect that attempts there will be fewer ad-hoc
+        attempts like unpyc37_ (which is based on a 3.3 decompiler) simply
+        because it is harder to do so. The good news, at least from my
+        standpoint, is that I think I understand what's needed to address the
+        problems in a more robust way. But right now until such time as
+        project is better funded, I do not intend to make any serious effort
+        to support Python versions 3.8 or 3.9, including bugs that might come
+        in. I imagine at some point I may be interested in it.
+        
+        You can easily find bugs by running the tests against the standard
+        test suite that Python uses to check itself. At any given time, there are
+        dozens of known problems that are pretty well isolated and that could
+        be solved if one were to put in the time to do so. The problem is that
+        there aren't that many people who have been working on bug fixing.
+        
+        Some of the bugs in 3.7 and 3.8 are simply a matter of back-porting
+        the fixes in decompyle3. Volunteers are welcome to do so.
+        
+        You may run across a bug, that you want to report. Please do so after
+        reading `How to report a bug
+        <https://github.com/rocky/python-uncompyle6/blob/master/HOW-TO-REPORT-A-BUG.md>`_ and
+        follow the `instructions when opening an issue <https://github.com/rocky/python-uncompyle6/issues/new?assignees=&labels=&template=bug-report.md>`_.
+        
+        Be aware that it might not get my attention for a while. If you
+        sponsor or support the project in some way, I'll prioritize your
+        issues above the queue of other things I might be doing instead.
+        
+        See Also
+        --------
+        
+        * https://github.com/rocky/python-decompile3 : Much smaller and more modern code, focusing on 3.7 and 3.8. Changes in that will get migrated back here.
+        * https://code.google.com/archive/p/unpyc3/ : supports Python 3.2 only. The above projects use a different decompiling technique than what is used here. Currently unmaintained.
+        * https://github.com/figment/unpyc3/ : fork of above, but supports Python 3.3 only. Includes some fixes like supporting function annotations. Currently unmaintained.
+        * https://github.com/wibiti/uncompyle2 : supports Python 2.7 only, but does that fairly well. There are situations where :code:`uncompyle6` results are incorrect while :code:`uncompyle2` results are not, but more often uncompyle6 is correct when uncompyle2 is not. Because :code:`uncompyle6` adheres to accuracy over idiomatic Python, :code:`uncompyle2` can produce more natural-looking code when it is correct. Currently :code:`uncompyle2` is lightly maintained. See its issue `tracker <https://github.com/wibiti/uncompyle2/issues>`_ for more details.
+        * `How to report a bug <https://github.com/rocky/python-uncompyle6/blob/master/HOW-TO-REPORT-A-BUG.md>`_
+        * The HISTORY_ file.
+        * https://github.com/rocky/python-xdis : Cross Python version disassembler
+        * https://github.com/rocky/python-xasm : Cross Python version assembler
+        * https://github.com/rocky/python-uncompyle6/wiki : Wiki Documents which describe the code and aspects of it in more detail
+        * https://github.com/zrax/pycdc : The README for this C++ code says it aims to support all versions of Python. You can aim your slign shot for the moon too, but I doubt you are going to hit it. This code is best for Python versions around 2.7 and 3.3 when the code was initially developed. Accuracy for current versions of Python3 and early versions of Python is lacking. Without major effort, it is unlikely it can be made to support current Python 3. See its `issue tracker <https://github.com/zrax/pycdc/issues>`_ for details. Currently lightly maintained.
+        
+        
+        .. _Cython: https://en.wikipedia.org/wiki/Cython
+        .. _trepan: https://pypi.python.org/pypi/trepan3k
+        .. _compiler: https://github.com/rocky/python-uncompyle6/wiki/How-does-this-code-work%3F
+        .. _HISTORY: https://github.com/rocky/python-uncompyle6/blob/master/HISTORY.md
+        .. _report_bug: https://github.com/rocky/python-uncompyle6/blob/master/HOW-TO-REPORT-A-BUG.md
+        .. _debuggers: https://pypi.python.org/pypi/trepan3k
+        .. _remake: https://bashdb.sf.net/remake
+        .. _pycdc: https://github.com/zrax/pycdc
+        .. _decompyle3: https://github.com/rocky/python-decompile3
+        .. _uncompyle2: https://github.com/wibiti/uncompyle2
+        .. _unpyc37: https://github.com/andrew-tavera/unpyc37
+        .. _this: https://github.com/rocky/python-uncompyle6/wiki/Deparsing-technology-and-its-use-in-exact-location-reporting
+        .. |buildstatus| image:: https://travis-ci.org/rocky/python-uncompyle6.svg
+        		 :target: https://travis-ci.org/rocky/python-uncompyle6
+        .. |packagestatus| image:: https://repology.org/badge/vertical-allrepos/python:uncompyle6.svg
+        		 :target: https://repology.org/project/python:uncompyle6/versions
+        .. _PJOrion: http://www.koreanrandom.com/forum/topic/15280-pjorion-%D1%80%D0%B5%D0%B4%D0%B0%D0%BA%D1%82%D0%B8%D1%80%D0%BE%D0%B2%D0%B0%D0%BD%D0%B8%D0%B5-%D0%BA%D0%BE%D0%BC%D0%BF%D0%B8%D0%BB%D1%8F%D1%86%D0%B8%D1%8F-%D0%B4%D0%B5%D0%BA%D0%BE%D0%BC%D0%BF%D0%B8%D0%BB%D1%8F%D1%86%D0%B8%D1%8F-%D0%BE%D0%B1%D1%84
+        .. _pydecipher: https://github.com/mitre/pydecipher
+        .. _Deobfuscator: https://github.com/extremecoders-re/PjOrion-Deobfuscator
+        .. _Py2EXE: https://en.wikipedia.org/wiki/Py2exe
+        .. |Supported Python Versions| image:: https://img.shields.io/pypi/pyversions/uncompyle6.svg
+        .. |Latest Version| image:: https://badge.fury.io/py/uncompyle6.svg
+        		 :target: https://badge.fury.io/py/uncompyle6
+        .. |Pypi Installs| image:: https://pepy.tech/badge/uncompyle6/month
+        
+        
 Platform: UNKNOWN
+Classifier: Development Status :: 5 - Production/Stable
+Classifier: Intended Audience :: Developers
+Classifier: License :: OSI Approved :: GNU General Public License v3 (GPLv3)
+Classifier: Operating System :: OS Independent
+Classifier: Programming Language :: Python
+Classifier: Programming Language :: Python :: 2
+Classifier: Programming Language :: Python :: 2.4
+Classifier: Programming Language :: Python :: 2.5
+Classifier: Programming Language :: Python :: 2.6
+Classifier: Programming Language :: Python :: 2.7
+Classifier: Programming Language :: Python :: 3
+Classifier: Programming Language :: Python :: 3.0
+Classifier: Programming Language :: Python :: 3.1
+Classifier: Programming Language :: Python :: 3.2
+Classifier: Programming Language :: Python :: 3.3
+Classifier: Programming Language :: Python :: 3.4
+Classifier: Programming Language :: Python :: 3.5
+Classifier: Programming Language :: Python :: 3.6
+Classifier: Programming Language :: Python :: 3.7
+Classifier: Programming Language :: Python :: 3.8
+Classifier: Programming Language :: Python :: 3.9
+Classifier: Programming Language :: Python :: 3.10
+Classifier: Programming Language :: Python :: 3.11
+Classifier: Programming Language :: Python :: 3.12
+Classifier: Programming Language :: Python :: Implementation :: PyPy
+Classifier: Topic :: Software Development :: Debuggers
+Classifier: Topic :: Software Development :: Libraries :: Python Modules

README.rst

@@ -1,4 +1,8 @@
-|buildstatus| |Latest Version| |Supported Python Versions|
+|buildstatus|  |Pypi Installs| |Latest Version| |Supported Python Versions|
+
+|packagestatus|
+
+.. contents::
 
 uncompyle6
 ==========
@@ -6,14 +10,16 @@ uncompyle6
 A native Python cross-version decompiler and fragment decompiler.
 The successor to decompyle, uncompyle, and uncompyle2.
 
+I gave a talk on this at `BlackHat Asia 2024 <https://youtu.be/H-7ZNrpsV50?si=nOaixgYHr7RbILVS>`_.
+
 
 Introduction
 ------------
 
 *uncompyle6* translates Python bytecode back into equivalent Python
-source code. It accepts bytecodes from Python version 1.3 to version
-3.7, spanning over 22 years of Python releases. We include Dropbox's
-Python 2.5 bytecode and some PyPy bytecode.
+source code. It accepts bytecodes from Python version 1.0 to version
+3.8, spanning over 24 years of Python releases. We include Dropbox's
+Python 2.5 bytecode and some PyPy bytecodes.
 
 Why this?
 ---------
@@ -37,21 +43,22 @@ although compatible with the original intention, is yet a little bit
 different.  See this_ for more information.
 
 Python fragment deparsing given an instruction offset is useful in
-showing stack traces and can be encorporated into any program that
+showing stack traces and can be incorporated into any program that
 wants to show a location in more detail than just a line number at
 runtime.  This code can be also used when source-code information does
 not exist and there is just bytecode. Again, my debuggers make use of
 this.
 
 There were (and still are) a number of decompyle, uncompyle,
-uncompyle2, uncompyle3 forks around. Almost all of them come basically
-from the same code base, and (almost?) all of them are no longer
-actively maintained. One was really good at decompiling Python 1.5-2.3
-or so, another really good at Python 2.7, but that only. Another
-handles Python 3.2 only; another patched that and handled only 3.3.
-You get the idea. This code pulls all of these forks together and
-*moves forward*. There is some serious refactoring and cleanup in this
-code base over those old forks.
+uncompyle2, uncompyle3 forks around. Many of them come basically from
+the same code base, and (almost?) all of them are no longer actively
+maintained. One was really good at decompiling Python 1.5-2.3, another
+really good at Python 2.7, but that only. Another handles Python 3.2
+only; another patched that and handled only 3.3.  You get the
+idea. This code pulls all of these forks together and *moves
+forward*. There is some serious refactoring and cleanup in this code
+base over those old forks. Even more experimental refactoring is going
+on in decompyle3_.
 
 This demonstrably does the best in decompiling Python across all
 Python versions. And even when there is another project that only
@@ -65,36 +72,55 @@ are syntactically correct by running the Python interpreter for that
 bytecode version.  Finally, in cases where the program has a test for
 itself, we can run the check on the decompiled code.
 
-We are serious about testing, and use automated processes to find
-bugs. In the issue trackers for other decompilers, you will find a
-number of bugs we've found along the way. Very few to none of them are
-fixed in the other decompilers.
+We use an automated processes to find bugs. In the issue trackers for
+other decompilers, you will find a number of bugs we've found along
+the way. Very few to none of them are fixed in the other decompilers.
 
 Requirements
 ------------
 
-The code here can be run on Python versions 2.6 or later, PyPy 3-2.4,
-or PyPy-5.0.1.  Python versions 2.4-2.7 are supported in the
-python-2.4 branch.  The bytecode files it can read have been tested on
-Python bytecodes from versions 1.4, 2.1-2.7, and 3.0-3.6 and the
-above-mentioned PyPy versions.
+The code in the git repository can be run from Python 2.4 to the
+latest Python version, with the exception of Python 3.0 through
+3.2. Volunteers are welcome to address these deficiencies if there a
+desire to do so.
+
+The way it does this though is by segregating consecutive Python versions into
+git branches:
+
+master
+   Python 3.6 and up (uses type annotations)
+python-3.3-to-3.5
+   Python 3.3 through 3.5 (Generic Python 3)
+python-2.4
+   Python 2.4 through 2.7 (Generic Python 2)
+
+PyPy 3-2.4 and later works as well.
+
+The bytecode files it can read have been tested on Python
+bytecodes from versions 1.4, 2.1-2.7, and 3.0-3.8 and later PyPy
+versions.
 
 Installation
 ------------
 
-This uses setup.py, so it follows the standard Python routine:
+You can install from PyPI using the name ``uncompyle6``::
 
-::
+    pip install uncompyle6
 
-    pip install -e .  # set up to run from source tree
-                      # Or if you want to install instead
-    python setup.py install # may need sudo
 
-A GNU makefile is also provided so :code:`make install` (possibly as root or
+To install from source code, this project uses setup.py, so it follows the standard Python routine::
+
+    $ pip install -e .  # set up to run from source tree
+
+or::
+
+    $ python setup.py install # may need sudo
+
+A GNU Makefile is also provided so :code:`make install` (possibly as root or
 sudo) will do the steps above.
 
-Testing
--------
+Running Tests
+-------------
 
 ::
 
@@ -122,16 +148,39 @@ For usage help:
 
    $ uncompyle6 -h
 
-If you want strong verification of the correctness of the
-decompilation process, add the `--verify` option. But there are
-situations where this will indicate a failure, although the generated
-program is semantically equivalent. Using option `--weak-verify` will
-tell you if there is something definitely wrong. Generally, large
-swaths of code are decompiled correctly, if not the entire program.
+Verification
+------------
 
-You can also cross compare the results with pycdc_ . Since they work
-differently, bugs here often aren't in that, and vice versa.
+In older versions of Python it was possible to verify bytecode by
+decompiling bytecode, and then compiling using the Python interpreter
+for that bytecode version. Having done this, the bytecode produced
+could be compared with the original bytecode. However as Python's code
+generation got better, this no longer was feasible.
 
+If you want Python syntax verification of the correctness of the
+decompilation process, add the :code:`--syntax-verify` option. However since
+Python syntax changes, you should use this option if the bytecode is
+the right bytecode for the Python interpreter that will be checking
+the syntax.
+
+You can also cross compare the results with another version of
+*uncompyle6* since there are sometimes regressions in decompiling
+specific bytecode as the overall quality improves.
+
+For Python 3.7 and 3.8, the code in decompyle3_ is generally
+better.
+
+Or try specific another python decompiler like uncompyle2_, unpyc37_,
+or pycdc_.  Since the later two work differently, bugs here often
+aren't in that, and vice versa.
+
+There is an interesting class of these programs that is readily
+available give stronger verification: those programs that when run
+test themselves. Our test suite includes these.
+
+And Python comes with another a set of programs like this: its test
+suite for the standard library. We have some code in :code:`test/stdlib` to
+facilitate this kind of checking too.
 
 Known Bugs/Restrictions
 -----------------------
@@ -146,98 +195,129 @@ All of the Python decompilers that I have looked at have problems
 decompiling Python's control flow. In some cases we can detect an
 erroneous decompilation and report that.
 
-In older versions of Python it was possible to verify bytecode by
-decompiling bytecode, and then compiling using the Python interpreter
-for that bytecode version. Having done this the bytecode produced
-could be compared with the original bytecode. However as Python's code
-generation got better, this is no longer feasible.
+Python support is pretty good for Python 2
 
-There verification that we use that doesn't check bytecode for
-equivalence but does check to see if the resulting decompiled source
-is a valid Python program by running the Python interpreter. Because
-the Python language has changed so much, for best results you should
-use the same Python version in checking as was used in creating the
-bytecode.
+On the lower end of Python versions, decompilation seems pretty good although
+we don't have any automated testing in place for Python's distributed tests.
+Also, we don't have a Python interpreter for versions 1.6, and 2.0.
 
-There are however an interesting class of these programs that is
-readily available give stronger verification: those programs that
-when run check some computation, or even better themselves.
-
-And already Python has a set of programs like this: the test suite
-for the standard library that comes with Python. We have some
-code in `test/stdlib` to facilitate this kind of checking.
-
-Python support is strongest in Python 2 for 2.7 and drops off as you
-get further away from that. Support is also probably pretty good for
-python 2.3-2.4 since a lot of the goodness of early the version of the
-decompiler from that era has been preserved (and Python compilation in
-that era was minimal)
-
-There is some work to do on the lower end Python versions which is
-more difficult for us to handle since we don't have a Python
-interpreter for versions 1.6, and 2.0.
-
-In the Python 3 series, Python support is is strongest around 3.4 or
+In the Python 3 series, Python support is strongest around 3.4 or
 3.3 and drops off as you move further away from those versions. Python
 3.0 is weird in that it in some ways resembles 2.6 more than it does
 3.1 or 2.7. Python 3.6 changes things drastically by using word codes
 rather than byte codes. As a result, the jump offset field in a jump
-instruction argument has been reduced. This makes the `EXTENDED_ARG`
+instruction argument has been reduced. This makes the :code:`EXTENDED_ARG`
 instructions are now more prevalent in jump instruction; previously
 they had been rare.  Perhaps to compensate for the additional
-`EXTENDED_ARG` instructions, additional jump optimization has been
+:code:`EXTENDED_ARG` instructions, additional jump optimization has been
 added. So in sum handling control flow by ad hoc means as is currently
 done is worse.
 
-Between Python 3.5, 3.6 and 3.7 there have been major changes to the
-`MAKE_FUNCTION` and `CALL_FUNCTION` instructions.
+Between Python 3.5, 3.6, 3.7 there have been major changes to the
+:code:`MAKE_FUNCTION` and :code:`CALL_FUNCTION` instructions.
+
+Python 3.8 removes :code:`SETUP_LOOP`, :code:`SETUP_EXCEPT`,
+:code:`BREAK_LOOP`, and :code:`CONTINUE_LOOP`, instructions which may
+make control-flow detection harder, lacking the more sophisticated
+control-flow analysis that is planned. We'll see.
 
 Currently not all Python magic numbers are supported. Specifically in
 some versions of Python, notably Python 3.6, the magic number has
-changes several times within a version. We support only the released
-magic. There are also customized Python interpreters, notably Dropbox,
-which use their own magic and encrypt bytcode. With the exception of
+changes several times within a version.
+
+**We support only released versions, not candidate versions.** Note
+however that the magic of a released version is usually the same as
+the *last* candidate version prior to release.
+
+There are also customized Python interpreters, notably Dropbox,
+which use their own magic and encrypt bytecode. With the exception of
 the Dropbox's old Python 2.5 interpreter this kind of thing is not
 handled.
 
-We also don't handle PJOrion_ obfuscated code. For that try: PJOrion
-Deobfuscator_ to unscramble the bytecode to get valid bytecode before
-trying this tool. This program can't decompile Microsoft Windows EXE
-files created by Py2EXE_, although we can probably decompile the code
-after you extract the bytecode properly. For situations like this, you
-might want to consider a decompilation service like `Crazy Compilers
-<http://www.crazy-compilers.com/decompyle/>`_.  Handling
-pathologically long lists of expressions or statements is slow.
+We also don't handle PJOrion_ or otherwise obfuscated code. For
+PJOrion try: PJOrion Deobfuscator_ to unscramble the bytecode to get
+valid bytecode before trying this tool; pydecipher_ might help with that.
 
+This program can't decompile Microsoft Windows EXE files created by
+Py2EXE_, although we can probably decompile the code after you extract
+the bytecode properly. `Pydeinstaller <https://github.com/charles-dyfis-net/pydeinstaller>`_ may help with unpacking Pyinstaller bundlers.
 
-There is lots to do, so please dig in and help.
+Handling pathologically long lists of expressions or statements is
+slow. We don't handle Cython_ or MicroPython which don't use bytecode.
+
+There are numerous bugs in decompilation. And that's true for every
+other CPython decompiler I have encountered, even the ones that
+claimed to be "perfect" on some particular version like 2.4.
+
+As Python progresses decompilation also gets harder because the
+compilation is more sophisticated and the language itself is more
+sophisticated. I suspect that attempts there will be fewer ad-hoc
+attempts like unpyc37_ (which is based on a 3.3 decompiler) simply
+because it is harder to do so. The good news, at least from my
+standpoint, is that I think I understand what's needed to address the
+problems in a more robust way. But right now until such time as
+project is better funded, I do not intend to make any serious effort
+to support Python versions 3.8 or 3.9, including bugs that might come
+in. I imagine at some point I may be interested in it.
+
+You can easily find bugs by running the tests against the standard
+test suite that Python uses to check itself. At any given time, there are
+dozens of known problems that are pretty well isolated and that could
+be solved if one were to put in the time to do so. The problem is that
+there aren't that many people who have been working on bug fixing.
+
+Some of the bugs in 3.7 and 3.8 are simply a matter of back-porting
+the fixes in *decompyle3*. Any volunteers?
+
+You may run across a bug, that you want to report. Please do so after
+reading `How to report a bug
+<https://github.com/rocky/python-uncompyle6/blob/master/HOW-TO-REPORT-A-BUG.md>`_ and
+follow the `instructions when opening an issue <https://github.com/rocky/python-uncompyle6/issues/new?assignees=&labels=&template=bug-report.md>`_.
+
+Be aware that it might not get my attention for a while. If you
+sponsor or support the project in some way, I'll prioritize your
+issues above the queue of other things I might be doing instead. In
+rare situtations, I can do a hand decompilation of bytecode for a fee.
+However this is expansive, usually beyond what most people are willing
+to spend.
 
 See Also
 --------
 
-* https://github.com/zrax/pycdc : purports to support all versions of Python. It is written in C++ and is most accurate for Python versions around 2.7 and 3.3 when the code was more actively developed. Accuracy for more recent versions of Python 3 and early versions of Python are especially lacking. See its `issue tracker <https://github.com/zrax/pycdc/issues>`_ for details. Currently lightly maintained.
+* https://rocky.github.io/blackhat-asia-2024-additional/all-notes-print.html :  How to Read and Write a High-Level Bytecode Decompiler: ``uncompyle6`` ``decompyle3`` -- BlackHat 2024 Asia (`video <https://www.youtube.com/watch?v=NA77SFncppE>`_). A big thanks to the Organizers and Reviewers for letting me speak. This kind of thing encourages me to work on projects like this.
+* https://github.com/rocky/python-decompile3 : Much smaller and more modern code, focusing on 3.7 and 3.8. Changes in that will get migrated back here.
 * https://code.google.com/archive/p/unpyc3/ : supports Python 3.2 only. The above projects use a different decompiling technique than what is used here. Currently unmaintained.
 * https://github.com/figment/unpyc3/ : fork of above, but supports Python 3.3 only. Includes some fixes like supporting function annotations. Currently unmaintained.
-* https://github.com/wibiti/uncompyle2 : supports Python 2.7 only, but does that fairly well. There are situtations where `uncompyle6` results are incorrect while `uncompyle2` results are not, but more often uncompyle6 is correct when uncompyle2 is not. Because `uncompyle6` adheres to accuracy over idiomatic Python, `uncompyle2` can produce more natural-looking code when it is correct. Currently `uncompyle2` is lightly maintained. See its issue `tracker <https://github.com/wibiti/uncompyle2/issues>`_ for more details
+* https://github.com/wibiti/uncompyle2 : supports Python 2.7 only, but does that fairly well. There are situations where :code:`uncompyle6` results are incorrect while :code:`uncompyle2` results are not, but more often uncompyle6 is correct when uncompyle2 is not. Because :code:`uncompyle6` adheres to accuracy over idiomatic Python, :code:`uncompyle2` can produce more natural-looking code when it is correct. Currently :code:`uncompyle2` is lightly maintained. See its issue `tracker <https://github.com/wibiti/uncompyle2/issues>`_ for more details.
 * `How to report a bug <https://github.com/rocky/python-uncompyle6/blob/master/HOW-TO-REPORT-A-BUG.md>`_
 * The HISTORY_ file.
 * https://github.com/rocky/python-xdis : Cross Python version disassembler
 * https://github.com/rocky/python-xasm : Cross Python version assembler
 * https://github.com/rocky/python-uncompyle6/wiki : Wiki Documents which describe the code and aspects of it in more detail
+* https://github.com/zrax/pycdc : The README for this C++ code says it aims to support all versions of Python. You can aim your slign shot for the moon too, but I doubt you are going to hit it. This code is best for Python versions around 2.7 and 3.3 when the code was initially developed. Accuracy for current versions of Python3 and early versions of Python is lacking. Without major effort, it is unlikely it can be made to support current Python 3. See its `issue tracker <https://github.com/zrax/pycdc/issues>`_ for details. Currently lightly maintained.
 
 
-.. _trepan: https://pypi.python.org/pypi/trepan2
-.. _compiler: https://pypi.python.org/pypi/spark_parser
+.. _Cython: https://en.wikipedia.org/wiki/Cython
+.. _trepan: https://pypi.python.org/pypi/trepan3k
+.. _compiler: https://github.com/rocky/python-uncompyle6/wiki/How-does-this-code-work%3F
 .. _HISTORY: https://github.com/rocky/python-uncompyle6/blob/master/HISTORY.md
+.. _report_bug: https://github.com/rocky/python-uncompyle6/blob/master/HOW-TO-REPORT-A-BUG.md
 .. _debuggers: https://pypi.python.org/pypi/trepan3k
 .. _remake: https://bashdb.sf.net/remake
 .. _pycdc: https://github.com/zrax/pycdc
+.. _decompyle3: https://github.com/rocky/python-decompile3
+.. _uncompyle2: https://github.com/wibiti/uncompyle2
+.. _unpyc37: https://github.com/andrew-tavera/unpyc37
 .. _this: https://github.com/rocky/python-uncompyle6/wiki/Deparsing-technology-and-its-use-in-exact-location-reporting
-.. |buildstatus| image:: https://travis-ci.org/rocky/python-uncompyle6.svg
-		 :target: https://travis-ci.org/rocky/python-uncompyle6
+.. |buildstatus| image:: https://circleci.com/gh/rocky/python-uncompyle6.svg?style=svg
+		 :target: https://app.circleci.com/pipelines/github/rocky/python-uncompyle6
+.. |packagestatus| image:: https://repology.org/badge/vertical-allrepos/python:uncompyle6.svg
+		 :target: https://repology.org/project/python:uncompyle6/versions
 .. _PJOrion: http://www.koreanrandom.com/forum/topic/15280-pjorion-%D1%80%D0%B5%D0%B4%D0%B0%D0%BA%D1%82%D0%B8%D1%80%D0%BE%D0%B2%D0%B0%D0%BD%D0%B8%D0%B5-%D0%BA%D0%BE%D0%BC%D0%BF%D0%B8%D0%BB%D1%8F%D1%86%D0%B8%D1%8F-%D0%B4%D0%B5%D0%BA%D0%BE%D0%BC%D0%BF%D0%B8%D0%BB%D1%8F%D1%86%D0%B8%D1%8F-%D0%BE%D0%B1%D1%84
+.. _pydecipher: https://github.com/mitre/pydecipher
 .. _Deobfuscator: https://github.com/extremecoders-re/PjOrion-Deobfuscator
 .. _Py2EXE: https://en.wikipedia.org/wiki/Py2exe
 .. |Supported Python Versions| image:: https://img.shields.io/pypi/pyversions/uncompyle6.svg
 .. |Latest Version| image:: https://badge.fury.io/py/uncompyle6.svg
 		 :target: https://badge.fury.io/py/uncompyle6
+.. |Pypi Installs| image:: https://pepy.tech/badge/uncompyle6/month

__pkginfo__.py

@@ -1,4 +1,4 @@
-# Copyright (C) 2018 Rocky Bernstein <rocky@gnu.org>
+# Copyright (C) 2018, 2020-2021 2024 Rocky Bernstein <rocky@gnu.org>
 #
 #  This program is free software: you can redistribute it and/or modify
 #  it under the terms of the GNU General Public License as published by
@@ -21,65 +21,89 @@
 # less elegant than having it here with reduced code, albeit there
 # still is some room for improvement.
 
+# Python-version | package | last-version |
+# -----------------------------------------
+# 2.5            | pip     |  1.1         |
+# 2.6            | pip     |  1.5.6       |
+# 2.7            | pip     | 19.2.3       |
+# 2.7            | pip     |  1.2.1       |
+# 3.1            | pip     |  1.5.6       |
+# 3.2            | pip     |  7.1.2       |
+# 3.3            | pip     | 10.0.1       |
+# 3.4            | pip     | 19.1.1       |
+
+import os.path as osp
+
 # Things that change more often go here.
-copyright   = """
-Copyright (C) 2015-2019 Rocky Bernstein <rb@dustyfeet.com>.
+copyright = """
+Copyright (C) 2015-2021, 2024 Rocky Bernstein <rb@dustyfeet.com>.
 """
 
-classifiers =  ['Development Status :: 5 - Production/Stable',
-                'Intended Audience :: Developers',
-                'License :: OSI Approved :: GNU General Public License v3 (GPLv3)',
-                'Operating System :: OS Independent',
-                'Programming Language :: Python',
-                'Programming Language :: Python :: 2.4',
-                'Programming Language :: Python :: 2.5',
-                'Programming Language :: Python :: 2.6',
-                'Programming Language :: Python :: 2.7',
-                'Programming Language :: Python :: 3.0',
-                'Programming Language :: Python :: 3.1',
-                'Programming Language :: Python :: 3.2',
-                'Programming Language :: Python :: 3.3',
-                'Programming Language :: Python :: 3.4',
-                'Programming Language :: Python :: 3.5',
-                'Programming Language :: Python :: 3.6',
-                'Programming Language :: Python :: 3.7',
-                'Topic :: Software Development :: Debuggers',
-                'Topic :: Software Development :: Libraries :: Python Modules',
-                ]
+classifiers = [
+    "Development Status :: 5 - Production/Stable",
+    "Intended Audience :: Developers",
+    "License :: OSI Approved :: GNU General Public License v3 (GPLv3)",
+    "Operating System :: OS Independent",
+    "Programming Language :: Python",
+    "Programming Language :: Python :: 2",
+    "Programming Language :: Python :: 2.4",
+    "Programming Language :: Python :: 2.5",
+    "Programming Language :: Python :: 2.6",
+    "Programming Language :: Python :: 2.7",
+    "Programming Language :: Python :: 3",
+    "Programming Language :: Python :: 3.0",
+    "Programming Language :: Python :: 3.1",
+    "Programming Language :: Python :: 3.2",
+    "Programming Language :: Python :: 3.3",
+    "Programming Language :: Python :: 3.4",
+    "Programming Language :: Python :: 3.5",
+    "Programming Language :: Python :: 3.6",
+    "Programming Language :: Python :: 3.7",
+    "Programming Language :: Python :: 3.8",
+    "Programming Language :: Python :: 3.9",
+    "Programming Language :: Python :: 3.10",
+    "Programming Language :: Python :: 3.11",
+    "Programming Language :: Python :: 3.12",
+    "Programming Language :: Python :: Implementation :: PyPy",
+    "Topic :: Software Development :: Debuggers",
+    "Topic :: Software Development :: Libraries :: Python Modules",
+]
 
 # The rest in alphabetic order
-author             = "Rocky Bernstein, Hartmut Goebel, John Aycock, and others"
-author_email       = "rb@dustyfeet.com"
-entry_points       = {
-    'console_scripts': [
-        'uncompyle6=uncompyle6.bin.uncompile:main_bin',
-        'pydisassemble=uncompyle6.bin.pydisassemble:main',
-    ]}
-ftp_url            = None
-install_requires   = ['spark-parser >= 1.8.7, < 1.9.0',
-                      'xdis >= 4.0.0, < 4.1.0']
+author = "Rocky Bernstein, Hartmut Goebel, John Aycock, and others"
+author_email = "rb@dustyfeet.com"
+entry_points = {
+    "console_scripts": [
+        "uncompyle6=uncompyle6.bin.uncompile:main_bin",
+        "pydisassemble=uncompyle6.bin.pydisassemble:main",
+    ]
+}
+ftp_url = None
+install_requires = ["click", "spark-parser >= 1.8.9, < 1.9.2", "xdis >= 6.1.1, < 6.2.0"]
 
-license            = 'GPL3'
-mailing_list       = 'python-debugger@googlegroups.com'
-modname            = 'uncompyle6'
-py_modules         = None
-short_desc         = 'Python cross-version byte-code decompiler'
-web                = 'https://github.com/rocky/python-uncompyle6/'
+license = "GPL3"
+mailing_list = "python-debugger@googlegroups.com"
+modname = "uncompyle6"
+py_modules = None
+short_desc = "Python cross-version byte-code decompiler"
+web = "https://github.com/rocky/python-uncompyle6/"
 
 # tracebacks in zip files are funky and not debuggable
 zip_safe = True
 
 
-import os.path
 def get_srcdir():
-    filename = os.path.normcase(os.path.dirname(os.path.abspath(__file__)))
-    return os.path.realpath(filename)
+    filename = osp.normcase(osp.dirname(osp.abspath(__file__)))
+    return osp.realpath(filename)
+
 
 srcdir = get_srcdir()
 
+
 def read(*rnames):
-    return open(os.path.join(srcdir, *rnames)).read()
+    return open(osp.join(srcdir, *rnames)).read()
+
 
 # Get info from files; set: long_description and VERSION
-long_description   = ( read("README.rst") + '\n' )
-exec(read('uncompyle6/version.py'))
+long_description = read("README.rst") + "\n"
+exec(read("uncompyle6/version.py"))

admin-tools/.gitignore

@@ -0,0 +1 @@
+/.python-version

admin-tools/README.md

@@ -8,4 +8,15 @@ They are customized to my environment:
 - I have git repos for xdis, and spark parser at the same level as uncompyle6
 
 There may be other rocky-specific things that need customization.
-how-to-make-a-release.txt has overall how I make a release
+how-to-make-a-release.md has overall how I make a release
+
+Since this project uses python over a wide variety of release, some versions
+of projects that should be used for specific Python versions
+
+for 3.2.6:
+   pytest==2.9.2
+
+for 3.1.5
+   pytset==2.1.0
+   py=1.8.0 and comment out line 10 of _builtin.py # callable = callable
+   six==1.10.0

admin-tools/check-2.4-2.7-versions.sh

@@ -1,4 +1,6 @@
 #!/bin/bash
+# Run tests over all Python versions in branch python-2.4-2.7
+set -e
 function finish {
   cd $owd
 }
@@ -6,11 +8,13 @@ owd=$(pwd)
 trap finish EXIT
 
 cd $(dirname ${BASH_SOURCE[0]})
-if ! source ./pyenv-older-versions ; then
+if ! source ./pyenv-2.4-2.7-versions ; then
     exit $?
 fi
 if ! source ./setup-python-2.4.sh ; then
-    exit $?
+    rc=$?
+    finish
+    exit $rc
 fi
 
 cd ..
@@ -21,7 +25,10 @@ for version in $PYVERSIONS; do
     fi
     make clean && python setup.py develop
     if ! make check ; then
+	finish
+	rc=$?
 	exit $?
     fi
     echo === $version ===
 done
+finish

admin-tools/check-3.0-3.2-versions.sh

@@ -0,0 +1,31 @@
+#!/bin/bash
+# Run tests over all Python versions in branch python-3.0-3.2
+set -e
+function finish {
+  cd $owd
+}
+
+owd=$(pwd)
+trap finish EXIT
+
+cd $(dirname ${BASH_SOURCE[0]})
+if ! source ./pyenv-3.0-3.2-versions ; then
+    exit $?
+fi
+if ! source ./setup-python-3.0.sh ; then
+    exit $?
+fi
+
+cd ..
+for version in $PYVERSIONS; do
+    echo --- $version ---
+    if ! pyenv local $version ; then
+	exit $?
+    fi
+    make clean && python setup.py develop
+    if ! make check ; then
+	exit $?
+    fi
+    echo === $version ===
+done
+finish

admin-tools/check-3.3-3.5-versions.sh

@@ -0,0 +1,30 @@
+#!/bin/bash
+# Run tests over all Python versions in branch python-3.3-3.5
+set -e
+function finish {
+  cd $owd
+}
+owd=$(pwd)
+trap finish EXIT
+
+cd $(dirname ${BASH_SOURCE[0]})
+if ! source ./pyenv-3.3-3.5-versions ; then
+    exit $?
+fi
+if ! source ./setup-python-3.3.sh ; then
+    exit $?
+fi
+
+cd ..
+for version in $PYVERSIONS; do
+    echo --- $version ---
+    if ! pyenv local $version ; then
+	exit $?
+    fi
+    make clean && python setup.py develop
+    if ! make check ; then
+	exit $?
+    fi
+    echo === $version ===
+done
+finish

admin-tools/check-newest-versions.sh

@@ -8,7 +8,7 @@ owd=$(pwd)
 trap finish EXIT
 
 cd $(dirname ${BASH_SOURCE[0]})
-if ! source ./pyenv-newer-versions ; then
+if ! source ./pyenv-newest-versions ; then
     exit $?
 fi
 if ! source ./setup-master.sh ; then
@@ -21,8 +21,9 @@ for version in $PYVERSIONS; do
 	exit $?
     fi
     make clean && pip install -e .
-    if ! make check; then
+    if ! make check-short; then
 	exit $?
     fi
     echo === $version ===
 done
+make check

admin-tools/checkout_common.sh

@@ -0,0 +1,21 @@
+# Common checkout routine
+export PATH=$HOME/.pyenv/bin/pyenv:$PATH
+bs=${BASH_SOURCE[0]}
+mydir=$(dirname $bs)
+fulldir=$(readlink -f $mydir)
+
+function setup_version {
+    local repo=$1
+    version=$2
+    echo Running setup $version on $repo ...
+    (cd ../$repo && . ./admin-tools/setup-${version}.sh)
+    return $?
+}
+
+function checkout_finish {
+    branch=$1
+    cd $uncompyle6_owd
+    git checkout $branch && pyenv local $PYTHON_VERSION && git pull
+    rc=$?
+    return $rc
+}

admin-tools/how-to-make-a-release.md

@@ -2,17 +2,17 @@
 **Table of Contents**
 
 - [Get latest sources:](#get-latest-sources)
-- [Change version in uncompyle6/version.py](#change-version-in-uncompyle6versionpy)
+- [Change version in uncompyle6/version.py:](#change-version-in-uncompyle6versionpy)
 - [Update ChangeLog:](#update-changelog)
-- [Update NEWS from ChangeLog:](#update-news-from-changelog)
+- [Update NEWS.md from ChangeLog:](#update-newsmd-from-changelog)
 - [Make sure pyenv is running and check newer versions](#make-sure-pyenv-is-running-and-check-newer-versions)
 - [Switch to python-2.4, sync that up and build that first since it creates a tarball which we don't want.](#switch-to-python-24-sync-that-up-and-build-that-first-since-it-creates-a-tarball-which-we-dont-want)
-- [Update NEWS from master branch](#update-news-from-master-branch)
-- [Check against all versions](#check-against-all-versions)
+- [Check against older versions](#check-against-older-versions)
 - [Make packages and tag](#make-packages-and-tag)
-- [Upload single package and look at Rst Formating](#upload-single-package-and-look-at-rst-formating)
-- [Upload rest of versions](#upload-rest-of-versions)
-- [Push tags:](#push-tags)
+- [Check package on github](#check-package-on-github)
+- [Release on Github](#release-on-github)
+- [Get onto PyPI](#get-onto-pypi)
+- [Update tags:](#update-tags)
 
 <!-- markdown-toc end -->
 # Get latest sources:
@@ -39,7 +39,7 @@
 
 # Make sure pyenv is running and check newer versions
 
-    $ pyenv local && source admin-tools/check-newer-versions.sh
+    $ admin-tools/check-newer-versions.sh
 
 # Switch to python-2.4, sync that up and build that first since it creates a tarball which we don't want.
 
@@ -50,35 +50,51 @@
 
 # Check against older versions
 
-    $ source admin-tools/check-older-versions.sh
+    $ admin-tools/check-older-versions.sh
 
 # Make packages and tag
 
     $ . ./admin-tools/make-dist-older.sh
+	$ pyenv local 3.8.5
+	$ twine check dist/uncompyle6-$VERSION*
     $ git tag release-python-2.4-$VERSION
+    $ ./admin-tools/make-dist-newer.sh
+	$ twine check dist/uncompyle6-$VERSION*
 
-    $ . ./admin-tools/make-dist-newer.sh
+# Check package on github
+
+	$ [[ ! -d /tmp/gittest ]] && mkdir /tmp/gittest; pushd /tmp/gittest
+	$ pyenv local 3.8.3
+	$ pip install -e git://github.com/rocky/python-uncompyle6.git#egg=uncompyle6
+	$ uncompyle6 --help
+	$ pip uninstall uncompyle6
+	$ popd
+
+# Release on Github
 
 Goto https://github.com/rocky/python-uncompyle6/releases
 
-# Upload single package and look at Rst Formating
+Now check the *tagged* release. (Checking the untagged release was previously done).
 
-    $ twine upload dist/uncompyle6-${VERSION}-py3.3.egg
+Todo: turn this into a script in `admin-tools`
 
-# Upload rest of versions
+	$ pushd /tmp/gittest
+	$ pip install -e git://github.com/rocky/python-uncompyle6.git@$VERSION#egg=uncompyle6
+	$ uncompyle6 --help
+	$ pip uninstall uncompyle6
+	$ popd
+
+
+# Get onto PyPI
 
     $ twine upload dist/uncompyle6-${VERSION}*
 
-# Push tags:
+
+# Update tags:
 
     $ git push --tags
+    $ git pull --tags
 
-# Check on a VM
+# Move dist files to uploaded
 
-    $ cd /virtual/vagrant/virtual/vagrant/ubuntu-zesty
-	$ vagrant up
-	$ vagrant ssh
-	$ pyenv local 3.5.2
-	$ pip install --upgrade uncompyle6
-	$ exit
-	$ vagrant halt
+	$ mv -v dist/uncompyle6-${VERSION}* dist/uploaded

admin-tools/make-dist-2.4-2.7.sh

@@ -3,13 +3,13 @@ PACKAGE=uncompyle6
 
 # FIXME put some of the below in a common routine
 function finish {
-  cd $owd
+  cd $make_dist_uncompyle6_owd
 }
-owd=$(pwd)
+make_dist_uncompyle6_owd=$(pwd)
 trap finish EXIT
 
 cd $(dirname ${BASH_SOURCE[0]})
-if ! source ./pyenv-older-versions ; then
+if ! source ./pyenv-2.4-2.7-versions ; then
     exit $?
 fi
 if ! source ./setup-python-2.4.sh ; then
@@ -18,9 +18,14 @@ fi
 
 cd ..
 source $PACKAGE/version.py
-echo $VERSION
+echo $__version__
 
 for pyversion in $PYVERSIONS; do
+    echo --- $pyversion ---
+    if [[ ${pyversion:0:4} == "pypy" ]] ; then
+	echo "$pyversion - PyPy does not get special packaging"
+	continue
+    fi
     if ! pyenv local $pyversion ; then
 	exit $?
     fi
@@ -29,11 +34,16 @@ for pyversion in $PYVERSIONS; do
     python setup.py bdist_egg
 done
 
+pyenv local 2.7.18
+python setup.py bdist_wheel
+mv -v dist/${PACKAGE}-$__version__-py2{.py3,}-none-any.whl
+
 # Pypi can only have one source tarball.
 # Tarballs can get created from the above setup, so make sure to remove them since we want
 # the tarball from master.
 
-tarball=dist/${PACKAGE}-$VERSION-tar.gz
+tarball=dist/${PACKAGE}-${__version_}_-tar.gz
 if [[ -f $tarball ]]; then
-    rm -v dist/${PACKAGE}-$VERSION-tar.gz
+    rm -v dist/${PACKAGE}-${__version__}-tar.gz
 fi
+finish

admin-tools/make-dist-3.0-3.2.sh

@@ -0,0 +1,49 @@
+#!/bin/bash
+PACKAGE=uncompyle6
+
+# FIXME put some of the below in a common routine
+function finish {
+  cd $uncompyle6_30_make_dist_owd
+}
+
+cd $(dirname ${BASH_SOURCE[0]})
+uncompyle6_30_make_dist_owd=$(pwd)
+trap finish EXIT
+
+if ! source ./pyenv-3.0-3.2-versions ; then
+    exit $?
+fi
+if ! source ./setup-python-3.0.sh ; then
+    exit $?
+fi
+
+cd ..
+source $PACKAGE/version.py
+echo $__version__
+
+for pyversion in $PYVERSIONS; do
+    echo --- $pyversion ---
+    if [[ ${pyversion:0:4} == "pypy" ]] ; then
+	echo "$pyversion - PyPy does not get special packaging"
+	continue
+    fi
+    if ! pyenv local $pyversion ; then
+	exit $?
+    fi
+    # pip bdist_egg create too-general wheels. So
+    # we narrow that by moving the generated wheel.
+
+    # Pick out first two number of version, e.g. 3.5.1 -> 35
+    first_two=$(echo $pyversion | cut -d'.' -f 1-2 | sed -e 's/\.//')
+    rm -fr build
+    python setup.py bdist_egg bdist_wheel
+    mv -v dist/${PACKAGE}-$__version__-{py2.py3,py$first_two}-none-any.whl
+    echo === $pyversion ===
+done
+
+python ./setup.py sdist
+tarball=dist/${PACKAGE}-${__version__}.tar.gz
+if [[ -f $tarball ]]; then
+    mv -v $tarball dist/${PACKAGE}_31-${__version__}.tar.gz
+fi
+finish

admin-tools/make-dist-3.3-3.5.sh

@@ -0,0 +1,49 @@
+#!/bin/bash
+PACKAGE=uncompyle6
+
+# FIXME put some of the below in a common routine
+function finish {
+  cd $uncompyle6_33_make_owd
+}
+
+cd $(dirname ${BASH_SOURCE[0]})
+uncompyle6_33_make_owd=$(pwd)
+trap finish EXIT
+
+if ! source ./pyenv-3.3-3.5-versions ; then
+    exit $?
+fi
+if ! source ./setup-python-3.3.sh ; then
+    exit $?
+fi
+
+cd ..
+source $PACKAGE/version.py
+echo $__version__
+
+for pyversion in $PYVERSIONS; do
+    echo --- $pyversion ---
+    if [[ ${pyversion:0:4} == "pypy" ]] ; then
+	echo "$pyversion - PyPy does not get special packaging"
+	continue
+    fi
+    if ! pyenv local $pyversion ; then
+	exit $?
+    fi
+    # pip bdist_egg create too-general wheels. So
+    # we narrow that by moving the generated wheel.
+
+    # Pick out first two number of version, e.g. 3.5.1 -> 35
+    first_two=$(echo $pyversion | cut -d'.' -f 1-2 | sed -e 's/\.//')
+    rm -fr build
+    python setup.py bdist_egg bdist_wheel
+    mv -v dist/${PACKAGE}-$__version__-{py2.py3,py$first_two}-none-any.whl
+    echo === $pyversion ===
+done
+
+python ./setup.py sdist
+tarball=dist/${PACKAGE}-${__version__}.tar.gz
+if [[ -f $tarball ]]; then
+    mv -v $tarball dist/${PACKAGE}_31-${__version__}.tar.gz
+fi
+finish

admin-tools/make-dist-newest.sh

@@ -3,14 +3,14 @@ PACKAGE=uncompyle6
 
 # FIXME put some of the below in a common routine
 function finish {
-  cd $owd
+  cd $make_uncompyle6_newest_owd
 }
 
 cd $(dirname ${BASH_SOURCE[0]})
-owd=$(pwd)
+make_uncompyle6_newest_owd=$(pwd)
 trap finish EXIT
 
-if ! source ./pyenv-newer-versions ; then
+if ! source ./pyenv-newest-versions ; then
     exit $?
 fi
 if ! source ./setup-master.sh ; then
@@ -19,9 +19,14 @@ fi
 
 cd ..
 source $PACKAGE/version.py
-echo $VERSION
+echo $__version__
 
 for pyversion in $PYVERSIONS; do
+    echo --- $pyversion ---
+    if [[ ${pyversion:0:4} == "pypy" ]] ; then
+	echo "$pyversion - PyPy does not get special packaging"
+	continue
+    fi
     if ! pyenv local $pyversion ; then
 	exit $?
     fi
@@ -32,7 +37,8 @@ for pyversion in $PYVERSIONS; do
     first_two=$(echo $pyversion | cut -d'.' -f 1-2 | sed -e 's/\.//')
     rm -fr build
     python setup.py bdist_egg bdist_wheel
-    mv -v dist/${PACKAGE}-$VERSION-{py2.py3,py$first_two}-none-any.whl
+    mv -v dist/${PACKAGE}-$__version__-{py2.py3,py$first_two}-none-any.whl
 done
 
 python ./setup.py sdist
+finish

admin-tools/merge-for-2.4.sh

@@ -0,0 +1,7 @@
+#/bin/bash
+uncompyle6_merge_24_owd=$(pwd)
+cd $(dirname ${BASH_SOURCE[0]})
+if . ./setup-python-2.4.sh; then
+    git merge python-3.0-to-3.2
+fi
+cd $uncompyle6_merge_24_owd

admin-tools/merge-for-3.0.sh

@@ -0,0 +1,7 @@
+#/bin/bash
+uncompyle6_merge_30_owd=$(pwd)
+cd $(dirname ${BASH_SOURCE[0]})
+if . ./setup-python-3.0.sh; then
+    git merge python-3.3-to-3.5
+fi
+cd $uncompyle6_merge_30_owd

admin-tools/merge-for-3.3.sh

@@ -0,0 +1,7 @@
+#/bin/bash
+uncompyle6_merge_33_owd=$(pwd)
+cd $(dirname ${BASH_SOURCE[0]})
+if . ./setup-python-3.3.sh; then
+    git merge python-3.6-to-3.10
+fi
+cd $uncompyle6_merge_33_owd

admin-tools/merge-for-3.6.sh

@@ -0,0 +1,7 @@
+#/bin/bash
+uncompyle6_merge_36_owd=$(pwd)
+cd $(dirname ${BASH_SOURCE[0]})
+if . ./setup-python-3.6.sh; then
+    git merge master
+fi
+cd $uncompyle6_merge_36_owd

admin-tools/pyenv-2.4-2.7-versions

@@ -0,0 +1,9 @@
+# -*- shell-script -*-
+# Sets PYVERSIONS to be pyenv versions that
+# we can use in the python-2.4-to-2.7 branch.
+
+if [[ $0 == ${BASH_SOURCE[0]} ]] ; then
+    echo "This script should be *sourced* rather than run directly through bash"
+    exit 1
+fi
+export PYVERSIONS='2.4.6 2.5.6 2.6.9 2.7.18'

admin-tools/pyenv-3.0-3.2-versions

@@ -6,4 +6,4 @@ if [[ $0 == ${BASH_SOURCE[0]} ]] ; then
     echo "This script should be *sourced* rather than run directly through bash"
     exit 1
 fi
-export PYVERSIONS='2.4.6 2.5.6'
+export PYVERSIONS='3.0.1 3.1.5 3.2.6'

admin-tools/pyenv-3.3-3.5-versions

@@ -0,0 +1,8 @@
+# -*- shell-script -*-
+# Sets PYVERSIONS to be pyenv versions that
+# we can use in the python-3.3-to-3.5 branch.
+if [[ $0 == ${BASH_SOURCE[0]} ]] ; then
+    echo "This script should be *sourced* rather than run directly through bash"
+    exit 1
+fi
+export PYVERSIONS='3.5.10 3.3.7 3.4.10'

admin-tools/pyenv-newest-versions

@@ -0,0 +1,8 @@
+# -*- shell-script -*-
+# Sets PYVERSIONS to be pyenv versions that
+# we can use in the master branch.
+if [[ $0 == ${BASH_SOURCE[0]} ]] ; then
+    echo "This script should be *sourced* rather than run directly through bash"
+    exit 1
+fi
+export PYVERSIONS='3.6.15 pypy3.6-7.3.1 3.7.16 pypy3.7-7.3.9 pypy3.8-7.3.10 pyston-2.3.5 3.8.18'

admin-tools/pyenv-versions

@@ -5,4 +5,4 @@ if [[ $0 == ${BASH_SOURCE[0]} ]] ; then
     echo "This script should be *sourced* rather than run directly through bash"
     exit 1
 fi
-export PYVERSIONS='3.2.6 3.6.8 3.7.2 2.6.9 3.3.7 2.7.15 3.2.6 3.1.5 3.4.8'
+export PYVERSIONS='3.7.13 pyston-2.3.3 3.8.13'

admin-tools/setup-master.sh

@@ -1,22 +1,20 @@
 #!/bin/bash
-PYTHON_VERSION=3.6.5
-
-# FIXME put some of the below in a common routine
-function finish {
-  cd $owd
-}
-
-export PATH=$HOME/.pyenv/bin/pyenv:$PATH
-owd=$(pwd)
+# Check out master branch and dependent development master branches
 bs=${BASH_SOURCE[0]}
 if [[ $0 == $bs ]] ; then
     echo "This script should be *sourced* rather than run directly through bash"
     exit 1
 fi
+
+PYTHON_VERSION=3.12
+
+uncompyle6_owd=$(pwd)
 mydir=$(dirname $bs)
 fulldir=$(readlink -f $mydir)
+cd $mydir
+. ./checkout_common.sh
 cd $fulldir/..
-(cd ../python-spark && git checkout master && pyenv local $PYTHON_VERSION) && git pull && \
-    (cd ../python-xdis && git checkout master && pyenv local $PYTHON_VERSION) && git pull && \
-    git checkout master && pyenv local $PYTHON_VERSION && git pull
-cd $owd
+(cd $fulldir/.. && \
+     setup_version python-spark master && \
+     setup_version python-xdis master )
+checkout_finish master

admin-tools/setup-python-2.4.sh

@@ -1,16 +1,23 @@
 #!/bin/bash
-PYTHON_VERSION=2.4.6
+# Check out python-2.4-to-2.7 and dependent development branches.
 
-owd=$(pwd)
 bs=${BASH_SOURCE[0]}
 if [[ $0 == $bs ]] ; then
     echo "This script should be *sourced* rather than run directly through bash"
     exit 1
 fi
+
+PYTHON_VERSION=2.4
+
+uncompyle6_owd=$(pwd)
 mydir=$(dirname $bs)
 fulldir=$(readlink -f $mydir)
-cd $fulldir/..
-(cd ../python-spark && git checkout python-2.4 && pyenv local $PYTHON_VERSION) && git pull && \
-    (cd ../python-xdis && git checkout python-2.4 && pyenv local $PYTHON_VERSION) && git pull && \
-    git checkout python-2.4 &&  pyenv local $PYTHON_VERSION && git pull
-cd $owd
+cd $mydir
+. ./checkout_common.sh
+
+(cd $fulldir/.. && \
+     setup_version python-spark python-2.4 && \
+     setup_version python-xdis python-2.4)
+
+
+checkout_finish python-2.4-to-2.7

admin-tools/setup-python-3.0.sh

@@ -0,0 +1,20 @@
+#!/bin/bash
+# Check out python-3.0-to-3.2 and dependent development branches.
+bs=${BASH_SOURCE[0]}
+if [[ $0 == $bs ]] ; then
+    echo "This script should be *sourced* rather than run directly through bash"
+    exit 1
+fi
+
+PYTHON_VERSION=3.0
+
+uncompyle6_owd=$(pwd)
+mydir=$(dirname $bs)
+fulldir=$(readlink -f $mydir)
+cd $mydir
+. ./checkout_common.sh
+(cd $fulldir/.. && \
+     setup_version python-spark python-3.0 && \
+     setup_version python-xdis python-3.0)
+
+checkout_finish python-3.0-to-3.2

admin-tools/setup-python-3.3.sh

@@ -0,0 +1,21 @@
+#!/bin/bash
+# Check out python-3.3-to-3.5 and dependent development branches.
+bs=${BASH_SOURCE[0]}
+if [[ $0 == $bs ]] ; then
+    echo "This script should be *sourced* rather than run directly through bash"
+    exit 1
+fi
+
+PYTHON_VERSION=3.3
+
+uncompyle6_owd=$(pwd)
+mydir=$(dirname $bs)
+cd $mydir
+fulldir=$(readlink -f $mydir)
+. ./checkout_common.sh
+cd $fulldir/..
+(cd $fulldir/.. && \
+     setup_version python-spark python-3.3 && \
+     setup_version python-xdis python-3.3 )
+
+checkout_finish python-3.3-to-3.5

admin-tools/setup-python-3.6.sh

@@ -0,0 +1,21 @@
+#!/bin/bash
+# Check out python-3.6-to-3.10 and dependent development branches.
+bs=${BASH_SOURCE[0]}
+if [[ $0 == $bs ]] ; then
+    echo "This script should be *sourced* rather than run directly through bash"
+    exit 1
+fi
+
+PYTHON_VERSION=3.6
+
+uncompyle6_owd=$(pwd)
+mydir=$(dirname $bs)
+cd $mydir
+fulldir=$(readlink -f $mydir)
+. ./checkout_common.sh
+cd $fulldir/..
+(cd $fulldir/.. && \
+     setup_version python-spark python-3.6 && \
+     setup_version python-xdis python-3.6 )
+
+checkout_finish python-3.6-to-3.10

appveyor.yml

@@ -1,78 +0,0 @@
-environment:
-  global:
-    # SDK v7.0 MSVC Express 2008's SetEnv.cmd script will fail if the
-    # /E:ON and /V:ON options are not enabled in the batch script intepreter
-    # See: http://stackoverflow.com/a/13751649/163740
-    CMD_IN_ENV: "cmd /E:ON /V:ON /C .\\appveyor\\run_with_env.cmd"
-
-  matrix:
-
-    # Pre-installed Python versions, which Appveyor may upgrade to
-    # a later point release.
-    # See: http://www.appveyor.com/docs/installed-software#python
-
-    # - PYTHON: "C:\\Python27"
-    #   PYTHON_VERSION: "2.7.x"
-    #   PYTHON_ARCH: "32"
-
-    - PYTHON: "C:\\Python27-x64"
-      PYTHON_VERSION: "2.7.x"
-      PYTHON_ARCH: "64"
-
-    # - PYTHON: "C:\\Python26"
-    #   PYTHON_VERSION: "2.6.x"
-    #   PYTHON_ARCH: "32"
-
-    # - PYTHON: "C:\\Python26-x64"
-    #   PYTHON_VERSION: "2.6.x"
-    #   PYTHON_ARCH: "64"
-
-install:
-  # We need wheel installed to build wheels
-  - "%PYTHON%\\python.exe -m pip install wheel"
-
-  # Install Python (from the official .msi of http://python.org) and pip when
-  # not already installed.
-  - ps: if (-not(Test-Path($env:PYTHON))) { & appveyor\install.ps1 }
-
-  # Prepend newly installed Python to the PATH of this build (this cannot be
-  # done from inside the powershell script as it would require to restart
-  # the parent CMD process).
-  - "SET PATH=%PYTHON%;%PYTHON%\\Scripts;%PATH%"
-  - "SET HOME=."
-
-  # Check that we have the expected version and architecture for Python
-  - "python --version"
-  - "python -c \"import struct; print(struct.calcsize('P') * 8)\""
-
-  # Upgrade to the latest version of pip to avoid it displaying warnings
-  # about it being out of date.
-  - "%PYTHON%\\python.exe -m pip install --disable-pip-version-check --user --upgrade pip"
-
-  # Install the build dependencies of the project. If some dependencies contain
-  # compiled extensions and are not provided as pre-built wheel packages,
-  # pip will build them from source using the MSVC compiler matching the
-  # target Python version and architecture
-  - "%CMD_IN_ENV% pip install -r requirements.txt"
-
-build_script:
-  # Build the compiled extension
-  - "%CMD_IN_ENV% python setup.py build"
-
-test_script:
-  # Run the project tests
-  - "%CMD_IN_ENV% python test/test_pyenvlib.py --native --weak-verify"
-
-after_test:
-  # If tests are successful, create binary packages for the project.
-  - "%CMD_IN_ENV% python setup.py bdist_wininst"
-  - "%CMD_IN_ENV% python setup.py bdist_msi"
-  - ps: "ls dist"
-
-artifacts:
-  # Archive the generated packages in the ci.appveyor.com build report.
-  - path: dist\*
-
-#on_success:
-#  - TODO: upload the content of dist/*.whl to a public wheelhouse
-#

pyproject.toml

@@ -0,0 +1,65 @@
+[build-system]
+requires = [
+    "setuptools",
+    # "setuptools>=59.6.0", # for 3.6
+]
+
+build-backend = "setuptools.build_meta"
+
+[project]
+authors = [
+    {name = "Rocky Bernstein",	email = "rb@dustyfeet.com"},
+]
+
+name = "uncompyle6"
+description = "Python cross-version byte-code library and disassembler"
+dependencies = [
+    "click",
+    "spark-parser >= 1.8.9, < 1.9.2",
+    "xdis >= 6.1.0, < 6.2.0",
+]
+readme = "README.rst"
+license = {text = "GPL"}
+keywords = ["Python bytecode", "bytecode", "disassembler"]
+classifiers = [
+    "Development Status :: 5 - Production/Stable",
+    "Intended Audience :: Developers",
+    "License :: OSI Approved :: MIT License",
+    "Programming Language :: Python",
+    "Topic :: Software Development :: Libraries :: Python Modules",
+    "Programming Language :: Python :: 2.4",
+    "Programming Language :: Python :: 2.5",
+    "Programming Language :: Python :: 2.6",
+    "Programming Language :: Python :: 2.7",
+    "Programming Language :: Python :: 3.0",
+    "Programming Language :: Python :: 3.1",
+    "Programming Language :: Python :: 3.2",
+    "Programming Language :: Python :: 3.3",
+    "Programming Language :: Python :: 3.4",
+    "Programming Language :: Python :: 3.5",
+    "Programming Language :: Python :: 3.6",
+    "Programming Language :: Python :: 3.7",
+    "Programming Language :: Python :: 3.8",
+    "Programming Language :: Python :: 3.9",
+    "Programming Language :: Python :: 3.10",
+    "Programming Language :: Python :: 3.11",
+    "Programming Language :: Python :: 3.12",
+]
+dynamic = ["version"]
+
+[project.urls]
+Homepage = "https://github.com/rocky/python-uncompyle6"
+Downloads = "https://github.com/rocky/python-uncompyle6/releases"
+
+[project.optional-dependencies]
+dev = [
+    "pre-commit",
+    "pytest",
+]
+
+[project.scripts]
+uncompyle6 = "uncompyle6.bin.uncompile:main_bin"
+uncompyle6-tokenize = "uncompyle6.bin.pydisassemble:main"
+
+[tool.setuptools.dynamic]
+version = {attr = "uncompyle6.version.__version__"}

pytest/Makefile

@@ -1,7 +1,11 @@
 PHONY=check test pytest
+SHELL=/bin/bash
 
 PYTHON ?= python
 
 #: Run all tests
 test check pytest:
-	py.test
+	@PYTHON_VERSION=`$(PYTHON) -V 2>&1 | cut -d ' ' -f 2 | cut -d'.' -f1,2`; \
+	 if [[ $$PYTHON_VERSION > 3.2 ]] || [[ $$PYTHON_VERSION == 2.7 ]] || [[ $$PYTHON_VERSION == 2.6 ]]; then \
+		$(PYTHON) -m pytest .; \
+	fi

pytest/test_build_const_key_map.py

@@ -1,10 +1,10 @@
 import pytest
 # uncompyle6
-from uncompyle6 import PYTHON_VERSION
+from xdis.version_info import PYTHON_VERSION_TRIPLE, IS_PYPY
 from validate import validate_uncompyle
 
 
-@pytest.mark.skipif(PYTHON_VERSION < 3.6, reason='need at least python 3.6')
+@pytest.mark.skipif(PYTHON_VERSION_TRIPLE < (3, 6) or IS_PYPY, reason="need at least Python 3.6 and not PyPY")
 @pytest.mark.parametrize('text', (
     "{0.: 'a', -1: 'b'}",   # BUILD_MAP
     "{'a':'b'}",            # BUILD_MAP

pytest/test_deparse.py

@@ -1,6 +1,5 @@
-import pytest
-from uncompyle6.semantics.fragments import code_deparse as deparse, deparsed_find
-from uncompyle6 import PYTHON_VERSION, PYTHON3
+from uncompyle6.semantics.fragments import code_deparse as deparse
+from xdis.version_info import PYTHON_VERSION_TRIPLE
 
 def map_stmts(x, y):
     x = []
@@ -30,20 +29,18 @@ def list_comp():
     [y for y in range(3)]
 
 def get_parsed_for_fn(fn):
-    code = fn.__code__ if PYTHON3 else fn.func_code
-    return deparse(code, version=PYTHON_VERSION)
+    code = fn.__code__
+    return deparse(code, version=PYTHON_VERSION_TRIPLE)
 
 def check_expect(expect, parsed, fn_name):
     debug = False
     i = 2
     max_expect = len(expect)
-    code = get_parsed_for_fn(fn_name)
     for name, offset in sorted(parsed.offsets.keys()):
         assert i+1 <= max_expect, (
             "%s: ran out if items in testing node" % fn_name)
         nodeInfo = parsed.offsets[name, offset]
         node = nodeInfo.node
-        nodeInfo2 = deparsed_find((name, offset), parsed, code)
         extractInfo = parsed.extract_node_info(node)
 
         assert expect[i] == extractInfo.selectedLine, \
@@ -319,5 +316,3 @@ for i in range(2): ...
 .
 """.split("\n")
     parsed = get_parsed_for_fn(for_range_stmt)
-    if not PYTHON3:
-        check_expect(expect, parsed, 'range_stmt')

pytest/test_disasm.py

@@ -1,7 +1,7 @@
 import os.path
 import pytest
 
-from uncompyle6.disas import disassemble_file
+from uncompyle6.code_fns import disassemble_file
 
 def get_srcdir():
     filename = os.path.normcase(os.path.dirname(__file__))

pytest/test_docstring.py

@@ -1,78 +0,0 @@
-import sys
-from uncompyle6 import PYTHON3
-if PYTHON3:
-    from io import StringIO
-    minint = -sys.maxsize-1
-    maxint = sys.maxsize
-else:
-    from StringIO import StringIO
-    minint = -sys.maxint-1
-    maxint = sys.maxint
-from uncompyle6.semantics.helper import print_docstring
-
-class PrintFake():
-    def __init__(self):
-        self.pending_newlines = 0
-        self.f = StringIO()
-
-    def write(self, *data):
-        if (len(data) == 0) or (len(data) == 1 and data[0] == ''):
-            return
-        out = ''.join((str(j) for j in data))
-        n = 0
-        for i in out:
-            if i == '\n':
-                n += 1
-                if n == len(out):
-                    self.pending_newlines = max(self.pending_newlines, n)
-                    return
-            elif n:
-                self.pending_newlines = max(self.pending_newlines, n)
-                out = out[n:]
-                break
-            else:
-                break
-
-        if self.pending_newlines > 0:
-            self.f.write('\n'*self.pending_newlines)
-            self.pending_newlines = 0
-
-        for i in out[::-1]:
-            if i == '\n':
-                self.pending_newlines += 1
-            else:
-                break
-
-        if self.pending_newlines:
-            out = out[:-self.pending_newlines]
-        self.f.write(out)
-    def println(self, *data):
-        if data and not(len(data) == 1 and data[0] == ''):
-            self.write(*data)
-        self.pending_newlines = max(self.pending_newlines, 1)
-        return
-    pass
-
-def test_docstring():
-
-    for doc, expect in (
-            ("Now is the time",
-             '  """Now is the time"""'),
-           ("""
-Now is the time
-""",
-            '''  """
-  Now is the time
-  """''')
-
-            # (r'''func placeholder - ' and with ("""\nstring\n  """)''',
-            #  """  r'''func placeholder - ' and with (\"\"\"\nstring\n\"\"\")'''"""),
-            # (r"""func placeholder - ' and with ('''\nstring\n''') and \"\"\"\nstring\n\"\"\" """,
-            # """  r\"\"\"func placeholder - ' and with ('''\nstring\n''') and \"\"\"\nstring\n\"\"\" \"\"\"""")
-            ):
-
-        o = PrintFake()
-        # print(doc)
-        # print(expect)
-        print_docstring(o, '  ', doc)
-        assert expect == o.f.getvalue()

pytest/test_fjt.py

@@ -1,5 +1,5 @@
 #!/usr/bin/env python
-from uncompyle6 import PYTHON_VERSION, IS_PYPY
+from xdis.version_info import PYTHON_VERSION_TRIPLE, IS_PYPY, version_tuple_to_str
 from uncompyle6.scanner import get_scanner
 def bug(state, slotstate):
     if state:
@@ -21,13 +21,13 @@ def bug_loop(disassemble, tb=None):
 
 def test_if_in_for():
     code = bug.__code__
-    scan = get_scanner(PYTHON_VERSION)
-    if 2.7 <= PYTHON_VERSION <= 3.0 and not IS_PYPY:
+    scan = get_scanner(PYTHON_VERSION_TRIPLE)
+    if (2, 7) <= PYTHON_VERSION_TRIPLE < (3, 1) and not IS_PYPY:
         scan.build_instructions(code)
         fjt = scan.find_jump_targets(False)
 
         ## FIXME: the data below is wrong.
-        ## we get different results currenty as well.
+        ## we get different results currently as well.
         ## We need to probably fix both the code
         ## and the test below
         # assert {15: [3], 69: [66], 63: [18]} == fjt
@@ -51,7 +51,7 @@ def test_if_in_for():
             # previous bug was not mistaking while-loop for if-then
             {'start': 48, 'end': 67, 'type': 'while-loop'}]
 
-    elif 3.2 < PYTHON_VERSION <= 3.4:
+    elif (3, 2) < PYTHON_VERSION_TRIPLE <= (3, 4):
         scan.build_instructions(code)
         fjt  = scan.find_jump_targets(False)
         assert {69: [66], 63: [18]} == fjt
@@ -62,6 +62,6 @@ def test_if_in_for():
            {'end': 59, 'type': 'for-loop', 'start': 31},
            {'end': 63, 'type': 'for-else', 'start': 62}]
     else:
-        print("FIXME: should fix for %s" % PYTHON_VERSION)
+        print("FIXME: should fix for %s" % version_tuple_to_str())
         assert True
     return

pytest/test_fstring.py

@@ -1,154 +0,0 @@
-# std
-# test
-from uncompyle6 import PYTHON_VERSION, deparse_code
-import pytest
-pytestmark = pytest.mark.skipif(PYTHON_VERSION <= 2.6,
-                                reason='hypothesis needs 2.7 or later')
-if PYTHON_VERSION > 2.6:
-
-    import hypothesis
-    from hypothesis import strategies as st
-
-    # uncompyle6
-
-
-    @st.composite
-    def expressions(draw):
-        # todo : would be nice to generate expressions using hypothesis however
-        # this is pretty involved so for now just use a corpus of expressions
-        # from which to select.
-        return draw(st.sampled_from((
-            'abc',
-            'len(items)',
-            'x + 1',
-            'lineno',
-            'container',
-            'self.attribute',
-            'self.method()',
-            # These expressions are failing, I think these are control
-            # flow problems rather than problems with FORMAT_VALUE,
-            # however I need to confirm this...
-            #'sorted(items, key=lambda x: x.name)',
-            #'func(*args, **kwargs)',
-            #'text or default',
-            #'43 if life_the_universe and everything else None'
-        )))
-
-
-    @st.composite
-    def format_specifiers(draw):
-        """
-        Generate a valid format specifier using the rules:
-
-        format_spec ::=  [[fill]align][sign][#][0][width][,][.precision][type]
-        fill        ::=  <any character>
-        align       ::=  "<" | ">" | "=" | "^"
-        sign        ::=  "+" | "-" | " "
-        width       ::=  integer
-        precision   ::=  integer
-        type        ::=  "b" | "c" | "d" | "e" | "E" | "f" | "F" | "g" | "G" | "n" | "o" | "s" | "x" | "X" | "%"
-
-        See https://docs.python.org/2/library/string.html
-
-        :param draw: Let hypothesis draw from other strategies.
-
-        :return: An example format_specifier.
-        """
-        alphabet_strategy = st.characters(min_codepoint=ord('a'), max_codepoint=ord('z'))
-        fill = draw(st.one_of(alphabet_strategy, st.none()))
-        align = draw(st.sampled_from(list('<>=^')))
-        fill_align = (fill + align or '') if fill else ''
-
-        type_ = draw(st.sampled_from('bcdeEfFgGnosxX%'))
-        can_have_sign = type_ in 'deEfFgGnoxX%'
-        can_have_comma = type_ in 'deEfFgG%'
-        can_have_precision = type_ in 'fFgG'
-        can_have_pound = type_ in 'boxX%'
-        can_have_zero = type_ in 'oxX'
-
-        sign = draw(st.sampled_from(list('+- ') + [''])) if can_have_sign else ''
-        pound = draw(st.sampled_from(('#', '',))) if can_have_pound else ''
-        zero = draw(st.sampled_from(('0', '',))) if can_have_zero else ''
-
-        int_strategy = st.integers(min_value=1, max_value=1000)
-
-        width = draw(st.one_of(int_strategy, st.none()))
-        width = str(width) if width is not None else ''
-
-        comma = draw(st.sampled_from((',', '',))) if can_have_comma else ''
-        if can_have_precision:
-            precision = draw(st.one_of(int_strategy, st.none()))
-            precision = '.' + str(precision) if precision else ''
-        else:
-            precision = ''
-
-        return ''.join((fill_align, sign, pound, zero, width, comma, precision, type_,))
-
-
-    @st.composite
-    def fstrings(draw):
-        """
-        Generate a valid f-string.
-        See https://www.python.org/dev/peps/pep-0498/#specification
-
-        :param draw: Let hypothsis draw from other strategies.
-
-        :return: A valid f-string.
-        """
-        character_strategy = st.characters(
-            blacklist_characters='\r\n\'\\s{}',
-            min_codepoint=1,
-            max_codepoint=1000,
-        )
-        is_raw = draw(st.booleans())
-        integer_strategy = st.integers(min_value=0, max_value=3)
-        expression_count = draw(integer_strategy)
-        content = []
-        for _ in range(expression_count):
-            expression = draw(expressions())
-            conversion = draw(st.sampled_from(('', '!s', '!r', '!a',)))
-            has_specifier = draw(st.booleans())
-            specifier = ':' + draw(format_specifiers()) if has_specifier else ''
-            content.append('{{{}{}}}'.format(expression, conversion, specifier))
-            content.append(draw(st.text(character_strategy)))
-        content = ''.join(content)
-        return "f{}'{}'".format('r' if is_raw else '', content)
-
-
-    @pytest.mark.skipif(PYTHON_VERSION != 3.6, reason='need Python 3.6')
-    @hypothesis.given(format_specifiers())
-    def test_format_specifiers(format_specifier):
-        """Verify that format_specifiers generates valid specifiers"""
-        try:
-            exec('"{:' + format_specifier + '}".format(0)')
-        except ValueError as e:
-            if 'Unknown format code' not in str(e):
-                raise
-
-
-    def run_test(text):
-        hypothesis.assume(len(text))
-        hypothesis.assume("f'{" in text)
-        expr = text + '\n'
-        code = compile(expr, '<string>', 'single')
-        deparsed = deparse_code(PYTHON_VERSION, code, compile_mode='single')
-        recompiled = compile(deparsed.text, '<string>', 'single')
-        if recompiled != code:
-            assert 'dis(' + deparsed.text.strip('\n') + ')' == 'dis(' + expr.strip('\n') + ')'
-
-
-    @pytest.mark.skipif(PYTHON_VERSION != 3.6, reason='need Python 3.6')
-    @hypothesis.given(fstrings())
-    def test_uncompyle_fstring(fstring):
-        """Verify uncompyling fstring bytecode"""
-        run_test(fstring)
-
-
-    @pytest.mark.skipif(PYTHON_VERSION < 3.6, reason='need Python 3.6+')
-    @pytest.mark.parametrize('fstring', [
-        "f'{abc}{abc!s}'",
-        "f'{abc}0'",
-    ])
-    def test_uncompyle_direct(fstring):
-        """useful for debugging"""
-        run_test(fstring)

pytest/test_function_call.py

@@ -1,185 +0,0 @@
-import string
-from uncompyle6 import PYTHON_VERSION
-import pytest
-pytestmark = pytest.mark.skip(PYTHON_VERSION < 2.7,
-                              reason="need at least Python 2.7")
-
-if PYTHON_VERSION > 2.6:
-    from hypothesis import given, assume, example, settings, strategies as st
-    from validate import validate_uncompyle
-    from test_fstring import expressions
-
-    alpha = st.sampled_from(string.ascii_lowercase)
-    numbers = st.sampled_from(string.digits)
-    alphanum = st.sampled_from(string.ascii_lowercase + string.digits)
-
-
-    @st.composite
-    def function_calls(draw,
-                       min_keyword_args=0, max_keyword_args=5,
-                       min_positional_args=0, max_positional_args=5,
-                       min_star_args=0, max_star_args=1,
-                       min_double_star_args=0, max_double_star_args=1):
-        """
-        Strategy factory for generating function calls.
-
-        :param draw: Callable which draws examples from other strategies.
-
-        :return: The function call text.
-        """
-        st_positional_args = st.lists(
-            alpha,
-            min_size=min_positional_args,
-            max_size=max_positional_args
-        )
-        st_keyword_args = st.lists(
-            alpha,
-            min_size=min_keyword_args,
-            max_size=max_keyword_args
-        )
-        st_star_args = st.lists(
-            alpha,
-            min_size=min_star_args,
-            max_size=max_star_args
-        )
-        st_double_star_args = st.lists(
-            alpha,
-            min_size=min_double_star_args,
-            max_size=max_double_star_args
-        )
-
-        positional_args = draw(st_positional_args)
-        keyword_args = draw(st_keyword_args)
-        st_values = st.lists(
-            expressions(),
-            min_size=len(keyword_args),
-            max_size=len(keyword_args)
-        )
-        keyword_args = [
-            x + '=' + e
-            for x, e in
-            zip(keyword_args, draw(st_values))
-        ]
-        star_args = ['*' + x for x in draw(st_star_args)]
-        double_star_args = ['**' + x for x in draw(st_double_star_args)]
-
-        arguments = positional_args + keyword_args + star_args + double_star_args
-        draw(st.randoms()).shuffle(arguments)
-        arguments = ','.join(arguments)
-
-        function_call = 'fn({arguments})'.format(arguments=arguments)
-        try:
-            # TODO: Figure out the exact rules for ordering of positional, keyword,
-            # star args, double star args and in which versions the various
-            # types of arguments are supported so we don't need to check that the
-            # expression compiles like this.
-            compile(function_call, '<string>', 'single')
-        except:
-            assume(False)
-        return function_call
-
-
-    def test_function_no_args():
-        validate_uncompyle("fn()")
-
-    @pytest.mark.skipif(PYTHON_VERSION < 2.7,
-                        reason="need at least Python 2.7")
-    def isolated_function_calls(which):
-        """
-        Returns a strategy for generating function calls, but isolated to
-        particular types of arguments, for example only positional arguments.
-
-        This can help reason about debugging errors in specific types of function
-        calls.
-
-        :param which: One of 'keyword', 'positional', 'star', 'double_star'
-
-        :return: Strategy for generating an function call isolated to specific
-                 argument types.
-        """
-        kwargs = dict(
-            max_keyword_args=0,
-            max_positional_args=0,
-            max_star_args=0,
-            max_double_star_args=0,
-        )
-        kwargs['_'.join(('min', which, 'args'))] = 1
-        kwargs['_'.join(('max', which, 'args'))] = 5 if 'star' not in which else 1
-        return function_calls(**kwargs)
-
-
-    with settings(max_examples=25):
-
-        @pytest.mark.skipif(PYTHON_VERSION < 2.7,
-                            reason="need at least Python 2.7")
-        @given(isolated_function_calls('positional'))
-        @example("fn(0)")
-        def test_function_positional_only(expr):
-            validate_uncompyle(expr)
-
-        @pytest.mark.skipif(PYTHON_VERSION < 2.7,
-                            reason="need at least Python 2.7")
-        @given(isolated_function_calls('keyword'))
-        @example("fn(a=0)")
-        def test_function_call_keyword_only(expr):
-            validate_uncompyle(expr)
-
-        @pytest.mark.skipif(PYTHON_VERSION < 2.7,
-                            reason="need at least Python 2.7")
-        @given(isolated_function_calls('star'))
-        @example("fn(*items)")
-        def test_function_call_star_only(expr):
-            validate_uncompyle(expr)
-
-        @pytest.mark.skipif(PYTHON_VERSION < 2.7,
-                            reason="need at least Python 2.7")
-        @given(isolated_function_calls('double_star'))
-        @example("fn(**{})")
-        def test_function_call_double_star_only(expr):
-            validate_uncompyle(expr)
-
-
-    @pytest.mark.xfail()
-    def test_BUILD_CONST_KEY_MAP_BUILD_MAP_UNPACK_WITH_CALL_BUILD_TUPLE_CALL_FUNCTION_EX():
-        validate_uncompyle("fn(w=0,m=0,**v)")
-
-
-    @pytest.mark.xfail()
-    def test_BUILD_MAP_BUILD_MAP_UNPACK_WITH_CALL_BUILD_TUPLE_CALL_FUNCTION_EX():
-        validate_uncompyle("fn(a=0,**g)")
-
-
-    @pytest.mark.xfail()
-    def test_CALL_FUNCTION_EX():
-        validate_uncompyle("fn(*g,**j)")
-
-
-    @pytest.mark.xfail()
-    def test_BUILD_MAP_CALL_FUNCTION_EX():
-        validate_uncompyle("fn(*z,u=0)")
-
-
-    @pytest.mark.xfail()
-    def test_BUILD_TUPLE_CALL_FUNCTION_EX():
-        validate_uncompyle("fn(**a)")
-
-
-    @pytest.mark.xfail()
-    def test_BUILD_MAP_BUILD_TUPLE_BUILD_TUPLE_UNPACK_WITH_CALL_CALL_FUNCTION_EX():
-        validate_uncompyle("fn(b,b,b=0,*a)")
-
-
-    @pytest.mark.xfail()
-    def test_BUILD_TUPLE_BUILD_TUPLE_UNPACK_WITH_CALL_CALL_FUNCTION_EX():
-        validate_uncompyle("fn(*c,v)")
-
-
-    @pytest.mark.xfail()
-    def test_BUILD_CONST_KEY_MAP_CALL_FUNCTION_EX():
-        validate_uncompyle("fn(i=0,y=0,*p)")
-
-
-    @pytest.mark.skip(reason='skipping property based test until all individual tests are passing')
-    @given(function_calls())
-    def test_function_call(function_call):
-        validate_uncompyle(function_call)

pytest/test_grammar.py

@@ -1,112 +1,147 @@
 import re
-from uncompyle6 import PYTHON_VERSION, PYTHON3, IS_PYPY # , PYTHON_VERSION
 from uncompyle6.parser import get_python_parser, python_parser
 from uncompyle6.scanner import get_scanner
+from xdis.version_info import PYTHON_VERSION_TRIPLE, IS_PYPY
+
 
 def test_grammar():
-
     def check_tokens(tokens, opcode_set):
         remain_tokens = set(tokens) - opcode_set
-        remain_tokens = set([re.sub(r'_\d+$','', t) for t in remain_tokens])
-        remain_tokens = set([re.sub('_CONT$','', t) for t in remain_tokens])
+        remain_tokens = set([re.sub(r"_\d+$", "", t) for t in remain_tokens])
+        remain_tokens = set([re.sub("_CONT$", "", t) for t in remain_tokens])
+        remain_tokens = set([re.sub("LOAD_CODE$", "", t) for t in remain_tokens])
         remain_tokens = set(remain_tokens) - opcode_set
-        assert remain_tokens == set([]), \
-            "Remaining tokens %s\n====\n%s" % (remain_tokens, p.dump_grammar())
+        assert remain_tokens == set([]), "Remaining tokens %s\n====\n%s" % (
+            remain_tokens,
+            p.dump_grammar(),
+        )
 
-    p = get_python_parser(PYTHON_VERSION, is_pypy=IS_PYPY)
-    (lhs, rhs, tokens,
-     right_recursive, dup_rhs) = p.check_sets()
+    p = get_python_parser(PYTHON_VERSION_TRIPLE, is_pypy=IS_PYPY)
+    (lhs, rhs, tokens, right_recursive, dup_rhs) = p.check_sets()
 
     # We have custom rules that create the below
-    expect_lhs = set(['pos_arg', 'attribute'])
-    if PYTHON_VERSION < 3.8:
-        expect_lhs.add('get_iter')
+    expect_lhs = set(["pos_arg"])
 
+    if PYTHON_VERSION_TRIPLE < (3, 8):
+        if PYTHON_VERSION_TRIPLE < (3, 7):
+            expect_lhs.add("attribute")
 
-    unused_rhs = set(['list', 'mkfunc',
-                      'mklambda',
-                      'unpack',])
+        expect_lhs.add("get_iter")
 
-    expect_right_recursive = set([('designList',
-                                   ('store', 'DUP_TOP', 'designList'))])
-
-    if PYTHON_VERSION < 3.7:
-        unused_rhs.add('call')
-
-    if PYTHON_VERSION > 2.6:
-        expect_lhs.add('kvlist')
-        expect_lhs.add('kv3')
-        unused_rhs.add('dict')
-
-    if PYTHON3:
-        expect_lhs.add('load_genexpr')
-
-        unused_rhs = unused_rhs.union(set("""
-        except_pop_except generator_exp
-        """.split()))
-        if PYTHON_VERSION >= 3.0:
-            expect_lhs.add("annotate_arg")
-            expect_lhs.add("annotate_tuple")
-            unused_rhs.add("mkfunc_annotate")
-            unused_rhs.add("dict_comp")
-            unused_rhs.add("classdefdeco1")
-            unused_rhs.add("tryelsestmtl")
-            if PYTHON_VERSION >= 3.5:
-                expect_right_recursive.add((('l_stmts',
-                                             ('lastl_stmt', 'come_froms', 'l_stmts'))))
-                pass
-            elif 3.0 < PYTHON_VERSION < 3.3:
-                expect_right_recursive.add((('l_stmts',
-                                             ('lastl_stmt', 'COME_FROM', 'l_stmts'))))
-                pass
-            pass
-        pass
+        if PYTHON_VERSION_TRIPLE >= (3, 8) or PYTHON_VERSION_TRIPLE < (3, 0):
+            expect_lhs.add("stmts_opt")
     else:
-        expect_lhs.add('kwarg')
+        expect_lhs.add("async_with_as_stmt")
+        expect_lhs.add("async_with_stmt")
 
-    assert expect_lhs == set(lhs)
+    unused_rhs = set(["list", "mkfunc", "lambda_body", "unpack"])
 
+    expect_right_recursive = set([("designList", ("store", "DUP_TOP", "designList"))])
+
+    if PYTHON_VERSION_TRIPLE[:2] <= (3, 6):
+        unused_rhs.add("call")
+
+    if PYTHON_VERSION_TRIPLE >= (2, 7):
+        expect_lhs.add("kvlist")
+        expect_lhs.add("kv3")
+        unused_rhs.add("dict")
+
+        if PYTHON_VERSION_TRIPLE < (3, 7) and PYTHON_VERSION_TRIPLE[:2] != (2, 7):
+            # NOTE: this may disappear
+            expect_lhs.add("except_handler_else")
+
+    expect_lhs.add("load_genexpr")
+
+    unused_rhs = unused_rhs.union(
+        set(
+            """
+    except_pop_except generator_exp
+    """.split()
+        )
+    )
+    if PYTHON_VERSION_TRIPLE < (3, 7):
+        expect_lhs.add("annotate_arg")
+        expect_lhs.add("annotate_tuple")
+        unused_rhs.add("mkfunc_annotate")
+
+    unused_rhs.add("dict_comp")
+    unused_rhs.add("classdefdeco1")
+    unused_rhs.add("tryelsestmtl")
+    if PYTHON_VERSION_TRIPLE >= (3, 5):
+        expect_right_recursive.add(
+            (("l_stmts", ("lastl_stmt", "come_froms", "l_stmts")))
+        )
+        pass
+    pass
+
+    if PYTHON_VERSION_TRIPLE >= (3, 7):
+        expect_lhs.add("set_for")
+        unused_rhs.add("set_iter")
+        pass
+    pass
     # FIXME
-    if PYTHON_VERSION != 3.8:
+    if PYTHON_VERSION_TRIPLE < (3, 8):
+        assert expect_lhs == set(lhs)
         assert unused_rhs == set(rhs)
 
     assert expect_right_recursive == right_recursive
 
-    expect_dup_rhs = frozenset([('COME_FROM',), ('CONTINUE',), ('JUMP_ABSOLUTE',),
-                                ('LOAD_CONST',),
-                                ('JUMP_BACK',), ('JUMP_FORWARD',)])
+    expect_dup_rhs = frozenset(
+        [
+            ("COME_FROM",),
+            ("CONTINUE",),
+            ("JUMP_ABSOLUTE",),
+            ("LOAD_CONST",),
+            ("JUMP_BACK",),
+            ("JUMP_FORWARD",),
+        ]
+    )
     reduced_dup_rhs = dict((k, dup_rhs[k]) for k in dup_rhs if k not in expect_dup_rhs)
-    for k in reduced_dup_rhs:
-        print(k, reduced_dup_rhs[k])
+    if reduced_dup_rhs:
+        print("\nPossible duplicate RHS that might be folded, into one of the LHS symbols")
+        for k in reduced_dup_rhs:
+            print(k, reduced_dup_rhs[k])
     # assert not reduced_dup_rhs, reduced_dup_rhs
 
-    s = get_scanner(PYTHON_VERSION, IS_PYPY)
+    s = get_scanner(PYTHON_VERSION_TRIPLE, IS_PYPY)
     ignore_set = set(
-            """
+        """
             JUMP_BACK CONTINUE
             COME_FROM COME_FROM_EXCEPT
             COME_FROM_EXCEPT_CLAUSE
             COME_FROM_LOOP COME_FROM_WITH
             COME_FROM_FINALLY ELSE
-            LOAD_GENEXPR LOAD_ASSERT LOAD_SETCOMP LOAD_DICTCOMP
+            LOAD_GENEXPR LOAD_ASSERT LOAD_SETCOMP LOAD_DICTCOMP LOAD_STR LOAD_CODE
             LAMBDA_MARKER
             RETURN_END_IF RETURN_END_IF_LAMBDA RETURN_VALUE_LAMBDA RETURN_LAST
-            """.split())
-    if 2.6 <= PYTHON_VERSION <= 2.7:
+            """.split()
+    )
+
+    if (2, 6) <= PYTHON_VERSION_TRIPLE <= (2, 7):
         opcode_set = set(s.opc.opname).union(ignore_set)
-        if PYTHON_VERSION == 2.6:
+        if PYTHON_VERSION_TRIPLE[:2] == (2, 6):
             opcode_set.add("THEN")
         check_tokens(tokens, opcode_set)
-    elif PYTHON_VERSION == 3.4:
-        ignore_set.add('LOAD_CLASSNAME')
-        ignore_set.add('STORE_LOCALS')
+    elif PYTHON_VERSION_TRIPLE[:2] == (3, 4):
+        ignore_set.add("LOAD_CLASSNAME")
+        ignore_set.add("STORE_LOCALS")
         opcode_set = set(s.opc.opname).union(ignore_set)
         check_tokens(tokens, opcode_set)
 
+
 def test_dup_rule():
     import inspect
-    python_parser(PYTHON_VERSION, inspect.currentframe().f_code,
-                  is_pypy=IS_PYPY,
-                  parser_debug={
-                      'dups': True, 'transition': False, 'reduce': False,
-                      'rules': False, 'errorstack': None, 'context': True})
+
+    python_parser(
+        PYTHON_VERSION_TRIPLE,
+        inspect.currentframe().f_code,
+        is_pypy=IS_PYPY,
+        parser_debug={
+            "dups": True,
+            "transition": False,
+            "reduce": False,
+            "rules": False,
+            "errorstack": None,
+            "context": True,
+        },
+    )

pytest/test_pysource.py-notyet

@@ -1,21 +1,15 @@
 import sys
-from uncompyle6 import PYTHON3
 from uncompyle6.scanner import get_scanner
 from uncompyle6.semantics.consts import (
     escape, NONE,
     # RETURN_NONE, PASS, RETURN_LOCALS
 )
 
-if PYTHON3:
-    from io import StringIO
-    def iteritems(d):
-        return d.items()
-else:
-    from StringIO import StringIO
-    def iteritems(d):
-        return d.iteritems()
+from io import StringIO
+def iteritems(d):
+    return d.items()
 
-from uncompyle6.semantics.pysource import SourceWalker as SourceWalker
+from uncompyle6.semantics.pysource import (SourceWalker, deparse_code2str)
 
 def test_template_engine():
     s = StringIO()
@@ -127,11 +121,17 @@ def test_tables():
                             "Full entry: %s" %
                             (name, k, arg, typ, entry[arg], type(entry[arg]), entry)
                             )
-                        assert len(tup) == 2
+                        assert 2 <= len(tup) <= 3
                         for j, x in enumerate(tup):
-                            assert isinstance(x, int), (
-                                "%s[%s][%d][%d] type '%s' is '%s should be an int but is %s. Full entry: %s" %
-                                (name, k, arg, j, typ, x, type(x), entry)
+                            if len(tup) == 3 and j == 1:
+                                assert isinstance(x, str), (
+                                    "%s[%s][%d][%d] type '%s' is '%s should be an string but is %s. Full entry: %s" %
+                                    (name, k, arg, j, typ, x, type(x), entry)
+                                    )
+                            else:
+                                assert isinstance(x, int), (
+                                    "%s[%s][%d][%d] type '%s' is '%s should be an int but is %s. Full entry: %s" %
+                                    (name, k, arg, j, typ, x, type(x), entry)
                                 )
                         pass
                     arg += 1
@@ -179,3 +179,11 @@ def test_tables():
             assert arg == len(entry), (
                 "%s[%s] arg %d should be length of entry %d. Full entry: %s" %
                             (name, k, arg, len(entry), entry))
+
+def test_deparse_code2str():
+    def deparse_test(co):
+        "This is a docstring"
+        s = deparse_code2str(co, debug_opts={"asm": "after", "tree": True})
+        assert s
+        return
+    deparse_test(deparse_test.__code__)

pytest/test_single_compile.py

@@ -1,22 +1,24 @@
 import pytest
-from uncompyle6 import PYTHON_VERSION, code_deparse
-pytestmark = pytest.mark.skip(PYTHON_VERSION < 2.7,
-                              reason="need at least Python 2.7")
+from uncompyle6 import code_deparse
+from xdis.version_info import PYTHON_VERSION_TRIPLE
 
-if PYTHON_VERSION > 2.6:
-    def test_single_mode():
-        single_expressions = (
-            'i = 1',
-            'i and (j or k)',
-            'i += 1',
-            'i = j % 4',
-            'i = {}',
-            'i = []',
-            'for i in range(10):\n    i\n',
-            'for i in range(10):\n    for j in range(10):\n        i + j\n',
-            'try:\n    i\nexcept Exception:\n    j\nelse:\n    k\n'
-        )
+pytest.mark.skip(PYTHON_VERSION_TRIPLE < (2, 7), reason="need Python < 2.7")
 
-        for expr in single_expressions:
-            code = compile(expr + '\n', '<string>', 'single')
-            assert code_deparse(code, compile_mode='single').text == expr + '\n'
+
+def test_single_mode():
+    single_expressions = (
+        "i = 1",
+        "i and (j or k)",
+        "i += 1",
+        "i = j % 4",
+        "i = {}",
+        "i = []",
+        "for i in range(10):\n    i\n",
+        "for i in range(10):\n    for j in range(10):\n        i + j\n",
+        # 'try:\n    i\nexcept Exception:\n    j\nelse:\n    k\n'
+    )
+
+    for expr in single_expressions:
+        code = compile(expr + "\n", "<string>", "single")
+        got = code_deparse(code, compile_mode="single").text
+        assert got == expr + "\n"

pytest/test_token.py

@@ -1,23 +1,24 @@
 from uncompyle6.scanners.tok import Token
 
+
 def test_token():
     # Test token formatting of: LOAD_CONST None
-    t = Token('LOAD_CONST', offset=0, attr=None, pattr=None, has_arg=True)
-    expect = '           0  LOAD_CONST               None'
+    t = Token("LOAD_CONST", offset=0, attr=None, pattr=None, has_arg=True)
+    expect = "0  LOAD_CONST               None"
     # print(t.format())
     assert t
-    assert t.format() == expect
+    assert t.format().strip() == expect.strip()
 
     # Make sure equality testing of tokens ignores offset
-    t2 = Token('LOAD_CONST', offset=2, attr=None, pattr=None, has_arg=True)
+    t2 = Token("LOAD_CONST", offset=2, attr=None, pattr=None, has_arg=True)
     assert t2 == t
 
-
     # Make sure formatting of: LOAD_CONST False. We assume False is the 0th index
     # of co_consts.
-    t = Token('LOAD_CONST', offset=1, attr=False, pattr=False, has_arg=True)
-    expect = '           1  LOAD_CONST            0  False'
-    assert t.format() == expect
+    t = Token("LOAD_CONST", offset=1, attr=False, pattr=False, has_arg=True)
+    expect = "1  LOAD_CONST               False"
+    assert t.format().strip() == expect.strip()
 
-if __name__ == '__main__':
+
+if __name__ == "__main__":
     test_token()

pytest/testdata/if-2.7.right

@@ -1,12 +1,12 @@
 # Python 2.7
 # Embedded file name: simple_source/branching/05_if.py
 
-   6       0  LOAD_NAME             0  'True'
-           3  POP_JUMP_IF_FALSE    15  'to 15'
+ L.   6         0  LOAD_NAME             0  'True'
+                3  POP_JUMP_IF_FALSE    15  'to 15'
 
-   7       6  LOAD_NAME             1  'False'
-           9  STORE_NAME            2  'b'
-          12  JUMP_FORWARD          0  'to 15'
-        15_0  COME_FROM            12  '12'
-          15  LOAD_CONST            0  None
-          18  RETURN_VALUE     
+ L.   7         6  LOAD_NAME             1  'False'
+                9  STORE_NAME            2  'b'
+               12  JUMP_FORWARD          0  'to 15'
+             15_0  COME_FROM            12  '12'
+               15  LOAD_CONST               None
+               18  RETURN_VALUE     

pytest/testdata/ifelse-2.7.right

@@ -1,15 +1,15 @@
 # Python 2.7
 # Embedded file name: simple_source/branching/05_ifelse.py
 
-   3       0  LOAD_NAME             0  'True'
-           3  POP_JUMP_IF_FALSE    15  'to 15'
+ L.   3         0  LOAD_NAME             0  'True'
+                3  POP_JUMP_IF_FALSE    15  'to 15'
 
-   4       6  LOAD_CONST            0  1
-           9  STORE_NAME            1  'b'
-          12  JUMP_FORWARD          6  'to 21'
+ L.   4         6  LOAD_CONST               1
+                9  STORE_NAME            1  'b'
+               12  JUMP_FORWARD          6  'to 21'
 
-   6      15  LOAD_CONST            1  2
-          18  STORE_NAME            2  'd'
-        21_0  COME_FROM            12  '12'
-          21  LOAD_CONST            2  None
-          24  RETURN_VALUE     
+ L.   6        15  LOAD_CONST               2
+               18  STORE_NAME            2  'd'
+             21_0  COME_FROM            12  '12'
+               21  LOAD_CONST               None
+               24  RETURN_VALUE     

pytest/validate.py

@@ -1,25 +1,24 @@
 # future
 from __future__ import print_function
+
 # std
 import os
 import difflib
 import subprocess
 import tempfile
 import functools
+
 # uncompyle6 / xdis
-from uncompyle6 import PYTHON_VERSION, PYTHON3, IS_PYPY, deparse_code
+from uncompyle6 import code_deparse
+from xdis.version_info import PYTHON_VERSION_TRIPLE, IS_PYPY
+
 # TODO : I think we can get xdis to support the dis api (python 3 version) by doing something like this there
-from xdis.bytecode import Bytecode
-from xdis.main import get_opcode
-opc = get_opcode(PYTHON_VERSION, IS_PYPY)
+from xdis import Bytecode, get_opcode
+
+opc = get_opcode(PYTHON_VERSION_TRIPLE, IS_PYPY)
 Bytecode = functools.partial(Bytecode, opc=opc)
 import six
 
-if PYTHON3:
-    from io import StringIO
-else:
-    from StringIO import StringIO
-
 def _dis_to_text(co):
     return Bytecode(co).dis()
 
@@ -33,36 +32,32 @@ def print_diff(original, uncompyled):
     :param original: Text describing the original code object.
     :param uncompyled: Text describing the uncompyled code object.
     """
-    original_lines = original.split('\n')
-    uncompyled_lines = uncompyled.split('\n')
-    args = original_lines, uncompyled_lines, 'original', 'uncompyled'
+    original_lines = original.split("\n")
+    uncompyled_lines = uncompyled.split("\n")
+    args = original_lines, uncompyled_lines, "original", "uncompyled"
     try:
         from bs4 import BeautifulSoup
+
         diff = difflib.HtmlDiff().make_file(*args)
         diff = BeautifulSoup(diff, "html.parser")
         diff.select_one('table[summary="Legends"]').extract()
     except ImportError:
-        print('\nTo display diff highlighting run:\n    pip install BeautifulSoup4')
+        print("\nTo display diff highlighting run:\n    pip install BeautifulSoup4")
         diff = difflib.HtmlDiff().make_table(*args)
 
     with tempfile.NamedTemporaryFile(delete=False) as f:
-        f.write(str(diff).encode('utf-8'))
+        f.write(str(diff).encode("utf-8"))
 
     try:
         print()
-        html = subprocess.check_output([
-            'elinks',
-            '-dump',
-            '-no-references',
-            '-dump-color-mode',
-            '1',
-            f.name,
-        ]).decode('utf-8')
+        html = subprocess.check_output(
+            ["elinks", "-dump", "-no-references", "-dump-color-mode", "1", f.name]
+        ).decode("utf-8")
         print(html)
     except:
-        print('\nFor side by side diff install elinks')
+        print("\nFor side by side diff install elinks")
         diff = difflib.Differ().compare(original_lines, uncompyled_lines)
-        print('\n'.join(diff))
+        print("\n".join(diff))
     finally:
         os.unlink(f.name)
 
@@ -72,7 +67,7 @@ def are_instructions_equal(i1, i2):
     Determine if two instructions are approximately equal,
     ignoring certain fields which we allow to differ, namely:
 
-    * code objects are ignore (should probaby be checked) due to address
+    * code objects are ignore (should probably be checked) due to address
     * line numbers
 
     :param i1: left instruction to compare
@@ -80,18 +75,19 @@ def are_instructions_equal(i1, i2):
 
     :return: True if the two instructions are approximately equal, otherwise False.
     """
-    result = (1 == 1
+    result = (
+        1 == 1
         and i1.opname == i2.opname
         and i1.opcode == i2.opcode
         and i1.arg == i2.arg
         # ignore differences due to code objects
         # TODO : Better way of ignoring address
-        and (i1.argval == i2.argval or '<code object' in str(i1.argval))
+        and (i1.argval == i2.argval or "<code object" in str(i1.argval))
         # TODO : Should probably recurse to check code objects
-        and (i1.argrepr == i2.argrepr or '<code object' in i1.argrepr)
+        and (i1.argrepr == i2.argrepr or "<code object" in i1.argrepr)
         and i1.offset == i2.offset
         # ignore differences in line numbers
-        #and i1.starts_line
+        # and i1.starts_line
         and i1.is_jump_target == i2.is_jump_target
     )
     return result
@@ -115,22 +111,21 @@ def are_code_objects_equal(co1, co2):
     return True
 
 
-def validate_uncompyle(text, mode='exec'):
+def validate_uncompyle(text, mode="exec"):
     """
     Validate decompilation of the given source code.
 
     :param text: Source to validate decompilation of.
     """
-    original_code = compile(text, '<string>', mode)
+    original_code = compile(text, "<string>", mode)
     original_dis = _dis_to_text(original_code)
     original_text = text
 
-    deparsed = deparse_code(PYTHON_VERSION, original_code,
-                            compile_mode=mode,
-                            out=six.StringIO(),
-                            is_pypy=IS_PYPY)
+    deparsed = code_deparse(
+        original_code, out=six.StringIO(), version=PYTHON_VERSION_TRIPLE, compile_mode=mode
+    )
     uncompyled_text = deparsed.text
-    uncompyled_code = compile(uncompyled_text, '<string>', 'exec')
+    uncompyled_code = compile(uncompyled_text, "<string>", "exec")
 
     if not are_code_objects_equal(uncompyled_code, original_code):
 
@@ -138,15 +133,17 @@ def validate_uncompyle(text, mode='exec'):
 
         def output(text, dis):
             width = 60
-            return '\n\n'.join([
-                ' SOURCE CODE '.center(width, '#'),
-                text.strip(),
-                ' BYTECODE '.center(width, '#'),
-                dis
-            ])
+            return "\n\n".join(
+                [
+                    " SOURCE CODE ".center(width, "#"),
+                    text.strip(),
+                    " BYTECODE ".center(width, "#"),
+                    dis,
+                ]
+            )
 
         original = output(original_text, original_dis)
         uncompyled = output(uncompyled_text, uncompyled_dis)
         print_diff(original, uncompyled)
 
-        assert 'original' == 'uncompyled'
+        assert "original" == "uncompyled"

requirements-dev.txt

@@ -1,2 +1,3 @@
 flake8
-hypothesis<=3.0.0
+six
+pytest==3.2.5 # for 2.7 < PYTHON_VERSION <= 3.2 use pytest 2.9.2; for 3.1 2.10

requirements.txt

@@ -2,3 +2,8 @@
 hypothesis==2.0.0
 pytest
 -e .
+
+Click~=7.0
+xdis>=6.0.4
+configobj~=5.0.6
+setuptools

setup-pretoml.py

@@ -0,0 +1,71 @@
+#!/usr/bin/env python
+import sys
+
+import setuptools
+
+"""Setup script for the 'uncompyle6' distribution."""
+
+SYS_VERSION = sys.version_info[0:2]
+if SYS_VERSION < (3, 6):
+    mess = "Python Release 3.6 .. 3.12 are supported in this code branch."
+    if (2, 4) <= SYS_VERSION <= (2, 7):
+        mess += (
+            "\nFor your Python, version %s, use the python-2.4 code/branch."
+            % sys.version[0:3]
+        )
+    if SYS_VERSION >= (3, 6):
+        mess += (
+            "\nFor your Python, version %s, use the master code/branch."
+            % sys.version[0:3]
+        )
+    if (3, 0) >= SYS_VERSION < (3, 3):
+        mess += (
+            "\nFor your Python, version %s, use the python-3.0-to-3.2 code/branch."
+            % sys.version[0:3]
+        )
+    if (3, 3) >= SYS_VERSION < (3, 6):
+        mess += (
+            "\nFor your Python, version %s, use the python-3.3-to-3.5 code/branch."
+            % sys.version[0:3]
+        )
+    elif SYS_VERSION < (2, 4):
+        mess += (
+            "\nThis package is not supported for Python version %s." % sys.version[0:3]
+        )
+    print(mess)
+    raise Exception(mess)
+
+from __pkginfo__ import (
+    __version__,
+    author,
+    author_email,
+    classifiers,
+    entry_points,
+    install_requires,
+    license,
+    long_description,
+    modname,
+    py_modules,
+    short_desc,
+    web,
+    zip_safe,
+)
+
+setuptools.setup(
+    author=author,
+    author_email=author_email,
+    classifiers=classifiers,
+    description=short_desc,
+    entry_points=entry_points,
+    install_requires=install_requires,
+    license=license,
+    long_description=long_description,
+    long_description_content_type="text/x-rst",
+    name=modname,
+    packages=setuptools.find_packages(),
+    py_modules=py_modules,
+    test_suite="nose.collector",
+    url=web,
+    version=__version__,
+    zip_safe=zip_safe,
+)

setup.cfg

@@ -1,11 +1,58 @@
 [bdist_rpm]
-release = 1
-packager = Mysterie <kajusska@gmail.com>
-doc_files = README
-#		CHANGES.txt
-#		USAGE.txt
+release = 0
+packager = rocky <rb@dustyfeet.com
+doc_files = README.rst
+		ChangeLog
+		COPYING
+		DECOMPYLE-2.4-CHANGELOG.txt
+		HISTORY.md
+		HOW-TO_REPORT-A-BUG.md
+		NEWS.md
 #		doc/
 #		examples/
 
 [bdist_wheel]
-universal=1
+universal = no
+
+[metadata]
+description_file = README.rst
+licences_files = COPYING
+
+[flake8]
+# max-line-length setting: NO we do not want everyone writing 120-character lines!
+# We are setting the maximum line length big here because there are longer
+# lines allowed by black in some cases that are forbidden by flake8. Since
+# black has the final say about code formatting issues, this setting is here to
+# make sure that flake8 doesn't fail the build on longer lines allowed by
+# black.
+max-line-length = 120
+max-complexity = 12
+select = E,F,W,C,B,B9
+ignore =
+    # E123 closing bracket does not match indentation of opening bracket's line
+    E123
+    # E203 whitespace before ':' (Not PEP8 compliant, Python Black)
+    E203
+    # E501 line too long (82 > 79 characters) (replaced by B950 from flake8-bugbear,
+    # https://github.com/PyCQA/flake8-bugbear)
+    E501
+    # W503 line break before binary operator (Not PEP8 compliant, Python Black)
+    W503
+    # W504 line break after binary operator (Not PEP8 compliant, Python Black)
+    W504
+    # C901 function too complex - since many of zz9 functions are too complex with a lot
+    # of if branching
+    C901
+    # module level import not at top of file. This is too restrictive. Can't even have a
+    # docstring higher.
+    E402
+per-file-ignores =
+    # These are config files. The `c` variable them is injected not defined.
+    pow/ansible/roles/jupyterhub/templates/jupyterhub_config*.py:F821
+    # Ignore some errors in files that are stolen from other projects to avoid lots
+    # of merge problems later .
+    pow/ansible/roles/webtier/files/supervisor_httpgroupok.py:E126,E128,E222,E225,E226,E261,E301,E302,E305,F841,E201,E202
+    silhouette/src/silhouette/gprof2dot.py:E711,E713,E741,F401
+    # Ignore undefined name errors in "expectation" test Python code.
+    # These files get exec'd in an environment that defines the variables.
+    server/tests/files/expectations/*.py:F821

setup.py

@@ -1,42 +1,6 @@
 #!/usr/bin/env python
-import sys
-
 """Setup script for the 'uncompyle6' distribution."""
 
-SYS_VERSION = sys.version_info[0:2]
-if not ((2, 6) <= SYS_VERSION  <= (3, 8)):
-    mess = "Python Release 2.6 .. 3.8 are supported in this code branch."
-    if ((2, 4) <= SYS_VERSION <= (2, 7)):
-        mess += ("\nFor your Python, version %s, use the python-2.4 code/branch." %
-                 sys.version[0:3])
-    elif SYS_VERSION < (2, 4):
-        mess += ("\nThis package is not supported for Python version %s."
-                 % sys.version[0:3])
-    print(mess)
-    raise Exception(mess)
+from setuptools import setup
 
-from __pkginfo__ import \
-    author,           author_email,       install_requires,          \
-    license,          long_description,   classifiers,               \
-    entry_points,     modname,            py_modules,                \
-    short_desc,       VERSION,            web,                       \
-    zip_safe
-
-from setuptools import setup, find_packages
-setup(
-       author             = author,
-       author_email       = author_email,
-       classifiers        = classifiers,
-       description        = short_desc,
-       entry_points       = entry_points,
-       install_requires   = install_requires,
-       license            = license,
-       long_description   = long_description,
-       name               = modname,
-       packages           = find_packages(),
-       py_modules         = py_modules,
-       test_suite         = 'nose.collector',
-       url                = web,
-       tests_require      = ['nose>=1.0'],
-       version            = VERSION,
-       zip_safe           = zip_safe)
+setup(packages=["uncompyle6"])

test/.gitignore

@@ -1 +1,8 @@
+/.coverage
+/.python-version
 /nohup.out
+/pycdc
+/test_pycdc_tests.sh
+/test_uncompyle2.py
+/test_unpy33.py
+/test_unpy37.py

test/Makefile

@@ -1,5 +1,6 @@
 PHONY=check clean dist distclean test test-unit test-functional rmChangeLog clean_pyc nosetests \
-      check-bytecode-1  check-bytecode-1.3 check-bytecode-1.4 check-bytecode-1.5 \
+      check-bytecode-1.0 check-bytecode-1.1 check-bytecode-1.2 check-bytecode-1.3 \
+      check-bytecode-1 check-bytecode-1.4 check-bytecode-1.5 check-bytecode-1.6 \
       check-bytecode-2 check-bytecode-3 check-bytecode-3-short \
       check-bytecode-2.2 check-byteocde-2.3 check-bytecode-2.4 \
       check-short check-2.6 check-2.7 check-3.0 check-3.1 check-3.2 check-3.3 \
@@ -12,7 +13,7 @@ PHONY=check clean dist distclean test test-unit test-functional rmChangeLog clea
 GIT2CL ?= git2cl
 PYTHON ?= python
 
-PYTHON_VERSION = $(shell $(PYTHON) -V 2>&1 | cut -d ' ' -f 2 | cut -d'.' -f1,2)
+PYTHON_VERSION = $(shell $(PYTHON) -V 2>&1 | cut -d ' ' -f 2 | cut -d'.' -f1,2 | head -1)
 NATIVE_CHECK = check-$(PYTHON_VERSION)
 
 # Set COMPILE='--compile' to force compilation before check
@@ -21,8 +22,8 @@ COVER_DIR=../tmp/grammar-cover
 
 # Run short tests
 check-short:
-	@$(PYTHON) -V && PYTHON_VERSION=`$(PYTHON) -V 2>&1 | cut -d ' ' -f 2 | cut -d'.' -f1,2`; \
-	$(MAKE) check-bytecode-short
+	@$(PYTHON) -V && PYTHON_VERSION=`$(PYTHON) -V 2>&1 | cut -d ' ' -f 2 | cut -d'.' -f1,2` | head -1; \
+	$(MAKE) check-bytecode-${PYTHON_VERSION}
 
 # Run all tests
 check:
@@ -34,58 +35,67 @@ check-2.6 check-2.7: check-bytecode-2 check-bytecode-3 check-bytecode-1 check-na
 #: Run working tests from Python 3.0
 check-3.0: check-bytecode
 	$(PYTHON) test_pythonlib.py --bytecode-3.0-run --verify-run
-	$(PYTHON) test_pythonlib.py --bytecode-3.0 --weak-verify $(COMPILE)
+	$(PYTHON) test_pythonlib.py --bytecode-3.0 --syntax-verify $(COMPILE)
 
 #: Run working tests from Python 3.1
 check-3.1: check-bytecode
 	$(PYTHON) test_pythonlib.py --bytecode-3.1-run --verify-run
-	$(PYTHON) test_pythonlib.py --bytecode-3.1 --weak-verify $(COMPILE)
+	$(PYTHON) test_pythonlib.py --bytecode-3.1 --syntax-verify $(COMPILE)
 
 #: Run working tests from Python 3.2
 check-3.2: check-bytecode
 	$(PYTHON) test_pythonlib.py --bytecode-3.2-run --verify-run
-	$(PYTHON) test_pythonlib.py --bytecode-3.2 --weak-verify $(COMPILE)
+	$(PYTHON) test_pythonlib.py --bytecode-3.2 --syntax-verify $(COMPILE)
 
 #: Run working tests from Python 3.3
 check-3.3: check-bytecode
 	$(PYTHON) test_pythonlib.py --bytecode-3.3-run --verify-run
-	$(PYTHON) test_pythonlib.py --bytecode-3.3 --weak-verify $(COMPILE)
+	$(PYTHON) test_pythonlib.py --bytecode-3.3 --syntax-verify $(COMPILE)
 
 #: Run working tests from Python 3.4
 check-3.4: check-bytecode check-3.4-ok check-2.7-ok
 	$(PYTHON) test_pythonlib.py --bytecode-3.4-run --verify-run
-	$(PYTHON) test_pythonlib.py --bytecode-3.4 --weak-verify $(COMPILE)
+	$(PYTHON) test_pythonlib.py --bytecode-3.4 --syntax-verify $(COMPILE)
 
 #: Run working tests from Python 3.5
 check-3.5: check-bytecode
 	$(PYTHON) test_pythonlib.py --bytecode-3.5-run --verify-run
-	$(PYTHON) test_pythonlib.py --bytecode-3.5 --weak-verify $(COMPILE)
+	$(PYTHON) test_pythonlib.py --bytecode-3.5 --syntax-verify $(COMPILE)
 
 #: Run working tests from Python 3.6
 check-3.6: check-bytecode
 	$(PYTHON) test_pythonlib.py --bytecode-3.6-run --verify-run
-	$(PYTHON) test_pythonlib.py --bytecode-3.6 --weak-verify $(COMPILE)
+	$(PYTHON) test_pythonlib.py --bytecode-3.6 --syntax-verify $(COMPILE)
 
 #: Run working tests from Python 3.7
 check-3.7: check-bytecode
 	$(PYTHON) test_pythonlib.py --bytecode-3.7-run --verify-run
-	$(PYTHON) test_pythonlib.py --bytecode-3.7 --weak-verify $(COMPILE)
+	$(PYTHON) test_pythonlib.py --bytecode-3.7 --syntax-verify $(COMPILE)
+
+check-pypy37: check-bytecode
+	$(PYTHON) test_pythonlib.py --bytecode-pypy37 --verify-run
 
 #: Run working tests from Python 3.8
 check-3.8: check-bytecode
 	$(PYTHON) test_pythonlib.py --bytecode-3.8-run --verify-run
-	$(PYTHON) test_pythonlib.py --bytecode-3.8 --weak-verify $(COMPILE)
+	$(PYTHON) test_pythonlib.py --bytecode-3.8 --syntax-verify $(COMPILE)
+
+check-3.9: check-bytecode
+	@echo "Note that we do not support decompiling Python 3.9 bytecode - no 3.9 tests run"
+
+check-3.10: check-bytecode
+	@echo "Note that we do not support decompiling Python 3.10 bytecode - no 3.10 tests run"
 
 # FIXME
-#: this is called when running under pypy3.5-5.8.0 or pypy2-5.6.0
-5.8 5.6:
+#: this is called when running under pypy3.5-5.8.0, pypy2-5.6.0, pypy3.6-7.3.0 or pypy3.8-7.3.7
+5.8 5.6 7.3:
 
 #: Check deparsing only, but from a different Python version
 check-disasm:
 	$(PYTHON) dis-compare.py
 
 #: Check deparsing bytecode 1.x only
-check-bytecode-1: check-bytecode-1.4 check-bytecode-1.5
+check-bytecode-1: check-bytecode-1.0 check-bytecode-1.1 check-bytecode-1.2 check-bytecode-1.3 check-bytecode-1.4 check-bytecode-1.5 check-bytecode-1.6
 
 #: Check deparsing bytecode 2.x only
 check-bytecode-2:
@@ -94,12 +104,19 @@ check-bytecode-2:
         --bytecode-2.5 --bytecode-2.6 --bytecode-2.7 --bytecode-pypy2.7
 
 #: Check deparsing bytecode 3.x only
+# check-bytecode-3:
+# 	$(PYTHON) test_pythonlib.py \
+#         --bytecode-3.0 \
+# 	--bytecode-3.1 --bytecode-3.2 --bytecode-3.3 \
+# 	--bytecode-3.4 --bytecode-3.5 --bytecode-3.6 \
+#         --bytecode-3.7  \
+# 	--bytecode-pypy3.2 --bytecode-pypy3.6 --bytecode-3.8
+
+# FIXME: Until we shaked out problems with xdis...
 check-bytecode-3:
-	$(PYTHON) test_pythonlib.py --bytecode-3.0 \
-	--bytecode-3.1 --bytecode-3.2 --bytecode-3.3 \
-	--bytecode-3.4 --bytecode-3.5 --bytecode-3.6 \
-        --bytecode-3.7 --bytecode-3.8 \
-	--bytecode-pypy3.2
+	$(PYTHON) test_pythonlib.py \
+	--bytecode-3.3 --bytecode-3.4 --bytecode-3.5 --bytecode-3.6 \
+        --bytecode-3.7  --bytecode-3.8
 
 #: Check deparsing on selected bytecode 3.x
 check-bytecode-3-short:
@@ -109,6 +126,7 @@ check-bytecode-3-short:
 #: Check deparsing bytecode on all Python 2 and Python 3 versions
 check-bytecode: check-bytecode-3
 	$(PYTHON) test_pythonlib.py \
+	--bytecode-1.0 --bytecode-1.1 --bytecode-1.2 \
 	--bytecode-1.3 --bytecode-1.4 --bytecode-1.5 \
         --bytecode-2.2 --bytecode-2.3 --bytecode-2.4 \
 	--bytecode-2.1 --bytecode-2.2 --bytecode-2.3 --bytecode-2.4 \
@@ -122,6 +140,18 @@ check-bytecode-short: check-bytecode-3-short
         --bytecode-2.6 --bytecode-2.7 --bytecode-pypy2.7
 
 
+#: Check deparsing bytecode 1.0 only
+check-bytecode-1.0:
+	$(PYTHON) test_pythonlib.py --bytecode-1.0
+
+#: Check deparsing bytecode 1.1 only
+check-bytecode-1.1:
+	$(PYTHON) test_pythonlib.py --bytecode-1.1
+
+#: Check deparsing bytecode 1.2 only
+check-bytecode-1.2:
+	$(PYTHON) test_pythonlib.py --bytecode-1.2
+
 #: Check deparsing bytecode 1.3 only
 check-bytecode-1.3:
 	$(PYTHON) test_pythonlib.py --bytecode-1.3
@@ -134,6 +164,10 @@ check-bytecode-1.4:
 check-bytecode-1.5:
 	$(PYTHON) test_pythonlib.py --bytecode-1.5
 
+#: Check deparsing bytecode 1.6 only
+check-bytecode-1.6:
+	$(PYTHON) test_pythonlib.py --bytecode-1.6
+
 #: Check deparsing Python 2.1
 check-bytecode-2.1:
 	$(PYTHON) test_pythonlib.py --bytecode-2.1
@@ -148,12 +182,24 @@ check-bytecode-2.3:
 
 #: Check deparsing Python 2.4
 check-bytecode-2.4:
+	$(PYTHON) test_pythonlib.py --bytecode-2.4-run --verify-run
 	$(PYTHON) test_pythonlib.py --bytecode-2.4
 
 #: Check deparsing Python 2.5
 check-bytecode-2.5:
+	$(PYTHON) test_pythonlib.py --bytecode-2.5-run --verify-run
 	$(PYTHON) test_pythonlib.py --bytecode-2.5
 
+#: Check deparsing Python 2.6
+check-bytecode-2.6:
+	$(PYTHON) test_pythonlib.py --bytecode-2.6-run --verify-run
+	$(PYTHON) test_pythonlib.py --bytecode-2.6 --syntax-verify
+
+#: Check deparsing Python 2.7
+check-bytecode-2.7:
+	$(PYTHON) test_pythonlib.py --bytecode-2.7-run --verify-run
+	$(PYTHON) test_pythonlib.py --bytecode-2.7 --syntax-verify
+
 #: Get grammar coverage for Python 2.4
 grammar-coverage-2.4:
 	-rm $(COVER_DIR)/spark-grammar-24.cover
@@ -176,7 +222,7 @@ grammar-coverage-2.6:
 grammar-coverage-2.7:
 	-rm $(COVER_DIR)/spark-grammar-2.7.cover || true
 	SPARK_PARSER_COVERAGE=$(COVER_DIR)/spark-grammar-2.7.cover $(PYTHON) test_pythonlib.py --bytecode-2.7
-	SPARK_PARSER_COVERAGE=$(COVER_DIR)/spark-grammar-2.7.cover $(PYTHON) test_pyenvlib.py --2.7.14 --max=600
+	SPARK_PARSER_COVERAGE=$(COVER_DIR)/spark-grammar-2.7.cover $(PYTHON) test_pyenvlib.py --2.7.16 --max=600
 
 #: Get grammar coverage for Python 3.0
 grammar-coverage-3.0:
@@ -219,81 +265,78 @@ grammar-coverage-3.5:
 grammar-coverage-3.6:
 	rm $(COVER_DIR)/spark-grammar-3.6.cover || /bin/true
 	SPARK_PARSER_COVERAGE=$(COVER_DIR)/spark-grammar-3.6.cover $(PYTHON) test_pythonlib.py --bytecode-3.6
-	SPARK_PARSER_COVERAGE=$(COVER_DIR)/spark-grammar-3.6.cover $(PYTHON) test_pyenvlib.py --3.6.4 --max=280
+	SPARK_PARSER_COVERAGE=$(COVER_DIR)/spark-grammar-3.6.cover $(PYTHON) test_pyenvlib.py --3.6.8 --max=280
 
-#: Check deparsing Python 2.6
-check-bytecode-2.6:
-	$(PYTHON) test_pythonlib.py --bytecode-2.6-run --verify-run
-	$(PYTHON) test_pythonlib.py --bytecode-2.6 --weak-verify
-
-#: Check deparsing Python 2.7
-check-bytecode-2.7:
-	$(PYTHON) test_pythonlib.py --bytecode-2.7-run --verify-run
-	$(PYTHON) test_pythonlib.py --bytecode-2.7 --weak-verify
+#: Get grammar coverage for Python 3.7
+grammar-coverage-3.7:
+	rm $(COVER_DIR)/spark-grammar-3.7.cover || /bin/true
+	SPARK_PARSER_COVERAGE=$(COVER_DIR)/spark-grammar-3.7.cover $(PYTHON) test_pyenvlib.py --3.7.3 --max=500
 
 #: Check deparsing Python 3.0
 check-bytecode-3.0:
 	$(PYTHON) test_pythonlib.py --bytecode-3.0-run --verify-run
-	$(PYTHON) test_pythonlib.py --bytecode-3.0 --weak-verify
+	$(PYTHON) test_pythonlib.py --bytecode-3.0 --syntax-verify
 
 #: Check deparsing Python 3.1
 check-bytecode-3.1:
 	$(PYTHON) test_pythonlib.py --bytecode-3.1-run --verify-run
-	$(PYTHON) test_pythonlib.py --bytecode-3.1 --weak-verify
+	$(PYTHON) test_pythonlib.py --bytecode-3.1 --syntax-verify
 
 #: Check deparsing Python 3.2
 check-bytecode-3.2:
 	$(PYTHON) test_pythonlib.py --bytecode-3.2-run --verify-run
-	$(PYTHON) test_pythonlib.py --bytecode-3.2 --weak-verify
+	$(PYTHON) test_pythonlib.py --bytecode-3.2 --syntax-verify
 
 #: Check deparsing Python 3.3
 check-bytecode-3.3:
 	$(PYTHON) test_pythonlib.py --bytecode-3.3-run --verify-run
-	$(PYTHON) test_pythonlib.py --bytecode-3.3 --weak-verify
+	$(PYTHON) test_pythonlib.py --bytecode-3.3 --syntax-verify
 
 #: Check deparsing Python 3.4
 check-bytecode-3.4:
 	$(PYTHON) test_pythonlib.py --bytecode-3.4-run --verify-run
-	$(PYTHON) test_pythonlib.py --bytecode-3.4 --weak-verify
+	$(PYTHON) test_pythonlib.py --bytecode-3.4 --syntax-verify
 
 #: Check deparsing Python 3.5
 check-bytecode-3.5:
 	$(PYTHON) test_pythonlib.py --bytecode-3.5-run --verify-run
-	$(PYTHON) test_pythonlib.py --bytecode-3.5 --weak-verify
+	$(PYTHON) test_pythonlib.py --bytecode-3.5 --syntax-verify
 
 #: Check deparsing Python 3.6
 check-bytecode-3.6:
-	$(PYTHON) test_pythonlib.py --bytecode-3.6 --weak-verify
 	$(PYTHON) test_pythonlib.py --bytecode-3.6-run --verify-run
+	$(PYTHON) test_pythonlib.py --bytecode-3.6 --syntax-verify
 
 #: Check deparsing Python 3.7
 check-bytecode-3.7:
-	$(PYTHON) test_pythonlib.py --bytecode-3.7 --weak-verify
+	$(PYTHON) test_pythonlib.py --bytecode-3.7-run --verify-run
+	$(PYTHON) test_pythonlib.py --bytecode-3.7 --syntax-verify
 
 #: Check deparsing Python 3.8
 check-bytecode-3.8:
-	$(PYTHON) test_pythonlib.py --bytecode-3.8 --weak-verify
+	$(PYTHON) test_pythonlib.py --bytecode-3.8-run --verify-run
+	$(PYTHON) test_pythonlib.py --bytecode-3.8 --syntax-verify
 
 #: short tests for bytecodes only for this version of Python
 check-native-short:
-	$(PYTHON) test_pythonlib.py --bytecode-$(PYTHON_VERSION) --weak-verify $(COMPILE)
+	$(PYTHON) test_pythonlib.py --bytecode-$(PYTHON_VERSION) --syntax-verify $(COMPILE)
 	$(PYTHON) test_pythonlib.py --bytecode-$(PYTHON_VERSION)-run --verify-run $(COMPILE)
 
 #: Run longer Python 2.6's lib files known to be okay
 check-2.6-ok:
-	$(PYTHON) test_pythonlib.py --ok-2.6 --weak-verify $(COMPILE)
+	$(PYTHON) test_pythonlib.py --ok-2.6 --syntax-verify $(COMPILE)
 
 #: Run longer Python 2.7's lib files known to be okay
 check-2.7-ok:
-	$(PYTHON) test_pythonlib.py --ok-2.7 --weak-verify $(COMPILE)
+	$(PYTHON) test_pythonlib.py --ok-2.7 --syntax-verify $(COMPILE)
 
 #: Run longer Python 3.2's lib files known to be okay
 check-3.2-ok:
-	$(PYTHON) test_pythonlib.py --ok-3.2 --weak-verify $(COMPILE)
+	$(PYTHON) test_pythonlib.py --ok-3.2 --syntax-verify $(COMPILE)
 
 #: Run longer Python 3.4's lib files known to be okay
 check-3.4-ok:
-	$(PYTHON) test_pythonlib.py --ok-3.4 --weak-verify $(COMPILE)
+	$(PYTHON) test_pythonlib.py --ok-3.4 --syntax-verify $(COMPILE)
 
 #: PyPy of some sort. E.g. [PyPy 5.0.1 with GCC 4.8.4]
 # Skip for now
@@ -307,6 +350,25 @@ pypy-2.7 5.0 5.3 6.0:
 pypy-3.2 2.4:
 	$(PYTHON) test_pythonlib.py --bytecode-pypy3.2 --verify
 
+#: PyPy 5.0.x with Python 3.6.1 ...
+check-bytecode-pypy3.6: 7.1
+7.1:
+	$(PYTHON) test_pythonlib.py --bytecode-pypy3.6-run --verify-run
+	$(PYTHON) test_pythonlib.py --bytecode-pypy3.6 --verify
+
+#: PyPy 5.0.x with Python 3.6.9
+check-bytecode-pypy3.6: 7.2 7.3
+
+7.2:
+	$(PYTHON) test_pythonlib.py --bytecode-pypy3.6-run --verify-run
+	$(PYTHON) test_pythonlib.py --bytecode-pypy3.6 --verify
+
+7.3:
+	$(PYTHON) test_pythonlib.py --bytecode-pypy3.7 --verify-run
+
+# Pyston 2.3
+2.3: check-3.8
+
 clean: clean-py-dis clean-dis clean-unverified
 
 clean-dis:

test/add-test.py

@@ -1,23 +1,49 @@
 #!/usr/bin/env python
-""" Trivial helper program to bytecompile and run an uncompile
+""" Trivial helper program to byte compile and uncompile the bytecode file.
 """
 import os, sys, py_compile
-assert len(sys.argv) >= 2
+from xdis.version_info import version_tuple_to_str, PYTHON_VERSION_TRIPLE
+
+if len(sys.argv) < 2:
+    print("Usage: add-test.py [--run] *python-source*... [optimize-level]")
+    sys.exit(1)
+
+assert 2 <= len(sys.argv) <= 4
 version = sys.version[0:3]
-if sys.argv[1] == '--run':
-    suffix = '_run'
+vers = sys.version_info[:2]
+if sys.argv[1] in ("--run", "-r"):
+    suffix = "_run"
+    assert len(sys.argv) >= 3
     py_source = sys.argv[2:]
+    i = 2
 else:
-    suffix = ''
+    suffix = ""
     py_source = sys.argv[1:]
+    i = 1
+try:
+    optimize = int(sys.argv[-1])
+    assert sys.argv >= i + 2
+    py_source = sys.argv[i:-1]
+    i = 2
+
+except:
+    optimize = 2
 
 for path in py_source:
     short = os.path.basename(path)
-    if hasattr(sys, 'pypy_version_info'):
-        cfile =  "bytecode_pypy%s%s/%s" % (version, suffix, short) + 'c'
+    if short.endswith(".py"):
+        short = short[: -len(".py")]
+    if hasattr(sys, "pypy_version_info"):
+        version = version_tuple_to_str(end=2, delimiter="")
+        bytecode = "bytecode_pypy%s%s/%spy%s.pyc" % (version, suffix, short, version)
     else:
-        cfile =  "bytecode_%s%s/%s" % (version, suffix, short) + 'c'
-    print("byte-compiling %s to %s" % (path, cfile))
-    py_compile.compile(path, cfile)
-    if isinstance(version, str) or version >= (2, 6, 0):
-        os.system("../bin/uncompyle6 -a -t %s" % cfile)
+        version = version_tuple_to_str(end=2)
+        bytecode = "bytecode_%s%s/%s.pyc" % (version, suffix, short)
+
+    print("byte-compiling %s to %s" % (path, bytecode))
+    if PYTHON_VERSION_TRIPLE >= (3, 2):
+        py_compile.compile(path, bytecode, optimize=optimize)
+    else:
+        py_compile.compile(path, bytecode)
+    if PYTHON_VERSION_TRIPLE >= (2, 6):
+        os.system("../bin/uncompyle6 -a -t %s" % bytecode)