Use xdis pattr extraction for LOAD_NAME

Tentative fix for issue #437.

.circleci/config.yml

@@ -1,12 +1,7 @@
-# This configuration was automatically generated from a CircleCI 1.0 config.
-# It should include any build commands you had along with commands that CircleCI
-# inferred from your project structure. We strongly recommend you read all the
-# comments in this file to understand the structure of CircleCI 2.0, as the idiom
-# for configuration has changed substantially in 2.0 to allow arbitrary jobs rather
-# than the prescribed lifecycle of 1.0. In general, we recommend using this generated
-# configuration as a reference rather than using it in production, though in most
-# cases it should duplicate the execution of your original 1.0 config.
 version: 2
+filters:
+  branches:
+    only: master
 jobs:
   build:
     working_directory: ~/rocky/python-uncompyle6
@@ -18,18 +13,10 @@ jobs:
       CIRCLE_ARTIFACTS: /tmp/circleci-artifacts
       CIRCLE_TEST_REPORTS: /tmp/circleci-test-results
       COMPILE: --compile
-    # In CircleCI 1.0 we used a pre-configured image with a large number of languages and other packages.
-    # In CircleCI 2.0 you can now specify your own image, or use one of our pre-configured images.
-    # The following configuration line tells CircleCI to use the specified docker image as the runtime environment for you job.
-    # We have selected a pre-built image that mirrors the build environment we use on
-    # the 1.0 platform, but we recommend you choose an image more tailored to the needs
-    # of each job. For more information on choosing an image (or alternatively using a
-    # VM instead of a container) see https://circleci.com/docs/2.0/executor-types/
     # To see the list of pre-built images that CircleCI provides for most common languages see
     # https://circleci.com/docs/2.0/circleci-images/
-    machine:
-      python:
-        version: 2.7.14
+    docker:
+      - image: circleci/python:3.6.9
     steps:
     # Machine Setup
     #   If you break your build into multiple jobs with workflows, you will probably want to do the parts of this that are relevant in each
@@ -42,7 +29,7 @@ jobs:
     # This is based on your 1.0 configuration file or project settings
     - run:
         working_directory: ~/rocky/python-uncompyle6
-        command: pip install virtualenv && pip install nose && pip install pep8 && pyenv rehash
+        command: pip install --user virtualenv && pip install --user nose && pip install --user pep8
     # Dependencies
     #   This would typically go in either a build or a build-and-test job when using workflows
     # Restore the dependency cache
@@ -52,10 +39,13 @@ jobs:
           # fallback to using the latest cache if no exact match is found
           - v2-dependencies-
 
-    # This is based on your 1.0 configuration file or project settings
-    - run: pip install --upgrade setuptools
-    - run: pip install -e .
-    - run: pip install -r requirements-dev.txt
+    - run:
+        command: | # Use pip to install dependengcies
+          pip install --user --upgrade setuptools
+          # Until the next release
+          pip install git+https://github.com/rocky/python-xdis#egg=xdis
+          pip install --user -e .
+          pip install --user -r requirements-dev.txt
 
     # Save dependency cache
     - save_cache:
@@ -73,7 +63,7 @@ jobs:
     # Test
     #   This would typically be a build job when using workflows, possibly combined with build
     # This is based on your 1.0 configuration file or project settings
-    - run: python ./setup.py develop && make check-2.7
+    - run: sudo python ./setup.py develop && make check-3.6
     - run: cd ./test/stdlib && bash ./runtests.sh 'test_[p-z]*.py'
     # Teardown
     #   If you break your build into multiple jobs with workflows, you will probably want to do the parts of this that are relevant in each

.editorconfig

@@ -0,0 +1,28 @@
+# THis is an EditorConfig file
+# https://EditorConfig.org
+
+root = true
+
+[*]
+end_of_line = lf
+insert_final_newline = true
+charset = utf-8
+indent_style = tab
+indent_size = 4
+insert_final_newline = true
+
+[*.yml]
+indent_style = space
+indent_size = 2
+end_of_line = lf
+insert_final_newline = true
+
+[*.py]
+indent_style = space
+indent_size = 4
+end_of_line = lf
+insert_final_newline = true
+
+# Tab indentation (no size specified)
+[Makefile]
+indent_style = tab

.github/FUNDING.yml

@@ -0,0 +1,12 @@
+# These are supported funding model platforms
+
+github: [rocky]
+patreon: # Replace with a single Patreon username
+open_collective: # Replace with a single Open Collective username
+ko_fi: # Replace with a single Ko-fi username
+tidelift: # Replace with a single Tidelift platform-name/package-name e.g., npm/babel
+community_bridge: # Replace with a single Community Bridge project-name e.g., cloud-foundry
+liberapay: # Replace with a single Liberapay username
+issuehunt: # Replace with a single IssueHunt username
+otechie: # Replace with a single Otechie username
+custom: # Replace with up to 4 custom sponsorship URLs e.g., ['link1', 'link2']

.github/ISSUE_TEMPLATE/bug-report.md

@@ -4,38 +4,72 @@ about: Tell us about uncompyle6 bugs
 
 ---
 
-<!-- __Note:__ Have you read https://github.com/rocky/python-uncompyle6/blob/master/HOW-TO-REPORT-A-BUG.md ?
+<!-- __Note:__ If you are using this program to do something illegal - don't.
+The issue may be flagged to make it easier for those looking for illegal activity.
 
+Bugs are not for asking questions about a problem you
+are trying to solve that involve the use of uncompyle6 along the way,
+although I may be more tolerant of this if you sponsor the project.
+
+Bugs are also not for general or novice kind help on how to install
+this Python program in your environment in the way you would like to
+have it set up, or how to interpret a Python traceback e.g. that winds
+up saying Python X.Y.Z is not supported.
+
+For these kinds of things, you will save yourself time by asking
+instead on forums like StackOverflow that are geared to helping people
+for such general or novice kinds questions and tasks. And unless you
+are a sponsor of the project, if your question seems to be of this
+category, the issue may just be closed.
+
+Also, the unless you are a sponsor of the project, it may take a
+while, maybe a week or so, before the bug report is noticed, let alone
+acted upon.
+
+To set expectations, some legitimate bugs can take years to fix, but
+they eventually do get fixed.
+
+Funding the project was added to partially address the problem that there are
+lots of people seeking help and reporting bugs, but few people who are
+willing or capable of providing help or fixing bugs.
+
+Tasks or the kinds of things others can do, but you can't do or don't
+want to do yourself are typically the kind of thing that you pay
+someone to do, especially when you are the primary beneficiary of the
+work, or the task is complex, long, or tedious. If your code is over
+30 lines long, it fits into this category.
+
+
+See also https://github.com/rocky/python-uncomp[yle6/blob/master/HOW-TO-REPORT-A-BUG.md ?
+-->
+
+<!--
 Please remove any of the optional sections if they are not applicable.
 
-Prerequisites
+Prerequisites/Caveats
 
 * Make sure the bytecode you have can be disassembled with a
-  disassembler.
+  disassembler and produces valid results.
 * Don't put bytecode and corresponding source code on any service that
   requires registration to download.
-* When you open a bug report there is no privacy. If the legitimacy of
-  the activity is deemed suspicous, I may flag it as suspicious,
+* When you open a bug report there is no privacy. If you need privacy, then
+  contact me by email and explain who you are and the need for privacy.
+  But be mindful that you may be asked to sponsor the project for the
+  personal and private help that you are requesting.
+* If the legitimacy of the activity is deemed suspicious, I may flag it as suspicious,
   making the issue even more easy to detect.
 
-Bug reports that violate a prerequisite may be discarded.
-
-Note that there are way more bug-fix requestors than there are bug
-fixers. If you want you need more immediate, confidential or urgent
-assistance
-
-http://www.crazy-compilers.com/decompyle/ offers a byte-code
-decompiler service for versions of Python up to 2.6.
+Bug reports that violate the above may be discarded.
 
 -->
 
 ## Description
 
-<!-- Add a clear and concise description of the bug. -->
+<!-- Please add a clear and concise description of the bug. Try to narrow the problem down to the smallest that exhibits the bug.-->
 
 ## How to Reproduce
 
-<!-- Please show both the input you gave and the
+<!-- Please show both the *input* you gave and the
 output you got in describing how to reproduce the bug:
 
 or give a complete console log with input and output
@@ -46,12 +80,22 @@ $ uncompyle6 <command-line-options>
 $
 ```
 
-Provide links to the Python bytecode. For example you can create a
+Provide links to the Python bytecode. For example, you can create a
 gist with the information. If you have the correct source code, you
 can add that too.
 
 -->
 
+## Output Given
+
+<!--
+Please include not just the error message but all output leading to the message which includes echoing input and messages up to the error.
+For a command-line environment include command invocation and all the output produced.
+
+If this is too long, then try narrowing the problem to something short.
+-->
+
+
 ## Expected behavior
 
 <!-- Add a clear and concise description of what you expected to happen. -->
@@ -63,12 +107,20 @@ can add that too.
 Please modify for your setup
 
 - Uncompyle6 version: output from  `uncompyle6 --version` or `pip show uncompyle6`
-- Python version: `python -V`
+- Python version for the version of Python the byte-compiled the file: `python -c "import sys; print(sys.version)"` where `python` is the correct CPython or PyPy binary.
 - OS and Version: [e.g. Ubuntu bionic]
 
 -->
 
-## Additional Environment or Context
+## Workarounds
+
+<!-- If there is a workaround for the problem, describe that here. -->
+
+## Priority
+
+<!-- If this is blocking some important activity let us know what activity it blocks. -->
+
+## Additional Context
 
 <!-- _This section is optional._
 

.github/ISSUE_TEMPLATE/config.yml

@@ -0,0 +1 @@
+blank_issues_enabled: False

.github/workflows/osx.yml

@@ -0,0 +1,31 @@
+name: uncompyle6 (osx)
+
+on:
+  push:
+    branches: [ master ]
+  pull_request:
+    branches: [ master ]
+
+jobs:
+  build:
+    runs-on: macos-latest
+    strategy:
+      matrix:
+        os: [macOS]
+        python-version: [3.7, 3.8]
+    steps:
+    - uses: actions/checkout@v2
+    - name: Set up Python ${{ matrix.python-version }}
+      uses: actions/setup-python@v2
+      with:
+        python-version: ${{ matrix.python-version }}
+    - name: Install dependencies
+      run: |
+        python -m pip install --upgrade pip
+        # Until the next xdis release
+        pip install git+https://github.com/rocky/python-xdis#egg=xdis
+        pip install -e .
+        pip install -r requirements-dev.txt
+    - name: Test uncompyle6
+      run: |
+        make check

.github/workflows/ubuntu.yml

@@ -0,0 +1,30 @@
+name: uncompyle6 (ubuntu)
+
+on:
+  push:
+    branches: [ master ]
+  pull_request:
+    branches: [ master ]
+
+jobs:
+  build:
+    runs-on: ubuntu-latest
+    strategy:
+      matrix:
+        python-version: [3.7, 3.8]
+    steps:
+    - uses: actions/checkout@v2
+    - name: Set up Python ${{ matrix.python-version }}
+      uses: actions/setup-python@v2
+      with:
+        python-version: ${{ matrix.python-version }}
+    - name: Install dependencies
+      run: |
+        python -m pip install --upgrade pip
+        # Until the next xdis release
+        pip install git+https://github.com/rocky/python-xdis#egg=xdis
+        pip install -e .
+        pip install -r requirements-dev.txt
+    - name: Test uncompyle6
+      run: |
+        make check

.github/workflows/windows.yml

@@ -0,0 +1,31 @@
+name: uncompyle6 (windows)
+
+on:
+  push:
+    branches: [ master ]
+  pull_request:
+    branches: [ master ]
+
+jobs:
+  build:
+    runs-on: macos-latest
+    strategy:
+      matrix:
+        os: [windows]
+        python-version: [3.7, 3.8]
+    steps:
+    - uses: actions/checkout@v2
+    - name: Set up Python ${{ matrix.python-version }}
+      uses: actions/setup-python@v2
+      with:
+        python-version: ${{ matrix.python-version }}
+    - name: Install dependencies
+      run: |
+        python -m pip install --upgrade pip
+        # Until the next xdis release
+        pip install git+https://github.com/rocky/python-xdis#egg=xdis
+        pip install -e .
+        pip install -r requirements-dev.txt
+    - name: Test uncompyle6
+      run: |
+        make check

.gitignore

@@ -6,9 +6,11 @@
 /.eggs
 /.hypothesis
 /.idea
+/.mypy_cache
 /.pytest_cache
 /.python-version
 /.tox
+.mypy_cache
 /.venv*
 /README
 /__pkginfo__.pyc

.isort.cfg

@@ -0,0 +1,11 @@
+[settings]
+multi_line_output = 3
+include_trailing_comma = True
+force_grid_wrap = 0
+use_parentheses = True
+line_length = 88
+known_crunch = cr, zz9d, zz9lib, pycrunch, silhouette
+sections = FUTURE,STDLIB,THIRDPARTY,FIRSTPARTY,CRUNCH,LOCALFOLDER
+default_section = THIRDPARTY
+combine_as_imports = 1
+profile = black

.travis.yml

@@ -1,10 +1,11 @@
 language: python
 
 python:
-  - '3.5'
-  - '2.7'
-  - '3.4'
+#  - '3.5'
+#  - '2.7'
+#  - '3.4'
   - '3.6'
+  - '3.8'
 
 matrix:
   include:
@@ -12,6 +13,8 @@ matrix:
       dist: xenial  # required for Python >= 3.7 (travis-ci/travis-ci#9069)
 
 install:
+# Remove the next line when xdis 6.0.0 is released
+# - pip install git://github.com/rocky/python-xdis.git#egg=xdis
 - pip install -e .
 - pip install -r requirements-dev.txt
 

HISTORY.md

@@ -1,213 +1,93 @@
-This project has history of over 18 years spanning back to Python 1.5
+# Introduction
 
-There have been a number of people who have worked on this. I am awed
-by the amount of work, number of people who have contributed to this,
-and the cleverness in the code.
+This project started around 1999 spanning back to Python 1.5
 
-The below is an annotated history from talking to participants
-involved and my reading of the code and sources cited.
+In the interest of shortening what is written here, I am going to start where we left off where [decompyle 2.4's history](https://github.com/rocky/decompile-2.4/blob/master/HISTORY.md) ends.
 
-In 1998, John Aycock first wrote a grammar parser in Python,
-eventually called SPARK, that was usable inside a Python program. This
-code was described in the
-[7th International Python Conference](http://legacy.python.org/workshops/1998-11/proceedings/papers/aycock-little/aycock-little.html). That
-paper doesn't talk about decompilation, nor did John have that in mind
-at that time. It does mention that a full parser for Python (rather
-than the simple languages in the paper) was being considered.
+For the earlier history up to 2006 and the code up until Python 2.4, which I find interesting, look at that link.
 
-[This](http://pages.cpsc.ucalgary.ca/~aycock/spark/content.html#contributors)
-contains a of people acknowledged in developing SPARK. What's amazing
-about this code is that it is reasonably fast and has survived up to
-Python 3 with relatively little change. This work was done in
-conjunction with his Ph.D Thesis. This was finished around 2001. In
-working on his thesis, John realized SPARK could be used to deparse
-Python bytecode. In the fall of 1999, he started writing the Python
-program, "decompyle", to do this.
+Sometime around 2014 was the dawn of ["uncompyle" and PyPI](https://pypi.python.org/pypi/uncompyle/1.1) — the era of
+public version control. Dan Pascu's code although not public used [darcs](http://darcs.net/) for version control. I converted the darcs to to git and put this at [decompyle-2.4](https://github.com/rocky/decompile-2.4).
 
-To help with control structure deparsing the instruction sequence was
-augmented with pseudo instruction COME_FROM. This code introduced
-another clever idea: using table-driven semantics routines, using
-format specifiers.
+# uncompyle, unpyc
 
-The last mention of a release of SPARK from John is around 2002. As
-released, although the Earley Algorithm parser was in good shape, this
-code was woefully lacking as serious Python deparser.
+In contrast to _decompyle_ that went up to Python 2.4, _uncompyle_, at least in its final versions, runs only on Python 2.7. However it accepts bytecode back to Python 2.5. Thomas Grainger is the package owner of this, although Hartmut is still listed as the author.
 
-In the fall of 2000, Hartmut Goebel
-[took over maintaining the code](https://groups.google.com/forum/#!searchin/comp.lang.python/hartmut$20goebel/comp.lang.python/35s3mp4-nuY/UZALti6ujnQJ). The
-first subsequent public release announcement that I can find is
-["decompyle - A byte-code-decompiler version 2.2 beta 1"](https://mail.python.org/pipermail/python-announce-list/2002-February/001272.html).
+The project exists not only on [github](https://github.com/gstarnberger/uncompyle) but also on
+[bitbucket](https://bitbucket.org/gstarnberger/uncompyle) and later the defunct [google
+code](https://code.google.com/archive/p/unpyc/) under the name _unpyc_. The git/svn history goes back to 2009. Somewhere in there the name was changed from "decompyle" to "unpyc" by Keknehv, and then to "uncompyle" by Guenther Starnberger.
 
-From the CHANGES file found in
-[the tarball for that release](http://old-releases.ubuntu.com/ubuntu/pool/universe/d/decompyle2.2/decompyle2.2_2.2beta1.orig.tar.gz),
-it appears that Hartmut did most of the work to get this code to
-accept the full Python language. He added precedence to the table
-specifiers, support for multiple versions of Python, the
-pretty-printing of docstrings, lists, and hashes. He also wrote test and verification routines of
-deparsed bytecode, and used this in an extensive set of tests that he also wrote. He says he could verify against the
-entire Python library. However I have subsequently found small and relatively obscure bugs in the decompilation code.
+The name Thomas Grainger isn't found in (m)any of the commits in the several years of active development. First Keknehv worked on this up to Python 2.5 or so while acceping Python bytecode back to 2.0 or so. Then "hamled" made a few commits earler on, while Eike Siewertsen made a few commits later on. But mostly "wibiti", and Guenther Starnberger got the code to where uncompyle2 was around 2012.
 
-decompyle2.2 was packaged for Debian (sarge) by
-[Ben Burton around 2002](https://packages.qa.debian.org/d/decompyle.html). As
-it worked on Python 2.2 only long after Python 2.3 and 2.4 were in
-widespread use, it was removed.
+While John Aycock and Hartmut Goebel were well versed in compiler technology, those that have come afterwards don't seem to have been as facile in it.  Furthermore, documentation or guidance on how the decompiler code worked, comparison to a conventional compiler pipeline, how to add new constructs, or debug grammars was weak. Some of the grammar tracing and error reporting was a bit weak as well.
 
-[Crazy Compilers](http://www.crazy-compilers.com/decompyle/) offers a
-byte-code decompiler service for versions of Python up to 2.6. As
-someone who worked in compilers, it is tough to make a living by
-working on compilers. (For example, based on
-[John Aycock's recent papers](http://pages.cpsc.ucalgary.ca/~aycock/)
-it doesn't look like he's done anything compiler-wise since SPARK). So
-I hope people will use the crazy-compilers service. I wish them the
-success that his good work deserves.
+Given this, perhaps it is not surprising that subsequent changes tended to shy away from using the built-in compiler technology mechanisms and addressed problems and extensions by some other means.
 
-Dan Pascu did a bit of work from late 2004 to early 2006 to get this
-code to handle first Python 2.3 and then 2.4 bytecodes. Because of
-jump optimization introduced in the CPython bytecode compiler at that
-time, various JUMP instructions were classified to assist parsing For
-example, due to the way that code generation and line number table
-work, jump instructions to an earlier offset must be looping jumps,
-such as those found in a "continue" statement; "COME FROM"
-instructions were reintroduced.  See
-[RELEASE-2.4-CHANGELOG.txt](https://github.com/rocky/python-uncompyle6/blob/master/DECOMPYLE-2.4-CHANGELOG.txt)
-for more details here. There wasn't a public release of RELEASE-2.4
-and bytecodes other than Python 2.4 weren't supported. Dan says the
-Python 2.3 version could verify the entire Python library. But given
-subsequent bugs found like simply recognizing complex-number constants
-in bytecode, decompilation wasn't perfect.
+Specifically, in `uncompyle`, decompilation of python bytecode 2.5 & 2.6 is done by transforming the byte code into a pseudo-2.7 Python bytecode and is based on code from Eloi Vanderbeken. A bit of this could have been easily added by modifying grammar rules.
 
-Next we get to ["uncompyle" and
-PyPI](https://pypi.python.org/pypi/uncompyle/1.1) and the era of
-public version control. (Dan's code although not public used
-[darcs](http://darcs.net/) for version control.)
 
-In contrast to _decompyle_, _uncompyle_ at least in its final versions,
-runs only on Python 2.7. However it accepts bytecode back to Python
-2.5. Thomas Grainger is the package owner of this, although Hartmut is
-still listed as the author.
+# uncompyle2, uncompyle3, uncompyle6
 
-The project exists not only on
-[github](https://github.com/gstarnberger/uncompyle) but also on
-[bitbucket](https://bitbucket.org/gstarnberger/uncompyle) and later
-the defunct [google
-code](https://code.google.com/archive/p/unpyc/). The git/svn history
-goes back to 2009. Somewhere in there the name was changed from
-"decompyle" to "unpyc" by Keknehv, and then to "uncompyle" by Guenther Starnberger.
+`Uncompyle6`, which I started in 2015, owes its existence to the fork of [uncompyle2](https://github.com/Mysterie/uncompyle2) by Myst herie (Mysterie) whose first commit picks up at 2012. I chose this since it seemed to have been at that time the most actively, if briefly, worked on. Also starting around 2012 is Dark Fenx's [uncompyle3](https://github.com/DarkFenX/uncompyle3) which I used for inspiration for Python3 support.
 
-The name Thomas Grainger isn't found in (m)any of the commits in the
-several years of active development. First Keknehv worked on this up
-to Python 2.5 or so while acceping Python bytecode back to 2.0 or
-so. Then hamled made a few commits earler on, while Eike Siewertsen
-made a few commits later on. But mostly wibiti, and Guenther
-Starnberger got the code to where uncompyle2 was around 2012.
-
-While John Aycock and Hartmut Goebel were well versed in compiler
-technology, those that have come afterwards don't seem to have been as
-facile in it.  Furthermore, documentation or guidance on how the
-decompiler code worked, comparison to a conventional compiler
-pipeline, how to add new constructs, or debug grammars was weak. Some
-of the grammar tracing and error reporting was a bit weak as well.
-
-Given this, perhaps it is not surprising that subsequent changes
-tended to shy away from using the built-in compiler technology
-mechanisms and addressed problems and extensions by some other means.
-
-Specifically, in `uncompyle`, decompilation of python bytecode 2.5 &
-2.6 is done by transforming the byte code into a pseudo-2.7 Python
-bytecode and is based on code from Eloi Vanderbeken. A bit of this
-could have been easily added by modifying grammar rules.
-
-This project, `uncompyle6`, abandons that approach for various
-reasons. Having a grammar per Python version is much cleaner and it
-scales indefinitely. That said, we don't have entire copies of the
-grammar, but work off of differences from some neighboring version.
-
-Should there be a desire to rebase or start a new base version to work
-off of, say for some future Python version, that can be done by
-dumping a grammar for a specific version after it has been loaded
-incrementally. You can get a full dump of the grammar by profiling the
-grammar on a large body of Python source code.
-
-Another problem with pseudo-2.7 bytecode is that that we need offsets
-in fragment deparsing to be exactly the same as the bytecode; the
-transformation process can remove instructions.  _Adding_ instructions
-with psuedo offsets is however okay.
-
-`Uncompyle6` however owes its existence to the fork of `uncompyle2` by
-Myst herie (Mysterie) whose first commit picks up at
-2012. I chose this since it seemed to have been at that time the most
-actively, if briefly, worked on. Also starting around 2012 is Dark
-Fenx's uncompyle3 which I used for inspiration for Python3 support.
-
-I started working on this late 2015, mostly to add fragment support.
-In that, I decided to make this runnable on Python 3.2+ and Python 2.6+
-while, handling Python bytecodes from Python versions 2.5+ and
-3.2+. In doing so, it has been expedient to separate this into three
-projects:
+I started working on this late 2015, mostly to add fragment support. In that, I decided to make this runnable on Python 3.2+ and Python 2.6+ while handling Python bytecodes from Python versions 2.5+ and
+3.2+. In doing so, it was expedient to separate this into three projects:
 
 * marshaling/unmarshaling, bytecode loading and disassembly ([xdis](https://pypi.python.org/pypi/xdis)),
 * parsing and tree building ([spark_parser](https://pypi.python.org/pypi/spark_parser)),
 * this project - grammar and semantic actions for decompiling
   ([uncompyle6](https://pypi.python.org/pypi/uncompyle6)).
 
+ `uncompyle6`, abandons the idea found in some 2.7 version of `uncompyle` that support Python 2.6 and 2.5 by trying to rewite opcodes at the bytecode level.
 
-Over the many years, code styles and Python features have
-changed. However brilliant the code was and still is, it hasn't really
-had a single public active maintainer. And there have been many forks
-of the code.  I have spent a great deal of time trying to organize and
-modularize the code so that it can handle more Python versions more
-gracefully (with still only moderate success).
+Having a grammar per Python version is simpler to maintain, cleaner and it scales indefinitely.
 
-That it has been in need of an overhaul has been recognized by the
-Hartmut a decade an a half ago:
+Over the many years, code styles and Python features have changed. However brilliant the code was and still is, it hasn't really had a single public active maintainer. And there have been many forks of the code.
+
+That this code has been in need of an overhaul has been recognized by the Hartmut more than two decades ago.
 
 [decompyle/uncompile__init__.py](https://github.com/gstarnberger/uncompyle/blob/master/uncompyle/__init__.py#L25-L26)
 
     NB. This is not a masterpiece of software, but became more like a hack.
     Probably a complete rewrite would be sensefull. hG/2000-12-27
 
-This project deparses using an Earley-algorithm parse with lots of
-massaging of tokens and the grammar in the scanner
-phase. Earley-algorithm parsers are context free and tend to be linear
-if the grammar is LR or left recursive. There is a technique for
-improving LL right recursion, but our parser doesn't have that yet.
+In 2021, I created three git branches in order to allow the decompiler to run on a wide variety of Python versions from 2.4 up to 3.10. (Note this doesn't mean we decompile these versions. In fact we decompile starting from Python 1.0 up to Python 3.8 and no later than that.)
 
-Another approach to decompiling, and one that doesn't use grammars is
-to do something like simulate execution symbolically and build
-expression trees off of stack results. Control flow in that approach
-still needs to be handled somewhat ad hoc.  The two important projects
-that work this way are [unpyc3](https://code.google.com/p/unpyc3/) and
-most especially [pycdc](https://github.com/zrax/pycdc) The latter
-project is largely by Michael Hansen and Darryl Pogue. If they
-supported getting source-code fragments, did a better job in
-supporting Python more fully, and had a way I could call it from
-Python, I'd probably would have ditched this and used that. The code
-runs blindingly fast and spans all versions of Python, although more
-recently Python 3 support has been lagging. The code is impressive for
-its smallness given that it covers many versions of Python. However, I
-think it has reached a scalability issue, same as all the other
-efforts. To handle Python versions more accurately, I think that code
-base will need to have a lot more code specially which specializes for
-Python versions. And then it will run into a modularity problem.
+Using the separate git branches allows me to continually improve the coding style and add feature support while still supporting older Pythons. Supporting older Pythons is nice (but not strictly necessary) when you want to debug decompilation on older Pythons.
 
-Tests for the project have been, or are being, culled from all of the
-projects mentioned. Quite a few have been added to improve grammar
-coverage and to address the numerous bugs that have been encountered.
+I have spent a great deal of time trying to organize, modularize and even modernize the code so that it can handle more Python versions more gracefully (with still only moderate success).
 
-If you think, as I am sure will happen in the future, "hey, I can just
-write a decompiler from scratch and not have to deal with all all of
-the complexity here", think again. What is likely to happen is that
-you'll get at best a 90% solution working for a single Python release
-that will be obsolete in about a year, and more obsolete each
-subsequent year.  Writing a decompiler for Python gets harder as it
-Python progresses, so writing one for Python 3.7 isn't as easy as it
-was for Python 2.2.  That said, if you still feel you want to write a
-single version decompiler, look at the test cases in this project and
-talk to me. I may have some ideas.
+Tests for the project have been, or are being, culled from all of the projects mentioned above or below. Quite a few have been added to improve grammar coverage and to address the numerous bugs that have been encountered.
 
 
-For a little bit of the history of changes to the Earley-algorithm parser,
-see the file [NEW-FEATURES.rst](https://github.com/rocky/python-spark/blob/master/NEW-FEATURES.rst) in the [python-spark github repository](https://github.com/rocky/python-spark).
+# unpyc3 and pydc
 
-NB. If you find mistakes, want corrections, or want your name added
-(or removed), please contact me.
+Another approach to decompiling, and one that doesn't use grammars is to do something like simulate execution symbolically and build expression trees off of stack results. Control flow in that approach
+still needs to be handled somewhat ad hoc.  The two important projects that work this way are [unpyc3](https://code.google.com/p/unpyc3/) and most especially [pycdc](https://github.com/zrax/pycdc) The latter
+project is largely by Michael Hansen and Darryl Pogue. If they supported getting source-code fragments, did a better job in supporting Python more fully, and had a way I could call it from Python, I'd probably would have ditched this and used that. The code runs blindingly fast and spans all versions of Python, although more recently Python 3 support has been lagging. The code is impressive for its smallness given that it covers many versions of Python. However, I think it has reached a scalability issue, same as all the other efforts. To handle Python versions more accurately, I think that code base will need to have a lot more code specially which specializes for Python versions. And then it will run into a modularity problem.
+
+# So you want to write a decompiler for Python?
+
+If you think, as I am sure will happen in the future, "hey, I can just write a decompiler from scratch and not have to deal with all all of the complexity in uncompyle6", think again. What is likely to happen is that you'll get at best a 90% solution working for a single Python release that will be obsolete in about a year, and more obsolete each subsequent year.
+
+Writing a decompiler for Python gets harder as it Python progresses. Writing decompiler for Python 3.7 isn't as easy as it was for Python 2.2. For one thing, now that Python has a well-established AST, that opens another interface by which code can be improved.
+
+In Python 3.10 I am seeing (for the first time?) bytecode getting moved around so that it is no longer the case that line numbers have to be strictly increasing as bytecode offsets increase. And I am seeing dead code appear as well.
+
+That said, if you still feel you want to write a single version decompiler, look at the test cases in this project and talk to me. I may have some ideas that I haven't made public yet. See also what I've wrtten about the on how this code works and on [decompilation in dynamic runtime languages](http://rocky.github.io/Deparsing-Paper.pdf) in general.
+
+
+
+# Earley Algorithm Parser
+
+This project deparses using an Earley-algorithm parse. But in order to do this accurately, the process of tokenization is a bit more involved in the scanner. We don't just disassemble bytecode and use the opcode name. That aspect hasn't changed from the very first decompilers. However understanding _what_ information needs to be made explicit and what pseudo instructions to add that accomplish this has taken some time to understand.
+
+Earley-algorithm parsers have gotten negative press, most notably by the dragon book. Having used this a bit, I am convinced having a system that handles ambiguous grammars is the right thing to do and matches the problem well. Iin practice the speed of the parser isn't a problem when one understand what's up. And this has taken a little while to understand.
+Earley-algorim parsers for context free languages or languages that are to a large extent context free and tend to be linear and the grammar stears towards left recursive rules. There is a technique for improving LL right recursion, but our parser doesn't have that yet.
+
+The [decompiling paper](http://rocky.github.io/Deparsing-Paper.pdf) discusses these aspects in a more detail.
+
+
+For a little bit of the history of changes to the Earley-algorithm parser, see the file [NEW-FEATURES.rst](https://github.com/rocky/python-spark/blob/master/NEW-FEATURES.rst) in the [python-spark github repository](https://github.com/rocky/python-spark).
+
+NB. If you find mistakes, want corrections, or want your name added (or removed), please contact me.

HOW-TO-REPORT-A-BUG.md

@@ -1,60 +1,66 @@
 <!-- markdown-toc start - Don't edit this section. Run M-x markdown-toc-refresh-toc -->
 **Table of Contents**
 
-- [The difficulty of the problem](#the-difficulty-of-the-problem)
+- [Ethics](#ethics)
+- [The importance of your bug report](#the-importance-of-your-bug-report)
+- [The difficulty of the problem and your bug](#the-difficulty-of-the-problem-and-your-bug)
 - [Is it really a bug?](#is-it-really-a-bug)
     - [Do you have valid bytecode?](#do-you-have-valid-bytecode)
     - [Semantic equivalence vs. exact source code](#semantic-equivalence-vs-exact-source-code)
 - [What to send (minimum requirements)](#what-to-send-minimum-requirements)
 - [What to send (additional helpful information)](#what-to-send-additional-helpful-information)
     - [But I don't *have* the source code!](#but-i-dont-have-the-source-code)
-    - [But I don't *have* the source code and am incapable of figuring how how to do a hand disassembly!](#but-i-dont-have-the-source-code-and-am-incapable-of-figuring-how-how-to-do-a-hand-disassembly)
+        - [But I don't *have* the source code and am incapable of figuring how to do a hand disassembly!](#but-i-dont-have-the-source-code-and-am-incapable-of-figuring-how-to-do-a-hand-disassembly)
 - [Narrowing the problem](#narrowing-the-problem)
 - [Karma](#karma)
 - [Confidentiality of Bug Reports](#confidentiality-of-bug-reports)
-- [Ethics](#ethics)
 
 <!-- markdown-toc end -->
-# The difficulty of the problem
+
+TL;DR (too long; didn't read)
+
+* Don't do something illegal. And don't ask me to do something illegal or help you do something illegal
+* We already have an infinite supply of decompilation bugs that need fixing, and an automated mechanism for finding more. Decompilation bugs get addressed by easiness to fix and by whim. If you expect yours to be fixed ahead of those, you need to justify why.
+* When asking for help, you may be asked for what you've tried on your own first. There are plenty of sources of information about this code.
+* If you are looking for *timely* help or support, well, that is typically known paid service. I don't really have a mechanism for that since I have a full-time job. But supporting the project is an approximation.
+* Submitting a bug or issue report that is likely to get acted upon may require a bit of effort on your part to make it easy for the problem solver. If you are not willing to do that, please don't waste our time. As indicated above, supporting the project will increase the likelihood of your issue getting noticed and acted upon.
+
+# Ethics
+
+I do not condone using this program for unethical or illegal purposes. More detestable, at least to me, is asking for help to assist you in something that might not legitimate.
+
+Don't use the issue tracker for such solicitations. To try to stave off illegitimate behavior, you should note that the issue tracker, the code, and bugs mentioned in that are in the open: there is no
+confidentiality. You may be asked about the authorship or claimed ownership of the bytecode. If I think something is not quite right, I may label the issue questionable which may make the it easier those who are looking for illegal activity.
+
+
+# The importance of your bug report
+
+For many open-source projects bugs where the expectation is that bugs are rare, reporting bugs in a *thoughtful* way can be helpful. See also [How to Ask Questions the Smart Way](http://www.catb.org/~esr/faqs/smart-questions.html).
+
+In this project though, most of the bug reports boil down to the something like: I have I am trying to reverse engineer some code that I am not the author/owner and that person doesn't want me to have access to. I am hitting a problem somewhere along the line which might have to do with decompilation, but it could be something else like how the bytecode was extracted, some problem in deliberately obfuscated code, or the use some kind of Python bytecode version that isn't supported by the decompiler.
+
+While you are free to report these, unless you sponsor the project, I may close them with about the same amount of effort spent that I think was used to open the report for them. And if you spent a considerable amount of time to create the bug report but didn't follow instructions given here and in the issue template, I am sorry in advance. Just go back, read, and follow instructions.
+
+This project already has an infinite supply of bugs that have been narrowed to the most minimal form and where I have source code to compare against. And in the unlikely event this supply runs out, I have automated means for generating *another* infinite supply.
+
+In this project the task of justifying why addressing your bug is of use to the community, and why it should be prioritized over the others, is the bug reporter's responsibility.
+
+While in the abstract, I have no problem answering questions about how to read a Python traceback or install Python software, or trying to understand what is going wrong in your particular setup, I am not a paid support person and there other things I'd rather be doing with my limited volunteer time. So save us both time, effort, and aggravation: use other avenues like StackOverflow. Again, justifying why you should receive unpaid help is the help requester's responsibility.
+
+
+# The difficulty of the problem and your bug
 
 This decompiler is a constant work in progress: Python keeps
 changing, and so does its code generation.
 
-There is no Python decompiler yet that I know about that will
-decompile everything. Overall, I think this one probably does the best
-job of *any* Python decompiler that handles such a wide range of
-versions.
+There is no Python decompiler yet that I know about that will decompile everything. Overall, I think this one probably does the best job of *any* Python decompiler that handles such a wide range of versions.
 
-But at any given time, there are a number of valid Python bytecode
-files that I know of that will cause problems. See, for example, the
-list in
+But at any given time, there are a number of valid Python bytecode files that I know of that will cause problems. See, for example, the list in
 [`test/stdlib/runtests.sh`](https://github.com/rocky/python-uncompyle6/blob/master/test/stdlib/runtests.sh).
 
-But I understand: you would the bugs _you_ encounter addressed before
-all the other known bugs.
+There are far more bug reporters than there are bug fixers.
 
-From my standpoint, the good thing about the bugs listed in
-`runtests.sh` is that each test case is small and isolated to a single
-kind of problem. And I'll tend to fix easier, more isolated cases than
-generic "something's wrong" kinds of bugs where I'd have to do a bit
-of work to figure out what's up, if not use some sort of mind reading,
-make some guesses, and perform some experiments to see if the guesses
-are correct. I can't read minds, nor am I into guessing games; I'd
-rather devote the effort spent instead towards fixing bugs that are
-precisely defined.
-
-And it often turns out that by just fixing the well-defined and
-prescribed cases, the ill-defined amorphous cases as well will get
-handled as well.
-
-In sum, you may need to do some work to have the bug you have found
-handled before the hundreds of other bugs, and other things I could be
-doing.
-
-No one is getting paid to work to work on this project, let alone the
-bugs you may have an interest in. If you require decompiling bytecode
-immediately, consider using a decompilation service, listed further
-down in this document.
+Unless you are a sponsor of this project, it may take a while, maybe a week or so, before the bug report is noticed, let alone acted upon. Things eventually get fixed, but it may take years. And if your bug hasn't been narrowed, it might happen as a result of some other bug fix.
 
 # Is it really a bug?
 
@@ -62,7 +68,7 @@ down in this document.
 ## Do you have valid bytecode?
 
 As mentioned in README.rst, this project doesn't handle obfuscated
-code. See README.rst for suggestions for how to remove some kinds of
+code, release candidates, and the most recent versions of Python: version 3.9 and up. See README.rst for suggestions for how to remove some kinds of
 obfuscation.
 
 Checking if bytecode is valid is pretty simple: disassemble the code.
@@ -72,18 +78,7 @@ disassembler called `pydisasm`.
 
 ## Semantic equivalence vs. exact source code
 
-Consider how Python compiles something like "(x*y) + 5". Early on
-Python creates an "abstract syntax tree" (AST) for this. And this is
-"abstract" in the sense that unimportant, redundant or unnecessary
-items have been removed. Here, this means that any notion that you
-wrote "x+y" in parenthesis is lost, since in this context they are
-unneeded. Also lost is the fact that the multiplication didn't have
-spaces around it while the addition did. It should not come as a
-surprise then that the bytecode which is derived from the AST also has
-no notion of such possible variation. Generally this kind of thing
-isn't noticed since the Python community has laid out a very rigid set
-of formatting guidelines; and it has largely beaten the community into
-compliance.
+Consider how Python compiles something like "(x*y) + 5". Early on Python creates an "abstract syntax tree" (AST) for this. And this is "abstract" in the sense that unimportant, redundant or unnecessary items have been removed. Here, this means that any notion that you wrote "x+y" in parenthesis is lost, since in this context they are unneeded. Also lost is the fact that the multiplication didn't have spaces around it while the addition did. It should not come as a surprise then that the bytecode which is derived from the AST also has no notion of such possible variation. Generally this kind of thing isn't noticed since the Python community has laid out a very rigid set of formatting guidelines; and it has largely beaten the community into compliance.
 
 Almost all versions of Python can perform some sort of code
 improvement that can't be undone. In earlier versions of Python it is
@@ -144,8 +139,7 @@ if False:
 
 Python will eliminate the entire "if" statement.
 
-So just because the text isn't the same, does not
-necessarily mean there's a bug.
+So just because the text isn't the same, this does not necessarily mean there's a bug.
 
 # What to send (minimum requirements)
 
@@ -166,7 +160,7 @@ Also try to narrow the bug. See below.
 Some kind folks also give the invocation they used and the output
 which usually includes an error message produced. This is
 helpful. From this, I can figure out what OS you are running this on
-and what version of *uncomplye6* was used. Therefore, if you _don't_
+and what version of *uncompyle6* was used. Therefore, if you _don't_
 provide the input command and the output from that, please give:
 
 * _uncompyle6_ version used
@@ -176,30 +170,18 @@ provide the input command and the output from that, please give:
 
 ## But I don't *have* the source code!
 
-Sure, I get it. No problem. There is Python assembly code on parse
-errors, so simply by hand decompile that. To get a full disassembly,
-use `pydisasm` from the [xdis](https://pypi.python.org/pypi/xdis)
-package. Opcodes are described in the documentation for
-the [dis](https://docs.python.org/3.6/library/dis.html) module.
+There is Python assembly code on parse errors, so simply by hand decompile that. To get a full disassembly, use `pydisasm` from the [xdis](https://pypi.python.org/pypi/xdis) package. Opcodes are described in the documentation for the [dis](https://docs.python.org/3.6/library/dis.html) module.
 
 ### But I don't *have* the source code and am incapable of figuring how to do a hand disassembly!
 
-Well, you could learn. No one is born into this world knowing how to
-disassemble Python bytecode. And as Richard Feynman once said, "What
-one fool can learn, so can another."
+Well, you could learn. No one is born into this world knowing how to disassemble Python bytecode. And as Richard Feynman once said, "What one fool can learn, so can another."
 
-If this is too difficult, or too time consuming, or not of interest to
-you, then perhaps what require is a decompilation service. [Crazy
-Compilers](http://www.crazy-compilers.com/decompyle/) offers a
-byte-code decompiler service for versions of Python up to 2.6. (If
-there are others around let me know and I'll list them here.)
+If this is too difficult, or too time consuming, or not of interest to you, then you might consider [sponsoring](https://github.com/sponsors/rocky) the project. [Crazy
+Compilers](http://www.crazy-compilers.com/decompyle/) offers a byte-code decompiler service for versions of Python up to 2.6. (If there are others around let me know and I'll list them here.) Don't be surprised if I ask you to pay for work (if I think the work is ethical) when you want me to work on your problem that I think isn't of interest or benefit to anyone but yourself or a small limited number of people, or I think the need is questionable.
 
 # Narrowing the problem
 
-I don't need or want the entire source code base for the file(s) or
-module(s) can't be decompiled. I just need those file(s) or module(s).
-If there are problems in several files, file a bug report for each
-file.
+I don't need or want the entire source code base for the file(s) or module(s) can't be decompiled. I just need those file(s) or module(s). If there are problems in several files, file a bug report for each file.
 
 Python modules can get quite large, and usually decompilation problems
 occur in a single function or maybe the main-line code but not any of
@@ -217,33 +199,29 @@ likely the problem will be fixed and fixed sooner.
 # Karma
 
 I realize that following the instructions given herein puts a bit of
-burden on the bug reporter. In my opinion, this is justified as
-attempts to balance somewhat the burden and effort needed to fix the
-bug and the attempts to balance number of would-be bug reporters with
-the number of bug fixers. Better bug reporters are more likely to move
-in the category of bug fixers.
+burden on the bug reporter. This is justified since it attempts to balance
+the burden and effort needed to fix the bug with the amount of effort to report the problem. And it attempts
+to balance number of would-be bug reporters with the number of bug
+fixers. Better bug reporters are more likely to move in the category
+of bug fixers.
 
 The barrier to reporting a big is pretty small: all you really need is
 a github account, and the ability to type something after clicking
 some buttons. So the reality is that many people just don't bother to
 read these instructions, let alone follow it to any simulacrum.
 
-And the reality is also that bugs sometimes get fixed even though
-these instructions are not followed.
+That said, bugs sometimes get fixed even though these instructions are not followed.
 
-So one factors I may take into consideration is the bug reporter's karma.
+I may take into consideration is the bug reporter's karma.
 
-* Have you demonstrably contributed to open source? I may look at your
-  github profile to see what contributions you have made, how popular
-  those contributions are, or how popular you are.
-* How appreciative are you? Have you starred this project that you are
-  seeking help from? Have you starred _any_ github project? And the above
-  two kind of feed into ...
+* Have you demonstrably contributed to open source? I may look at your github profile to see what contributions you have made, how popular those contributions are, or how popular you are.
+* How appreciative are you? Have you starred this project that you are seeking help from? Have you starred _any_ github project? And the above two kind of feed into ...
 * Attitude. Some people feel that they are doing me and the world a
-  great favor by just pointing out that there is a problem whose solution
-  would greatly benefit them. Perhaps this is why they feel that
-  instructions are not to be followed by them, nor any need for
-  showing evidence gratitude when help is offered them.
+  great favor by just pointing out that there is a problem whose
+  solution would greatly benefit them. (This might account partially
+  for the fact that those that have this attitude often don't read or
+  follow instructions such as those given here.)
+
 
 # Confidentiality of Bug Reports
 
@@ -255,16 +233,6 @@ remains would not be an issue.
 However feel free to remove any comments, and modify variable names
 or constants in the source code.
 
-# Ethics
+If there is some legitimate reason to keep confidentiality, you can contact me by email to explain the extenuating circumstances. However I tend to discard without reading anonymous email.
 
-I do not condone using this program for unethical or illegal purposes.
-More detestable, at least to me, is asking for help to assist you in
-something that might not legitimate.
-
-Don't use the issue tracker for such solicitations. To try to stave
-off illegitimate behavior, you should note that the issue tracker, the
-code, and bugs mentioned in that are in the open: there is no
-confidentiality. You may be asked about the authorship or claimed
-ownership of the bytecode. If I think something is not quite right, I
-may label the issue questionable which may make the it easier those
-who are looking for illegal activity.
+Private consulting available via https://calendly.com/rb3216 rates: $150 for 30 minutes; $250 for 60 minutes.

Makefile

@@ -11,7 +11,10 @@ RM      ?= rm
 LINT    = flake8
 
 #EXTRA_DIST=ipython/ipy_trepan.py trepan
-PHONY=all check clean distcheck pytest check-long dist distclean lint flake8 test rmChangeLog clean_pyc
+PHONY=all check check-2.7 check-3.4 \
+     clean distcheck pytest check-long check-short \
+     dist distclean lint flake8 test rmChangeLog clean_pyc \
+     2.6 5.0 5.3 5.6 5.8 7.2 7.3 check-short
 
 TEST_TYPES=check-long check-short check-2.7 check-3.4
 
@@ -47,8 +50,16 @@ check-3.8:
 # Skip for now
 2.6 5.0 5.3 5.6 5.8:
 
-#:PyPy pypy3-2.4.0 Python 3:
-pypy-3.2 2.4:
+#:PyPy pypy3-2.4.0 Python 3.6.1:
+7.1 pypy-3.2 2.4:
+	$(MAKE) -C test $@
+
+#:PyPy versions
+7.2 7.3:
+	$(MAKE) -C test $@
+
+#:pyston versions
+2.3:
 	$(MAKE) -C test $@
 
 #: Run py.test tests

NEWS.md

@@ -1,3 +1,313 @@
+3.9.0: 2022-12-22
+=================
+
+* deparse generator expressions for Python 3.0 .. 3.2
+* Python 3.0 list comprehension.
+* Fix Issues #310, #344, #377, #391, #409, #414
+* Limited support for 3.8+ f-string "=" specifier
+* Correct 2.5-7 relative import formatting
+* Miscellaneous bug fixing
+* remove \n in lambda
+* Python 2.6 gramar cleanup
+* Correct some Python 2.6 chain compare decompilation
+* Ensure no parenthesis subscript slices
+* Correct 2.x formatting "slice2" nonterminal
+* Correct 3.7 imports
+* Improve "async for" parsing
+* Handle BUILD_MAP opcode
+* match Python AT better
+* Correct 3.7 positional args
+* PyPy 3.7 and PyPy 3.8 support
+* Miscellaneous linting, isorting, blacking
+
+3.8.0: 2021-10-29
+=================
+
+* Better handling of invalid bytecode magic
+* Support running from 3.9 and 3.10 although we do not support those bytecodes
+* Redo version comparisons using tuples instead of floats. This is needed for Python 3.10
+* Split out into 3 branches so that the master branch can assume Python 3.6+ conventions, especially type annotations
+* Source Fragment fixes
+* Lambda-bug fixes #360
+* Bug fixes
+
+3.7.4: 2020-8-05
+================
+
+* Fragment parsing was borked. This means deparsing in trepan2/trepan3k was broken
+* 3.7+: narrow precedence for call tatement
+* del_stmt -> delete to better match Python AST
+* 3.8+ Add another `forelsestmt` (found only in a loop)
+* 3.8+ Add precedence on walrus operator
+* More files blackened
+* bump min xdis version
+
+3.7.3: 2020-7-25
+================
+
+Mostly small miscellaneous bug fixes
+
+* `__doc__ = DocDescr()` from `test_descr.py` was getting confused as a docstring.
+* detect 2.7 exchandler range better
+* Add for .. else reduction checks on 2.6 and before
+* Add reduce check for 2.7 augmented assign
+* Add VERSION in a pydoc-friendly way
+
+
+3.7.2: 2020-6-27
+================
+
+* Use newer xdis
+* Docstrings (again) which were broken again on earlier Python
+* Fix 2.6 and 2.7 decompilation bug in handling "list if" comprehensions
+
+
+
+3.7.1: 2020-6-12 Fleetwood66
+====================================================
+
+Released to pick up new xdis version which has fixes to read bytestings better on 3.x
+
+* Handle 3.7+ "else" branch removal adAs seen in `_cmp()` of `python3.8/distutils/version.py` with optimization `-O2`
+* 3.6+ "with" and "with .. as" grammar improvements
+* ast-check for "for" loop was missing some grammar rules
+
+3.7.0: 2020-5-19 Primidi 1st Prairial - Alfalfa - HF
+====================================================
+
+The main impetus for this release is to pull in the recent changes from xdis.
+We simplify imports using xdis 4.6.0.
+
+There were some bugfixes to Python 3.4-3.8. See the ChangeLog for details
+
+
+3.6.7: 2020-4-27 xdis again
+===========================
+
+More upheaval in xdis which we need to track here.
+
+3.6.6: 2020-4-20 Love in the time of Cholera
+============================================
+
+The main reason for this release is an incompatablity bump in xdis which handles
+3.7 SipHash better.
+
+* Go over "yield" as an expression precidence
+* Some small alignment with code in decompyle3 for "or" and "and" was done
+
+
+3.6.5: 2020-4-1 April Fool
+==========================
+
+Back port some of the changes in decompile3 here which mostly helps 3.7 and 3.8 decompilation, although this may also help 3.6ish versions too.
+
+- Handle nested `async for in for...`  and better async comprehension detection via `xdis`.  Still more work is needed.
+- include token number in listings when `-g` and there is a parser error
+- remove unneeded `Makefile`s now that remake 4.3+1.5dbg is a thing that has `-c`
+- Bug in finding annotations in functions with docstrings
+- Fix bug found by 2.4 sre_parse.py testing
+- Fix `transform` module's  `ifelseif` bugs
+- Fix bug in 3.0 name module detection
+- Fix docstring detection
+
+3.6.4: 2020-2-9 Plateau
+=======================
+
+The main focus in this release was fix some of the more glaring problems creapt in from the last release due to that refactor.
+
+`uncompyle6` code is at a plateau where what is most needed is a code refactoring. In doing this, until everything refactored and replaced, decomplation may get worse.
+Therefore, this release largely serves as a checkpoint before more major upheaval.
+
+The upheaval, in  started last release, I believe the pinnicle was around c90ff51 which wasn't a release. I suppose I should tag that.
+
+After c90ff5, I started down the road of redoing control flow in a more comprehensible, debuggable, and scalable way. See [The Control Flow Mess](https://github.com/rocky/python-uncompyle6/wiki/The-Control-Flow-Mess)
+
+The bulk of the refactoring going on in the [decompyle3](https://github.com/rocky/python-decompil3) project, but I try to trickle down the changes.
+
+It is tricky because the changes are large and I have to figure decompose things so that little testable pieces can be done. And there is also the problem that what is in decompyle3 is incomplete as well.
+
+Other than control flow, another change that will probably happen in the next release is to redo the grammar for lambda expressions. Right now, we treat them as Python statements, you know, things with compound statements in them. But lambda aren't that. And so there is hackery to paper over difference making a statement out of an expression the wrong thing to do. For example, a return of an "and" expression can be expressed as nested "if" statements with return inside them, but the "if" variant of the bytecode is not valid in a lambda.
+
+In the decompyle3 code, I've gone down the road making the grammar goal symbol be an expression. This also offers the opportunity to split the grammar making parsing inside lambda not only more reliable because the wrong choices don't exist, but also simpler and faster because all those rules just need don't need to exist in parsing.
+
+I cringe in thinking about how the code has lived for so long without noticing such a simple stupidity, and lapse of sufficient thought.
+
+Some stats from testing. The below give numbers of decompiled tests from Python's test suite which succesfully ran
+
+```
+   Version  test-suites passing
+   -------  -------------------
+   2.4.6     243
+   2.5.6     265
+   2.6.9     305
+   3.3.7     300
+   3.4.10    304
+   3.5.9     260
+   3.6.10    236
+   3.7.6     306
+   3.8.1     114
+```
+
+Decompiled bytecode files distributed with Python (syntax check only):
+
+```
+2.7.17  647 files:   0 failed
+3.2.6   900 files:   0 failed
+3.3.7  1256 files:   0 failed
+3.4.10  800 files:   0 failed
+3.5.9   900 files:   0 failed
+3.6.10 1300 files:  28 failed
+```
+
+
+3.6.3: 2020-1-26 Martin and Susanne
+===================================
+
+Of late, every release fixes major gaps and embarrassments of the last release....
+
+And in some cases, like this one, exposes lacuna and rot.
+
+I now have [control] flow under control, even if it isn't the most optimal way.
+
+I now have greatly expanded automated testing.
+
+On the most recent Python versions I regularly decompile thousands of Python programs that are distributed with Python.  when it is possible, I then decompile Python's standard test suite distributed with Python and run the decompiled source code which basically checks itself. This amounts to about 250 test programs per version. This is in addition to the 3 CI testing services which do different things.
+
+Does this mean the decompiler works perfectly? No. There are still a dozen or so failing programs, although the actual number of bugs is probably smaller though.
+
+However, in perparation of a more major refactoring of the parser grammar, this release was born.
+
+In many cases, decompilation is better. But there are some cases where decompilation has gotten worse. For lack of time (and interest) 3.0 bytecode suffered a hit. Possibly some code in the 3.x range did too. In time and with cleaner refactored code, this will come back.
+
+Commit c90ff51 was a local maxiumum before, I started reworking the grammar to separate productions that were specific to loops versus those that are not in loops.
+In the middle of that I added another grammar simplication to remove singleton productions of the form `sstmts-> stmts`. These were always was a bit ugly, and complicated output.
+
+At any rate if decompilation fails, you can try c90ff51. Or another decompiler. `unpyc37` is pretty good for 3.7. wibiti `uncompyle2` is great for 2.7. `pycdc` is mediocre for Python before 3.5 or so, and not that good for the most recent Python. Geerally these programs will give some sort of answer even if it isn't correct.
+
+decompyle3 isn't that good for 3.7 and worse for 3.8, but right now it does things no other Python decompiler like `unpyc37` or `pycdc` does. For example, `decompyle3` handles variable annotations. As always, the issue trackers for the various programs will give you a sense for what needs to be done. For now, I've given up on reporting issues in the other decompilers because there are already enough issues reported, and they are just not getting fixed anyway.
+
+
+3.6.2: 2020-1-5 Samish
+======================
+
+Yet again the focus has been on just fixing bugs, mostly geared in the
+later 3.x range. To get some sense what sill needs fixing, consult
+test/stdlib/runtests.sh. And that only has a portion of what's known.
+
+`make_function.py` has gotten so complex that it was split out into 3 parts
+to handle different version ranges: Python <3, Python 3.0..3.6 and Python 3.7+.
+
+An important fix is that we had been dropping docstrings in Python 3 code as a result
+of a incomplete merge from the decompile3 base with respect to the transform phase.
+
+Also important (at least to me) is that we can now handle 3.6+
+variable type annotations.  Some of the decompile3 code uses that in
+its source code, and I now use variable annotations in conjunction
+with mypy in some of my other Python projects
+
+Code generation for imports, especially where the import is dotted
+changed a bit in 3.7; with this release are just now tracking that
+change better. For this I've added pseudo instruction
+`IMPORT_NAME_ATTR`, derived from the `IMPORT_NAME` instruction, to
+indicate when an import contains a dotted import. Similarly, code for
+3.7 `import .. as ` is basically the same as `from .. import`, the
+only difference is the target of the name changes to an "alias" in the
+former. As a result, the disambiguation is now done on the semantic
+action side, rathero than in parsing grammar rules.
+
+Some small specific fixes:
+
+* 3.7+ some chained compare parsing has been fixed. Other remain.
+* better if/else rule checking in the 3.4 and below range.
+* 3.4+ keyword-only parameter handling was fixed more generally
+* 3.3 .. 3.5 keyword-only parameter args in lambda was fixed
+
+
+3.6.1: 2019-12-10 Christmas Hannukah
+====================================
+
+Overall, as in the past, the focus has been on just fixing bugs, more geared
+in the later 3.x range. Handling "async for/with" in 3.8+ works better.
+
+Numerous bugs around handling `lambda` with keyword-only and `*` args in the
+3.0-3.8 have been fixed. However many still remain.
+
+`binary_expr` and `unary_expr` have been renamed to `bin_op` and
+`unary_op` to better correspond the Python AST names.
+
+Some work was done Python 3.7+ to handle `and` better; less was done
+along the lines of handling `or`. Much more is needed to improve
+parsing stability of 3.7+. More of what was done with `and` needs to
+be done with `or` and this will happen first in the "decompyle3"
+project.
+
+Later this will probably be extended backwards to handle the 3.6-
+versions better. This however comes with a big decompilation speed
+penalty. When we redo control flow this should go back to normal, but
+for now, accuracy is more important than speed.
+
+Another `assert` transform rule was added. Parser rules to distingish
+`try/finally` in 3.8 were added and we are more stringent about what
+can be turned into an `assert`. There was some grammar cleanup here
+too.
+
+A number of small bugs were fixed, and some administrative changes to
+make `make check-short` really be short, but check more throughly what
+it checks. minimum xdis version needed was bumped to include in the
+newer 3.6-3.9 releases. See the `ChangeLog` for details.
+
+
+3.6.0: 2019-12-10 gecko gecko
+=============================
+
+The main focus in this release was more accurate decompilation especially
+for 3.7 and 3.8. However there are some improvments to Python 2.x as well,
+including one of the long-standing problems of detecting the difference between
+`try ... ` and `try else ...`.
+
+With this release we now rebase Python 3.7 on off of a 3.7 base; This
+is also as it is (now) in decompyle3.  This facilitates removing some of the
+cruft in control-flow detection in the 2.7 uncompyle2 base.
+
+Alas, decompilation speed for 3.7 on is greatly increased. Hopefull
+this is temporary (cough, cough) until we can do a static control flow
+pass.
+
+Finally, runing in 3.9-dev is tolerated. We can disassemble, but no parse tables yet.
+
+
+3.5.1 2019-11-17 JNC
+====================
+
+- Pypy 3.3, 3.5, 3.6, and 3.6.9 support
+- bump xdis version to handle newer Python releases, e.g. 2.7.17, 3.5.8, and 3.5.9
+- Improve 3.0 decompilation
+    - no parse errors on stlib bytecode. However accurate translation in
+	  control-flow and and/or detection needs work
+- Remove extraneous iter() in "for" of list comprehension  Fixes #272
+- "for" block without a `POP_BLOCK `and confusing `JUMP_BACK` for `CONTINUE`. Fixes #293
+- Fix unmarshal incompletness detected in Pypy 3.6
+- Miscellaneous bugs fixed
+
+3.5.0 2019-10-12 Stony Brook Ride
+=================================
+
+- Fix fragment bugs
+   * missing `RETURN_LAST` introduced when adding transformation layer
+   * more parent entries on tokens
+- Preliminary support for decompiling Python 1.0, 1.1, 1.2, and 1.6
+   * Newer _xdis_ version needed
+
+3.4.1 2019-10-02
+================
+
+- Correct assert{,2} transforms Fixes #289
+- Fragment parsing fixes:
+     * Wasn't handling 3-arg `%p`
+   	 * fielding error in `code_deparse()`
+- Use newer _xdis_ to better track Python 3.8.0
+
+
 3.4.0 2019-08-24 Totoro
 =======================
 
@@ -105,8 +415,7 @@ Lots of decomplation bugs, especially in the 3.x series fixed. Don't worry thoug
 3.3.0 2019-04-14 Holy Week
 ==========================
 
-* First cut at Python 3.8 (many bugs remain)
-* Reinstate -c | --compile (compile before disassembly) option
+* First cut at Python 3.8 (many bug remain)
 * The usual smattering of bug and doc fixes
 
 3.2.6 2019-03-23 Mueller Report

NEW_FEATURES.rst

@@ -0,0 +1,221 @@
+Introduction
+============
+
+The original versions of this code up until the time I started were
+pretty awesome.  You can get a sense of this by running it.  For the
+most part it was remarkably fast, and a single module with few dependencies.
+
+Here I will largely give what are the major improvements over old code.
+
+This also serves to outline a little bit about what is in this code.
+
+See also `How does this code work? <https://github.com/rocky/python-uncompyle6/wiki/How-does-this-code-work%3F>`_.
+
+Old Cool Features
+==================
+
+Before getting to the new stuff, I'll describe cool things that was there before.
+
+I particularly liked the ability to show the assembly, grammar
+reduction rules as they occurred, and the resulting parse tree. It is
+neat that you could follow the process and steps that deparser takes,
+and in this not only see the result how the bytecode corresponds to
+the resulting source. Compare this with other Python decompilers.
+
+And of course also neat was that this used a grammar and table-driven
+approach to decompile.
+
+
+Expanding decompilation to multiple Python Versions
+==================================================
+
+Aside from ``pycdc``, most of the Python decompilers handle a small
+number of Python versions, if they supported more than one. And even
+when more than one version is supported if you have to be running the
+Python version that the bytecode was compiled for.
+
+There main reason that you have to be running the Python bytecode
+interpreter as the one you want to decompile largely stems from the
+fact that Python's ``dis`` module is often what is used and that has this limitation.
+
+``pycdc`` doesn't suffer this problem because it is written in C++,
+not Python.  Hartmut Goebel's code had provisions for multiple Python
+versions running from an interpreter different from the one that was
+running the decompiler. That however used compiled code in the process
+was tied a bit to the Python C headers for a particular version.
+
+You need to not only to account for different "marshal" and "unmarshal"
+routines for the different Python versions, but also, as the Python versions
+extend, you need a different code type as well.
+
+Enter ``xdis``
+--------------
+
+To handle all of these problems, I split off the marshal loading
+portion and disassembly routines into a separate module,
+`xdis <https://pypi.org/project/xdis/>`_. This also allows older Pythons to have access to features
+found in newer Pythons, such as parsing the bytecode, a uniform stream
+of bytes, into a list of structured bytecode instructions.
+
+Python 2.7's ``dis`` module doesn't has provide a instruction abstraction.
+Therefore in ``uncompyle2`` and other earlier decompilers you see code with magic numbers like 4 in::
+
+    if end > jump_back+4 and code[end] in (JF, JA):
+        if code[jump_back+4] in (JA, JF):
+            if self.get_target(jump_back+4) == self.get_target(end):
+                self.fixed_jumps[pos] = jump_back+4
+                end = jump_back+4
+    elif target < pos:
+        self.fixed_jumps[pos] = jump_back+4
+        end = jump_back+4
+
+and in other code -1 and 3 in::
+
+        if self.get_target(jmp) != start_else:
+            end_else = self.get_target(jmp)
+        if self.code[jmp] == JF:
+            self.fixed_jumps[jmp] = -1
+        self.structs.append({'type':  'except',
+                       'start': i,
+                       'end':   jmp})
+        i = jmp + 3
+
+All of that offset arithmetic is trying to find the next instruction
+offset or the previous offset. Using a list of instructions you simply
+take the ``offset`` field of the previous or next instruction.
+
+The above code appears in the ``uncompyle2`` "Scanner" class in
+service of trying to figure out control flow. Note also that there
+isn't a single comment in there about what specifically it is trying
+to do, the logic or that would lead one to be confident that this is
+correct, let alone assumptions that are needed for this to be true.
+
+While this might largely work for Python 2.7, and ``uncompyle2`` does
+get control flow wrong sometimes, it is impossible to adapt code for
+other versions of Python.
+
+In addition adding an instruction structure, ``xdis`` adds various
+flags and features that assist in working with instructions. In the
+example above this replaces code like ``... in (JF, JA)`` which is
+some sort of unconditional jump instruction.
+
+Although not needed in the decompiler, ``xdis`` also has nicer
+instruction print format. It can show you the bytes as well as the
+interpreted instructions. It will interpret flag bits and packed
+structures in operands so you don't have to. It can even do a limited
+form of inspection at previous instructions to give a more complete
+description of an operand. For example on ``LOAD_ATTR`` which loads
+the attribute of a variable, often the variable name can be found as
+the previous instruction. When that is the case the disassembler can
+include that in the disassembly display for the ``LOAD_ATTR`` operand.
+
+
+Python Grammar Isolation
+------------------------
+
+If you want to support multiple versions of Python in a manageable way
+you really need to provide different grammars for the different
+versions, in a grammar-based system. None of the published versions of
+this decompiler did this.
+
+If you look at the changes in this code, right now there are no
+grammar changes needed between 1.0 to 1.3. (Some of this may be wrong
+though since we haven't extensively tested these earliest Python versions
+
+For Python 1.4 which is based off of the grammar for 1.5 though there
+are number of changes, about 6 grammar rules. Later versions of though
+we start to see larger upheaval and at certain places, especially
+those where new opcodes are introduced, especially those that change
+the way calls or exceptions get handled, we have major upheaval in the
+grammar. It is not just that some rules get added, but we also need to
+*remove* some grammar rules as well.
+
+I have been largely managing this as incremental differences between versions.
+However in the future I am leaning more towards totally separate grammars.
+A well constructed grammar doesn't need to be that large.
+
+When starting out a new version, we can just copy the grammar from the
+prior version.  Within a Python version though, I am breaking these
+into composable pieces. In particular the grammar for handling what
+can appear as the body of a lambda, is a subset of the full Python
+language. The language allowed in an ``eval`` is also a subset of the
+full Python language, as are what can appear in the various
+compilation modes like "single" versus "exec".
+
+Another nice natural self-contain grammar section is what can appear
+in list comprehensions and generators. The bodies of these are
+generally represented in a self-contained code block.
+
+Often in decompilation you may be interested not just in decompiling
+the entire code but you may be interested in only focusing on a
+specific part of the code. And if there is a problem in decompiling
+the entire piece of code, having these smaller breaking points can be
+of assistance.
+
+Other Modularity
+----------------
+
+Above we have mentioned the need for separate grammars or to isolate
+these per versions. But there are other major pieces that make up this
+decompiler. In particular there is a scanner and the source code
+generation part.
+
+Even though differences in version that occur in disassembly are
+handled by ``xdis``, we still have to do conversion of that to a token
+stream for parsing. So the scanners are again broken out per version
+with various OO mechanisms for reusing code. The same is true for
+source code generation.
+
+
+Expanding decompiler availability to multiple Python Versions
+--------------------------------------------------------------
+
+Above we mention decompiling multiple versions of bytecode from a
+single Python interpreter. We we talk about having the decompiler
+runnable from multiple versions of Python, independent of the set of
+bytecode that the decompiler supports.
+
+
+There are slight advantages in having a decompiler that runs the same
+version as the code you are decompiling. The most obvious one is that
+it makes it easy to test to see whether the decompilation correct
+because you can run the decompiled code. Python comes with a suite of
+Python programs that check themselves and that aspects of Python are
+implemented correctly. These also make excellent programs to check
+whether a program has decompiled correctly.
+
+Aside from this, debugging can be easier as well. To assist
+understanding bytcode and single stepping it see `x-python
+<https://pypi.org/project/x-python/>`_ and the debugger for it
+`trepan-xpy <https://pypi.org/project/trepanxpy/>`_.
+
+Handling Language Drift
+-----------------------
+
+Given the desirability of having this code running on logs of Python
+versions, how can we get this done?
+
+The solution used here is to have several git branches of the
+code. Right now there are 3 branches. Each branch handles works across
+3 or so different releases of Python. In particular one branch handles
+Python 2.4 to 2.7 Another handles Python 3.3 to 3.5, and the master
+branch handles 3.6 to 3.10. (Again note that the 3.9 and 3.10
+decompilers do not decompile Python 3.9 or 3.10, but they do handle
+bytecode for all earlier versions.)
+
+
+Cool features of the Parser
+===========================
+
+* reduction rule checking
+* numbering tokens
+* showing a stack of completions
+
+Cool features Semantic Analysis
+===============================
+
+* ``--tree++`` (``-T``) option
+* showing precedence
+* See `Adding a tree transformation phase to uncompyle6 <https://github.com/rocky/python-uncompyle6/wiki/Adding-a-tree-transformation-phase-to-uncompyle6>`_
+* following AST
+* Fragment deparsing

README.rst

@@ -1,7 +1,9 @@
-|buildstatus| |Latest Version| |Supported Python Versions|
+|buildstatus|  |Pypi Installs| |Latest Version| |Supported Python Versions|
 
 |packagestatus|
 
+.. contents::
+
 uncompyle6
 ==========
 
@@ -13,9 +15,9 @@ Introduction
 ------------
 
 *uncompyle6* translates Python bytecode back into equivalent Python
-source code. It accepts bytecodes from Python version 1.3 to version
+source code. It accepts bytecodes from Python version 1.0 to version
 3.8, spanning over 24 years of Python releases. We include Dropbox's
-Python 2.5 bytecode and some PyPy bytecode.
+Python 2.5 bytecode and some PyPy bytecodes.
 
 Why this?
 ---------
@@ -46,14 +48,15 @@ not exist and there is just bytecode. Again, my debuggers make use of
 this.
 
 There were (and still are) a number of decompyle, uncompyle,
-uncompyle2, uncompyle3 forks around. Almost all of them come basically
-from the same code base, and (almost?) all of them are no longer
-actively maintained. One was really good at decompiling Python 1.5-2.3
-or so, another really good at Python 2.7, but that only. Another
-handles Python 3.2 only; another patched that and handled only 3.3.
-You get the idea. This code pulls all of these forks together and
-*moves forward*. There is some serious refactoring and cleanup in this
-code base over those old forks.
+uncompyle2, uncompyle3 forks around. Many of them come basically from
+the same code base, and (almost?) all of them are no longer actively
+maintained. One was really good at decompiling Python 1.5-2.3, another
+really good at Python 2.7, but that only. Another handles Python 3.2
+only; another patched that and handled only 3.3.  You get the
+idea. This code pulls all of these forks together and *moves
+forward*. There is some serious refactoring and cleanup in this code
+base over those old forks. Even more experimental refactoring is going
+on in decompyle3_.
 
 This demonstrably does the best in decompiling Python across all
 Python versions. And even when there is another project that only
@@ -67,32 +70,51 @@ are syntactically correct by running the Python interpreter for that
 bytecode version.  Finally, in cases where the program has a test for
 itself, we can run the check on the decompiled code.
 
-We are serious about testing, and use automated processes to find
-bugs. In the issue trackers for other decompilers, you will find a
-number of bugs we've found along the way. Very few to none of them are
-fixed in the other decompilers.
+We use an automated processes to find bugs. In the issue trackers for
+other decompilers, you will find a number of bugs we've found along
+the way. Very few to none of them are fixed in the other decompilers.
 
 Requirements
 ------------
 
-The code here can be run on Python versions 2.6 or later, PyPy 3-2.4,
-or PyPy-5.0.1.  Python versions 2.4-2.7 are supported in the
-python-2.4 branch.  The bytecode files it can read have been tested on
-Python bytecodes from versions 1.4, 2.1-2.7, and 3.0-3.8 and the
-above-mentioned PyPy versions.
+The code in the git repository can be run from Python 2.4 to the
+latest Python version, with the exception of Python 3.0 through
+3.2. Volunteers are welcome to address these deficiencies if there a
+desire to do so.
+
+The way it does this though is by segregating consecutive Python versions into
+git branches:
+
+master
+   Python 3.6 and up (uses type annotations)
+python-3.3-to-3.5
+   Python 3.3 through 3.5 (Generic Python 3)
+python-2.4
+   Python 2.4 through 2.7 (Generic Python 2)
+
+PyPy 3-2.4 and later works as well.
+
+The bytecode files it can read have been tested on Python
+bytecodes from versions 1.4, 2.1-2.7, and 3.0-3.8 and later PyPy
+versions.
 
 Installation
 ------------
 
-This uses setup.py, so it follows the standard Python routine:
+You can install from PyPI using the name ``uncompyle6``::
 
-::
+    pip install uncompyle6
 
-    pip install -e .  # set up to run from source tree
-                      # Or if you want to install instead
-    python setup.py install # may need sudo
 
-A GNU makefile is also provided so :code:`make install` (possibly as root or
+To install from source code, this project uses setup.py, so it follows the standard Python routine::
+
+    $ pip install -e .  # set up to run from source tree
+
+or::
+
+    $ python setup.py install # may need sudo
+
+A GNU Makefile is also provided so :code:`make install` (possibly as root or
 sudo) will do the steps above.
 
 Running Tests
@@ -129,7 +151,7 @@ Verification
 
 In older versions of Python it was possible to verify bytecode by
 decompiling bytecode, and then compiling using the Python interpreter
-for that bytecode version. Having done this the bytecode produced
+for that bytecode version. Having done this, the bytecode produced
 could be compared with the original bytecode. However as Python's code
 generation got better, this no longer was feasible.
 
@@ -139,9 +161,16 @@ Python syntax changes, you should use this option if the bytecode is
 the right bytecode for the Python interpreter that will be checking
 the syntax.
 
-You can also cross compare the results with another python decompiler
-like pycdc_ . Since they work differently, bugs here often aren't in
-that, and vice versa.
+You can also cross compare the results with either another version of
+`uncompyle6` since there are are sometimes regressions in decompiling
+specific bytecode as the overall quality improves.
+
+For Python 3.7 and 3.8, the code in decompyle3_ is generally
+better.
+
+Or try specific another python decompiler like uncompyle2_, unpyc37_,
+or pycdc_.  Since the later two work differently, bugs here often
+aren't in that, and vice versa.
 
 There is an interesting class of these programs that is readily
 available give stronger verification: those programs that when run
@@ -164,15 +193,11 @@ All of the Python decompilers that I have looked at have problems
 decompiling Python's control flow. In some cases we can detect an
 erroneous decompilation and report that.
 
-Python support is strongest in Python 2 for 2.7 and drops off as you
-get further away from that. Support is also probably pretty good for
-python 2.3-2.4 since a lot of the goodness of early the version of the
-decompiler from that era has been preserved (and Python compilation in
-that era was minimal)
+Python support is pretty good for Python 2
 
-There is some work to do on the lower end Python versions which is
-more difficult for us to handle since we don't have a Python
-interpreter for versions 1.6, and 2.0.
+On the lower end of Python versions, decompilation seems pretty good although
+we don't have any automated testing in place for Python's distributed tests.
+Also, we don't have a Python interpreter for versions 1.6, and 2.0.
 
 In the Python 3 series, Python support is is strongest around 3.4 or
 3.3 and drops off as you move further away from those versions. Python
@@ -186,61 +211,107 @@ they had been rare.  Perhaps to compensate for the additional
 added. So in sum handling control flow by ad hoc means as is currently
 done is worse.
 
-Between Python 3.5, 3.6 and 3.7 there have been major changes to the
+Between Python 3.5, 3.6, 3.7 there have been major changes to the
 :code:`MAKE_FUNCTION` and :code:`CALL_FUNCTION` instructions.
 
+Python 3.8 removes :code:`SETUP_LOOP`, :code:`SETUP_EXCEPT`,
+:code:`BREAK_LOOP`, and :code:`CONTINUE_LOOP`, instructions which may
+make control-flow detection harder, lacking the more sophisticated
+control-flow analysis that is planned. We'll see.
+
 Currently not all Python magic numbers are supported. Specifically in
 some versions of Python, notably Python 3.6, the magic number has
 changes several times within a version.
 
-**We support only released versions, not candidate versions.** Note however
-that the magic of a released version is usually the same as the *last* candidate version prior to release.
+**We support only released versions, not candidate versions.** Note
+however that the magic of a released version is usually the same as
+the *last* candidate version prior to release.
 
 There are also customized Python interpreters, notably Dropbox,
-which use their own magic and encrypt bytcode. With the exception of
+which use their own magic and encrypt bytecode. With the exception of
 the Dropbox's old Python 2.5 interpreter this kind of thing is not
 handled.
 
-We also don't handle PJOrion_ obfuscated code. For that try: PJOrion
-Deobfuscator_ to unscramble the bytecode to get valid bytecode before
-trying this tool. This program can't decompile Microsoft Windows EXE
-files created by Py2EXE_, although we can probably decompile the code
-after you extract the bytecode properly. For situations like this, you
-might want to consider a decompilation service like `Crazy Compilers
-<http://www.crazy-compilers.com/decompyle/>`_.  Handling
-pathologically long lists of expressions or statements is slow.
+We also don't handle PJOrion_ or otherwise obfuscated code. For
+PJOrion try: PJOrion Deobfuscator_ to unscramble the bytecode to get
+valid bytecode before trying this tool; pydecipher_ might help with that.
 
+This program can't decompile Microsoft Windows EXE files created by
+Py2EXE_, although we can probably decompile the code after you extract
+the bytecode properly. `Pydeinstaller <https://github.com/charles-dyfis-net/pydeinstaller>`_ may help with unpacking Pyinstaller bundlers.
 
-There is lots to do, so please dig in and help.
+Handling pathologically long lists of expressions or statements is
+slow. We don't handle Cython_ or MicroPython which don't use bytecode.
+
+There are numerous bugs in decompilation. And that's true for every
+other CPython decompiler I have encountered, even the ones that
+claimed to be "perfect" on some particular version like 2.4.
+
+As Python progresses decompilation also gets harder because the
+compilation is more sophisticated and the language itself is more
+sophisticated. I suspect that attempts there will be fewer ad-hoc
+attempts like unpyc37_ (which is based on a 3.3 decompiler) simply
+because it is harder to do so. The good news, at least from my
+standpoint, is that I think I understand what's needed to address the
+problems in a more robust way. But right now until such time as
+project is better funded, I do not intend to make any serious effort
+to support Python versions 3.8 or 3.9, including bugs that might come
+in. I imagine at some point I may be interested in it.
+
+You can easily find bugs by running the tests against the standard
+test suite that Python uses to check itself. At any given time, there are
+dozens of known problems that are pretty well isolated and that could
+be solved if one were to put in the time to do so. The problem is that
+there aren't that many people who have been working on bug fixing.
+
+Some of the bugs in 3.7 and 3.8 are simply a matter of back-porting
+the fixes in decompyle3. Volunteers are welcome to do so.
+
+You may run across a bug, that you want to report. Please do so after
+reading `How to report a bug
+<https://github.com/rocky/python-uncompyle6/blob/master/HOW-TO-REPORT-A-BUG.md>`_ and
+follow the `instructions when opening an issue <https://github.com/rocky/python-uncompyle6/issues/new?assignees=&labels=&template=bug-report.md>`_.
+
+Be aware that it might not get my attention for a while. If you
+sponsor or support the project in some way, I'll prioritize your
+issues above the queue of other things I might be doing instead.
 
 See Also
 --------
 
-* https://github.com/zrax/pycdc : purports to support all versions of Python. It is written in C++ and is most accurate for Python versions around 2.7 and 3.3 when the code was more actively developed. Accuracy for more recent versions of Python 3 and early versions of Python are especially lacking. See its `issue tracker <https://github.com/zrax/pycdc/issues>`_ for details. Currently lightly maintained.
+* https://github.com/rocky/python-decompile3 : Much smaller and more modern code, focusing on 3.7 and 3.8. Changes in that will get migrated back here.
 * https://code.google.com/archive/p/unpyc3/ : supports Python 3.2 only. The above projects use a different decompiling technique than what is used here. Currently unmaintained.
 * https://github.com/figment/unpyc3/ : fork of above, but supports Python 3.3 only. Includes some fixes like supporting function annotations. Currently unmaintained.
-* https://github.com/wibiti/uncompyle2 : supports Python 2.7 only, but does that fairly well. There are situations where :code:`uncompyle6` results are incorrect while :code:`uncompyle2` results are not, but more often uncompyle6 is correct when uncompyle2 is not. Because :code:`uncompyle6` adheres to accuracy over idiomatic Python, :code:`uncompyle2` can produce more natural-looking code when it is correct. Currently :code:`uncompyle2` is lightly maintained. See its issue `tracker <https://github.com/wibiti/uncompyle2/issues>`_ for more details
+* https://github.com/wibiti/uncompyle2 : supports Python 2.7 only, but does that fairly well. There are situations where :code:`uncompyle6` results are incorrect while :code:`uncompyle2` results are not, but more often uncompyle6 is correct when uncompyle2 is not. Because :code:`uncompyle6` adheres to accuracy over idiomatic Python, :code:`uncompyle2` can produce more natural-looking code when it is correct. Currently :code:`uncompyle2` is lightly maintained. See its issue `tracker <https://github.com/wibiti/uncompyle2/issues>`_ for more details.
 * `How to report a bug <https://github.com/rocky/python-uncompyle6/blob/master/HOW-TO-REPORT-A-BUG.md>`_
 * The HISTORY_ file.
 * https://github.com/rocky/python-xdis : Cross Python version disassembler
 * https://github.com/rocky/python-xasm : Cross Python version assembler
 * https://github.com/rocky/python-uncompyle6/wiki : Wiki Documents which describe the code and aspects of it in more detail
+* https://github.com/zrax/pycdc : The README for this C++ code says it aims to support all versions of Python. You can aim your slign shot for the moon too, but I doubt you are going to hit it. This code is best for Python versions around 2.7 and 3.3 when the code was initially developed. Accuracy for current versions of Python3 and early versions of Python is lacking. Without major effort, it is unlikely it can be made to support current Python 3. See its `issue tracker <https://github.com/zrax/pycdc/issues>`_ for details. Currently lightly maintained.
 
 
-.. _trepan: https://pypi.python.org/pypi/trepan2g
-.. _compiler: https://pypi.python.org/pypi/spark_parser
+.. _Cython: https://en.wikipedia.org/wiki/Cython
+.. _trepan: https://pypi.python.org/pypi/trepan3k
+.. _compiler: https://github.com/rocky/python-uncompyle6/wiki/How-does-this-code-work%3F
 .. _HISTORY: https://github.com/rocky/python-uncompyle6/blob/master/HISTORY.md
+.. _report_bug: https://github.com/rocky/python-uncompyle6/blob/master/HOW-TO-REPORT-A-BUG.md
 .. _debuggers: https://pypi.python.org/pypi/trepan3k
 .. _remake: https://bashdb.sf.net/remake
 .. _pycdc: https://github.com/zrax/pycdc
+.. _decompyle3: https://github.com/rocky/python-decompile3
+.. _uncompyle2: https://github.com/wibiti/uncompyle2
+.. _unpyc37: https://github.com/andrew-tavera/unpyc37
 .. _this: https://github.com/rocky/python-uncompyle6/wiki/Deparsing-technology-and-its-use-in-exact-location-reporting
 .. |buildstatus| image:: https://travis-ci.org/rocky/python-uncompyle6.svg
 		 :target: https://travis-ci.org/rocky/python-uncompyle6
 .. |packagestatus| image:: https://repology.org/badge/vertical-allrepos/python:uncompyle6.svg
 		 :target: https://repology.org/project/python:uncompyle6/versions
 .. _PJOrion: http://www.koreanrandom.com/forum/topic/15280-pjorion-%D1%80%D0%B5%D0%B4%D0%B0%D0%BA%D1%82%D0%B8%D1%80%D0%BE%D0%B2%D0%B0%D0%BD%D0%B8%D0%B5-%D0%BA%D0%BE%D0%BC%D0%BF%D0%B8%D0%BB%D1%8F%D1%86%D0%B8%D1%8F-%D0%B4%D0%B5%D0%BA%D0%BE%D0%BC%D0%BF%D0%B8%D0%BB%D1%8F%D1%86%D0%B8%D1%8F-%D0%BE%D0%B1%D1%84
+.. _pydecipher: https://github.com/mitre/pydecipher
 .. _Deobfuscator: https://github.com/extremecoders-re/PjOrion-Deobfuscator
 .. _Py2EXE: https://en.wikipedia.org/wiki/Py2exe
 .. |Supported Python Versions| image:: https://img.shields.io/pypi/pyversions/uncompyle6.svg
 .. |Latest Version| image:: https://badge.fury.io/py/uncompyle6.svg
 		 :target: https://badge.fury.io/py/uncompyle6
+.. |Pypi Installs| image:: https://pepy.tech/badge/uncompyle6/month

__pkginfo__.py

@@ -1,4 +1,4 @@
-# Copyright (C) 2018 Rocky Bernstein <rocky@gnu.org>
+# Copyright (C) 2018, 2020-2021 Rocky Bernstein <rocky@gnu.org>
 #
 #  This program is free software: you can redistribute it and/or modify
 #  it under the terms of the GNU General Public License as published by
@@ -21,66 +21,88 @@
 # less elegant than having it here with reduced code, albeit there
 # still is some room for improvement.
 
+# Python-version | package | last-version |
+# -----------------------------------------
+# 2.5            | pip     |  1.1         |
+# 2.6            | pip     |  1.5.6       |
+# 2.7            | pip     | 19.2.3       |
+# 2.7            | pip     |  1.2.1       |
+# 3.1            | pip     |  1.5.6       |
+# 3.2            | pip     |  7.1.2       |
+# 3.3            | pip     | 10.0.1       |
+# 3.4            | pip     | 19.1.1       |
+
 # Things that change more often go here.
-copyright   = """
-Copyright (C) 2015-2019 Rocky Bernstein <rb@dustyfeet.com>.
+copyright = """
+Copyright (C) 2015-2021 Rocky Bernstein <rb@dustyfeet.com>.
 """
 
-classifiers =  ["Development Status :: 5 - Production/Stable",
-                "Intended Audience :: Developers",
-                "License :: OSI Approved :: GNU General Public License v3 (GPLv3)",
-                "Operating System :: OS Independent",
-                "Programming Language :: Python",
-                "Programming Language :: Python :: 2.4",
-                "Programming Language :: Python :: 2.5",
-                "Programming Language :: Python :: 2.6",
-                "Programming Language :: Python :: 2.7",
-                "Programming Language :: Python :: 3.0",
-                "Programming Language :: Python :: 3.1",
-                "Programming Language :: Python :: 3.2",
-                "Programming Language :: Python :: 3.3",
-                "Programming Language :: Python :: 3.4",
-                "Programming Language :: Python :: 3.5",
-                "Programming Language :: Python :: 3.6",
-                "Programming Language :: Python :: 3.7",
-                "Programming Language :: Python :: 3.8",
-                "Topic :: Software Development :: Debuggers",
-                "Topic :: Software Development :: Libraries :: Python Modules",
-                ]
+classifiers = [
+    "Development Status :: 5 - Production/Stable",
+    "Intended Audience :: Developers",
+    "License :: OSI Approved :: GNU General Public License v3 (GPLv3)",
+    "Operating System :: OS Independent",
+    "Programming Language :: Python",
+    "Programming Language :: Python :: 2",
+    "Programming Language :: Python :: 2.4",
+    "Programming Language :: Python :: 2.5",
+    "Programming Language :: Python :: 2.6",
+    "Programming Language :: Python :: 2.7",
+    "Programming Language :: Python :: 3",
+    "Programming Language :: Python :: 3.0",
+    "Programming Language :: Python :: 3.1",
+    "Programming Language :: Python :: 3.2",
+    "Programming Language :: Python :: 3.3",
+    "Programming Language :: Python :: 3.4",
+    "Programming Language :: Python :: 3.5",
+    "Programming Language :: Python :: 3.6",
+    "Programming Language :: Python :: 3.7",
+    "Programming Language :: Python :: 3.8",
+    "Programming Language :: Python :: 3.9",
+    "Programming Language :: Python :: 3.10",
+    "Programming Language :: Python :: Implementation :: PyPy",
+    "Topic :: Software Development :: Debuggers",
+    "Topic :: Software Development :: Libraries :: Python Modules",
+]
 
 # The rest in alphabetic order
-author             = "Rocky Bernstein, Hartmut Goebel, John Aycock, and others"
-author_email       = "rb@dustyfeet.com"
-entry_points       = {
+author = "Rocky Bernstein, Hartmut Goebel, John Aycock, and others"
+author_email = "rb@dustyfeet.com"
+entry_points = {
     "console_scripts": [
         "uncompyle6=uncompyle6.bin.uncompile:main_bin",
         "pydisassemble=uncompyle6.bin.pydisassemble:main",
-    ]}
-ftp_url            = None
-install_requires   = ["spark-parser >= 1.8.9, < 1.9.0",
-                      "xdis >= 4.0.3, < 4.1.0"]
+    ]
+}
+ftp_url = None
+install_requires = ["spark-parser >= 1.8.9, < 1.9.0", "xdis >= 6.0.2, < 6.2.0"]
 
-license            = "GPL3"
-mailing_list       = "python-debugger@googlegroups.com"
-modname            = "uncompyle6"
-py_modules         = None
-short_desc         = "Python cross-version byte-code decompiler"
-web                = "https://github.com/rocky/python-uncompyle6/"
+license = "GPL3"
+mailing_list = "python-debugger@googlegroups.com"
+modname = "uncompyle6"
+py_modules = None
+short_desc = "Python cross-version byte-code decompiler"
+web = "https://github.com/rocky/python-uncompyle6/"
 
 # tracebacks in zip files are funky and not debuggable
 zip_safe = True
 
 
 import os.path
+
+
 def get_srcdir():
     filename = os.path.normcase(os.path.dirname(os.path.abspath(__file__)))
     return os.path.realpath(filename)
 
+
 srcdir = get_srcdir()
 
+
 def read(*rnames):
     return open(os.path.join(srcdir, *rnames)).read()
 
-# Get info from files; set: long_description and VERSION
-long_description   = ( read("README.rst") + "\n" )
+
+# Get info from files; set: long_description and __version__
+long_description = read("README.rst") + "\n"
 exec(read("uncompyle6/version.py"))

admin-tools/README.md

@@ -8,4 +8,15 @@ They are customized to my environment:
 - I have git repos for xdis, and spark parser at the same level as uncompyle6
 
 There may be other rocky-specific things that need customization.
-how-to-make-a-release.txt has overall how I make a release
+how-to-make-a-release.md has overall how I make a release
+
+Since this project uses python over a wide variety of release, some versions
+of projects that should be used for specific Python versions
+
+for 3.2.6:
+   pytest==2.9.2
+
+for 3.1.5
+   pytset==2.1.0
+   py=1.8.0 and comment out line 10 of _builtin.py # callable = callable
+   six==1.10.0

admin-tools/check-2.4-2.7-versions.sh

@@ -6,7 +6,7 @@ owd=$(pwd)
 trap finish EXIT
 
 cd $(dirname ${BASH_SOURCE[0]})
-if ! source ./pyenv-older-versions ; then
+if ! source ./pyenv-2.4-2.7-versions ; then
     exit $?
 fi
 if ! source ./setup-python-2.4.sh ; then

admin-tools/check-3.3-3.5-versions.sh

@@ -0,0 +1,27 @@
+#!/bin/bash
+function finish {
+  cd $owd
+}
+owd=$(pwd)
+trap finish EXIT
+
+cd $(dirname ${BASH_SOURCE[0]})
+if ! source ./pyenv-3.3-3.5-versions ; then
+    exit $?
+fi
+if ! source ./setup-python-3.3.sh ; then
+    exit $?
+fi
+
+cd ..
+for version in $PYVERSIONS; do
+    echo --- $version ---
+    if ! pyenv local $version ; then
+	exit $?
+    fi
+    make clean && python setup.py develop
+    if ! make check ; then
+	exit $?
+    fi
+    echo === $version ===
+done

admin-tools/check-newest-versions.sh

@@ -8,7 +8,7 @@ owd=$(pwd)
 trap finish EXIT
 
 cd $(dirname ${BASH_SOURCE[0]})
-if ! source ./pyenv-newer-versions ; then
+if ! source ./pyenv-newest-versions ; then
     exit $?
 fi
 if ! source ./setup-master.sh ; then
@@ -21,8 +21,9 @@ for version in $PYVERSIONS; do
 	exit $?
     fi
     make clean && pip install -e .
-    if ! make check; then
+    if ! make check-short; then
 	exit $?
     fi
     echo === $version ===
 done
+make check

admin-tools/how-to-make-a-release.md

@@ -2,17 +2,17 @@
 **Table of Contents**
 
 - [Get latest sources:](#get-latest-sources)
-- [Change version in uncompyle6/version.py](#change-version-in-uncompyle6versionpy)
+- [Change version in uncompyle6/version.py:](#change-version-in-uncompyle6versionpy)
 - [Update ChangeLog:](#update-changelog)
-- [Update NEWS from ChangeLog:](#update-news-from-changelog)
+- [Update NEWS.md from ChangeLog:](#update-newsmd-from-changelog)
 - [Make sure pyenv is running and check newer versions](#make-sure-pyenv-is-running-and-check-newer-versions)
 - [Switch to python-2.4, sync that up and build that first since it creates a tarball which we don't want.](#switch-to-python-24-sync-that-up-and-build-that-first-since-it-creates-a-tarball-which-we-dont-want)
-- [Update NEWS from master branch](#update-news-from-master-branch)
-- [Check against all versions](#check-against-all-versions)
+- [Check against older versions](#check-against-older-versions)
 - [Make packages and tag](#make-packages-and-tag)
-- [Upload single package and look at Rst Formating](#upload-single-package-and-look-at-rst-formating)
-- [Upload rest of versions](#upload-rest-of-versions)
-- [Push tags:](#push-tags)
+- [Check package on github](#check-package-on-github)
+- [Release on Github](#release-on-github)
+- [Get onto PyPI](#get-onto-pypi)
+- [Update tags:](#update-tags)
 
 <!-- markdown-toc end -->
 # Get latest sources:
@@ -39,7 +39,7 @@
 
 # Make sure pyenv is running and check newer versions
 
-    $ pyenv local && source admin-tools/check-newer-versions.sh
+    $ admin-tools/check-newer-versions.sh
 
 # Switch to python-2.4, sync that up and build that first since it creates a tarball which we don't want.
 
@@ -50,35 +50,51 @@
 
 # Check against older versions
 
-    $ source admin-tools/check-older-versions.sh
+    $ admin-tools/check-older-versions.sh
 
 # Make packages and tag
 
     $ . ./admin-tools/make-dist-older.sh
+	$ pyenv local 3.8.5
+	$ twine check dist/uncompyle6-$VERSION*
     $ git tag release-python-2.4-$VERSION
+    $ ./admin-tools/make-dist-newer.sh
+	$ twine check dist/uncompyle6-$VERSION*
 
-    $ . ./admin-tools/make-dist-newer.sh
+# Check package on github
+
+	$ [[ ! -d /tmp/gittest ]] && mkdir /tmp/gittest; pushd /tmp/gittest
+	$ pyenv local 3.8.3
+	$ pip install -e git://github.com/rocky/python-uncompyle6.git#egg=uncompyle6
+	$ uncompyle6 --help
+	$ pip uninstall uncompyle6
+	$ popd
+
+# Release on Github
 
 Goto https://github.com/rocky/python-uncompyle6/releases
 
-# Upload single package and look at Rst Formating
+Now check the *tagged* release. (Checking the untagged release was previously done).
 
-    $ twine upload dist/uncompyle6-${VERSION}-py3.3.egg
+Todo: turn this into a script in `admin-tools`
 
-# Upload rest of versions
+	$ pushd /tmp/gittest
+	$ pip install -e git://github.com/rocky/python-uncompyle6.git@$VERSION#egg=uncompyle6
+	$ uncompyle6 --help
+	$ pip uninstall uncompyle6
+	$ popd
+
+
+# Get onto PyPI
 
     $ twine upload dist/uncompyle6-${VERSION}*
 
-# Push tags:
+
+# Update tags:
 
     $ git push --tags
+    $ git pull --tags
 
-# Check on a VM
+# Move dist files to uploaded
 
-    $ cd /virtual/vagrant/virtual/vagrant/ubuntu-zesty
-	$ vagrant up
-	$ vagrant ssh
-	$ pyenv local 3.5.2
-	$ pip install --upgrade uncompyle6
-	$ exit
-	$ vagrant halt
+	$ mv -v dist/uncompyle6-${VERSION}* dist/uploaded

admin-tools/make-dist-2.4-2.7.sh

@@ -9,7 +9,7 @@ owd=$(pwd)
 trap finish EXIT
 
 cd $(dirname ${BASH_SOURCE[0]})
-if ! source ./pyenv-older-versions ; then
+if ! source ./pyenv-2.4-2.7-versions ; then
     exit $?
 fi
 if ! source ./setup-python-2.4.sh ; then
@@ -18,7 +18,7 @@ fi
 
 cd ..
 source $PACKAGE/version.py
-echo $VERSION
+echo $__version__
 
 for pyversion in $PYVERSIONS; do
     if ! pyenv local $pyversion ; then
@@ -29,11 +29,15 @@ for pyversion in $PYVERSIONS; do
     python setup.py bdist_egg
 done
 
+pyenv local 2.7.18
+python setup.py bdist_wheel
+mv -v dist/${PACKAGE}-$__version__-py2{.py3,}-none-any.whl
+
 # Pypi can only have one source tarball.
 # Tarballs can get created from the above setup, so make sure to remove them since we want
 # the tarball from master.
 
-tarball=dist/${PACKAGE}-$VERSION-tar.gz
+tarball=dist/${PACKAGE}-${__version_}_-tar.gz
 if [[ -f $tarball ]]; then
-    rm -v dist/${PACKAGE}-$VERSION-tar.gz
+    rm -v dist/${PACKAGE}-${__version__}-tar.gz
 fi

admin-tools/make-dist-3.3-3.5.sh

@@ -0,0 +1,38 @@
+#!/bin/bash
+PACKAGE=uncompyle6
+
+# FIXME put some of the below in a common routine
+function finish {
+  cd $owd
+}
+
+cd $(dirname ${BASH_SOURCE[0]})
+owd=$(pwd)
+trap finish EXIT
+
+if ! source ./pyenv-3.3-3.5-versions ; then
+    exit $?
+fi
+if ! source ./setup-python-3.3.sh ; then
+    exit $?
+fi
+
+cd ..
+source $PACKAGE/version.py
+echo $__version__
+
+for pyversion in $PYVERSIONS; do
+    if ! pyenv local $pyversion ; then
+	exit $?
+    fi
+    # pip bdist_egg create too-general wheels. So
+    # we narrow that by moving the generated wheel.
+
+    # Pick out first two number of version, e.g. 3.5.1 -> 35
+    first_two=$(echo $pyversion | cut -d'.' -f 1-2 | sed -e 's/\.//')
+    rm -fr build
+    python setup.py bdist_egg bdist_wheel
+    mv -v dist/${PACKAGE}-$__version__-{py2.py3,py$first_two}-none-any.whl
+done
+
+python ./setup.py sdist

admin-tools/make-dist-newest.sh

@@ -10,7 +10,7 @@ cd $(dirname ${BASH_SOURCE[0]})
 owd=$(pwd)
 trap finish EXIT
 
-if ! source ./pyenv-newer-versions ; then
+if ! source ./pyenv-newest-versions ; then
     exit $?
 fi
 if ! source ./setup-master.sh ; then
@@ -19,7 +19,7 @@ fi
 
 cd ..
 source $PACKAGE/version.py
-echo $VERSION
+echo $__version__
 
 for pyversion in $PYVERSIONS; do
     if ! pyenv local $pyversion ; then
@@ -32,7 +32,7 @@ for pyversion in $PYVERSIONS; do
     first_two=$(echo $pyversion | cut -d'.' -f 1-2 | sed -e 's/\.//')
     rm -fr build
     python setup.py bdist_egg bdist_wheel
-    mv -v dist/${PACKAGE}-$VERSION-{py2.py3,py$first_two}-none-any.whl
+    mv -v dist/${PACKAGE}-$__version__-{py2.py3,py$first_two}-none-any.whl
 done
 
 python ./setup.py sdist

admin-tools/pyenv-2.4-2.7-versions

@@ -0,0 +1,9 @@
+# -*- shell-script -*-
+# Sets PYVERSIONS to be pyenv versions that
+# we can use in the python-2.4-to-2.7 branch.
+
+if [[ $0 == ${BASH_SOURCE[0]} ]] ; then
+    echo "This script should be *sourced* rather than run directly through bash"
+    exit 1
+fi
+export PYVERSIONS='2.4.6 2.5.6 2.6.9 2.7.18'

admin-tools/pyenv-3.0-3.2-versions

@@ -6,4 +6,4 @@ if [[ $0 == ${BASH_SOURCE[0]} ]] ; then
     echo "This script should be *sourced* rather than run directly through bash"
     exit 1
 fi
-export PYVERSIONS='2.4.6 2.5.6 2.6.9'
+export PYVERSIONS='3.0.1 3.1.5 3.2.6'

admin-tools/pyenv-3.3-3.5-versions

@@ -0,0 +1,8 @@
+# -*- shell-script -*-
+# Sets PYVERSIONS to be pyenv versions that
+# we can use in the python-3.3-to-3.5 branch.
+if [[ $0 == ${BASH_SOURCE[0]} ]] ; then
+    echo "This script should be *sourced* rather than run directly through bash"
+    exit 1
+fi
+export PYVERSIONS='3.5.10 3.3.7 3.4.10'

admin-tools/pyenv-newest-versions

@@ -0,0 +1,8 @@
+# -*- shell-script -*-
+# Sets PYVERSIONS to be pyenv versions that
+# we can use in the master branch.
+if [[ $0 == ${BASH_SOURCE[0]} ]] ; then
+    echo "This script should be *sourced* rather than run directly through bash"
+    exit 1
+fi
+export PYVERSIONS='3.6.15 pypy3.6-7.3.1 3.7.16 pypy3.7-7.3.9 pypy3.8-7.3.10 pyston-2.3.5 3.8.16'

admin-tools/pyenv-versions

@@ -5,4 +5,4 @@ if [[ $0 == ${BASH_SOURCE[0]} ]] ; then
     echo "This script should be *sourced* rather than run directly through bash"
     exit 1
 fi
-export PYVERSIONS='3.6.8 3.7.3 2.6.9 3.3.7 2.7.16 3.2.6 3.1.5 3.4.8'
+export PYVERSIONS='3.7.13 pyston-2.3.3 3.8.13'

admin-tools/setup-master.sh

@@ -1,5 +1,14 @@
 #!/bin/bash
-PYTHON_VERSION=3.6.8
+PYTHON_VERSION=3.7.16
+
+function checkout_version {
+    local repo=$1
+    version=${2:-master}
+    echo Checking out $version on $repo ...
+    (cd ../$repo && git checkout $version && pyenv local $PYTHON_VERSION) && \
+	git pull
+    return $?
+}
 
 # FIXME put some of the below in a common routine
 function finish {
@@ -16,7 +25,7 @@ fi
 mydir=$(dirname $bs)
 fulldir=$(readlink -f $mydir)
 cd $fulldir/..
-(cd ../python-spark && git checkout master && pyenv local $PYTHON_VERSION) && git pull && \
-    (cd ../python-xdis && git checkout master && pyenv local $PYTHON_VERSION) && git pull && \
-    git checkout master && pyenv local $PYTHON_VERSION && git pull
+(cd $fulldir/.. && checkout_version python-spark && checkout_version python-xdis &&
+      checkout_version python-uncompyle6)
 cd $owd
+rm -v */.python-version >/dev/null 2>&1 || true

admin-tools/setup-python-2.4.sh

@@ -1,6 +1,15 @@
 #!/bin/bash
 PYTHON_VERSION=2.4.6
 
+function checkout_version {
+    local repo=$1
+    version=${2:-python-2.4}
+    echo Checking out $version.4 on $repo ...
+    (cd ../$repo && git checkout $version && pyenv local $PYTHON_VERSION) && \
+	git pull
+    return $?
+}
+
 owd=$(pwd)
 bs=${BASH_SOURCE[0]}
 if [[ $0 == $bs ]] ; then
@@ -9,8 +18,7 @@ if [[ $0 == $bs ]] ; then
 fi
 mydir=$(dirname $bs)
 fulldir=$(readlink -f $mydir)
-cd $fulldir/..
-(cd ../python-spark && git checkout python-2.4 && pyenv local $PYTHON_VERSION) && git pull && \
-    (cd ../python-xdis && git checkout python-2.4 && pyenv local $PYTHON_VERSION) && git pull && \
-    git checkout python-2.4 &&  pyenv local $PYTHON_VERSION && git pull
+(cd $fulldir/.. && checkout_version python-spark && checkout_version python-xdis python-2.4-to-2.7 &&
+      checkout_version python-uncompyle6)
 cd $owd
+rm -v */.python-version || true

admin-tools/setup-python-3.0.sh

@@ -0,0 +1,35 @@
+#!/bin/bash
+PYTHON_VERSION=3.0.1
+pyenv local $PYTHON_VERSION
+
+# FIXME put some of the below in a common routine
+function checkout_version {
+    local repo=$1
+    version=${2:-python-3.0-to-3.2}
+    echo Checking out $version on $repo ...
+    (cd ../$repo && git checkout $version && pyenv local $PYTHON_VERSION) && \
+	git pull
+    return $?
+}
+
+function finish {
+  cd $owd
+}
+
+export PATH=$HOME/.pyenv/bin/pyenv:$PATH
+owd=$(pwd)
+bs=${BASH_SOURCE[0]}
+if [[ $0 == $bs ]] ; then
+    echo "This script should be *sourced* rather than run directly through bash"
+    exit 1
+fi
+
+mydir=$(dirname $bs)
+fulldir=$(readlink -f $mydir)
+cd $fulldir/..
+(cd $fulldir/.. && checkout_version python-spark master && checkout_version python-xdis &&
+      checkout_version python-uncompyle6)
+cd $owd
+rm -v */.python-version || true
+
+git checkout python-3.0-to-3.2  && git pull && pyenv local $PYTHON_VERSION

admin-tools/setup-python-3.3.sh

@@ -0,0 +1,35 @@
+#!/bin/bash
+PYTHON_VERSION=3.3.7
+pyenv local $PYTHON_VERSION
+
+# FIXME put some of the below in a common routine
+function checkout_version {
+    local repo=$1
+    version=${2:-python-3.3-to-3.5}
+    echo Checking out $version on $repo ...
+    (cd ../$repo && git checkout $version && pyenv local $PYTHON_VERSION) && \
+	git pull
+    return $?
+}
+
+function finish {
+  cd $owd
+}
+
+export PATH=$HOME/.pyenv/bin/pyenv:$PATH
+owd=$(pwd)
+bs=${BASH_SOURCE[0]}
+if [[ $0 == $bs ]] ; then
+    echo "This script should be *sourced* rather than run directly through bash"
+    exit 1
+fi
+
+mydir=$(dirname $bs)
+fulldir=$(readlink -f $mydir)
+cd $fulldir/..
+(cd $fulldir/.. && checkout_version python-spark master && checkout_version python-xdis &&
+      checkout_version python-uncompyle6)
+cd $owd
+rm -v */.python-version || true
+
+git checkout python-3.3-to-3.5  && git pull && pyenv local $PYTHON_VERSION

appveyor.yml

@@ -1,78 +0,0 @@
-environment:
-  global:
-    # SDK v7.0 MSVC Express 2008's SetEnv.cmd script will fail if the
-    # /E:ON and /V:ON options are not enabled in the batch script intepreter
-    # See: http://stackoverflow.com/a/13751649/163740
-    CMD_IN_ENV: "cmd /E:ON /V:ON /C .\\appveyor\\run_with_env.cmd"
-
-  matrix:
-
-    # Pre-installed Python versions, which Appveyor may upgrade to
-    # a later point release.
-    # See: http://www.appveyor.com/docs/installed-software#python
-
-    # - PYTHON: "C:\\Python27"
-    #   PYTHON_VERSION: "2.7.x"
-    #   PYTHON_ARCH: "32"
-
-    - PYTHON: "C:\\Python27-x64"
-      PYTHON_VERSION: "2.7.x"
-      PYTHON_ARCH: "64"
-
-    # - PYTHON: "C:\\Python26"
-    #   PYTHON_VERSION: "2.6.x"
-    #   PYTHON_ARCH: "32"
-
-    # - PYTHON: "C:\\Python26-x64"
-    #   PYTHON_VERSION: "2.6.x"
-    #   PYTHON_ARCH: "64"
-
-install:
-  # We need wheel installed to build wheels
-  - "%PYTHON%\\python.exe -m pip install wheel"
-
-  # Install Python (from the official .msi of http://python.org) and pip when
-  # not already installed.
-  - ps: if (-not(Test-Path($env:PYTHON))) { & appveyor\install.ps1 }
-
-  # Prepend newly installed Python to the PATH of this build (this cannot be
-  # done from inside the powershell script as it would require to restart
-  # the parent CMD process).
-  - "SET PATH=%PYTHON%;%PYTHON%\\Scripts;%PATH%"
-  - "SET HOME=."
-
-  # Check that we have the expected version and architecture for Python
-  - "python --version"
-  - "python -c \"import struct; print(struct.calcsize('P') * 8)\""
-
-  # Upgrade to the latest version of pip to avoid it displaying warnings
-  # about it being out of date.
-  - "%PYTHON%\\python.exe -m pip install --disable-pip-version-check --user --upgrade pip"
-
-  # Install the build dependencies of the project. If some dependencies contain
-  # compiled extensions and are not provided as pre-built wheel packages,
-  # pip will build them from source using the MSVC compiler matching the
-  # target Python version and architecture
-  - "%CMD_IN_ENV% pip install -r requirements.txt"
-
-build_script:
-  # Build the compiled extension
-  - "%CMD_IN_ENV% python setup.py build"
-
-test_script:
-  # Run the project tests
-  - "%CMD_IN_ENV% python test/test_pyenvlib.py --native --syntax-verify"
-
-after_test:
-  # If tests are successful, create binary packages for the project.
-  - "%CMD_IN_ENV% python setup.py bdist_wininst"
-  - "%CMD_IN_ENV% python setup.py bdist_msi"
-  - ps: "ls dist"
-
-artifacts:
-  # Archive the generated packages in the ci.appveyor.com build report.
-  - path: dist\*
-
-#on_success:
-#  - TODO: upload the content of dist/*.whl to a public wheelhouse
-#

pytest/Makefile

@@ -1,7 +1,11 @@
 PHONY=check test pytest
+SHELL=/bin/bash
 
 PYTHON ?= python
 
 #: Run all tests
 test check pytest:
-	py.test
+	@PYTHON_VERSION=`$(PYTHON) -V 2>&1 | cut -d ' ' -f 2 | cut -d'.' -f1,2`; \
+	 if [[ $$PYTHON_VERSION > 3.2 ]] || [[ $$PYTHON_VERSION == 2.7 ]] || [[ $$PYTHON_VERSION == 2.6 ]]; then \
+		$(PYTHON) -m pytest .; \
+	fi

pytest/test_build_const_key_map.py

@@ -1,10 +1,10 @@
 import pytest
 # uncompyle6
-from uncompyle6 import PYTHON_VERSION
+from xdis.version_info import PYTHON_VERSION_TRIPLE, IS_PYPY
 from validate import validate_uncompyle
 
 
-@pytest.mark.skipif(PYTHON_VERSION < 3.6, reason='need at least python 3.6')
+@pytest.mark.skipif(PYTHON_VERSION_TRIPLE < (3, 6) or IS_PYPY, reason="need at least Python 3.6 and not PyPY")
 @pytest.mark.parametrize('text', (
     "{0.: 'a', -1: 'b'}",   # BUILD_MAP
     "{'a':'b'}",            # BUILD_MAP

pytest/test_deparse.py

@@ -1,6 +1,5 @@
-import pytest
-from uncompyle6.semantics.fragments import code_deparse as deparse, deparsed_find
-from uncompyle6 import PYTHON_VERSION, PYTHON3
+from uncompyle6.semantics.fragments import code_deparse as deparse
+from xdis.version_info import PYTHON_VERSION_TRIPLE
 
 def map_stmts(x, y):
     x = []
@@ -30,20 +29,18 @@ def list_comp():
     [y for y in range(3)]
 
 def get_parsed_for_fn(fn):
-    code = fn.__code__ if PYTHON3 else fn.func_code
-    return deparse(code, version=PYTHON_VERSION)
+    code = fn.__code__
+    return deparse(code, version=PYTHON_VERSION_TRIPLE)
 
 def check_expect(expect, parsed, fn_name):
     debug = False
     i = 2
     max_expect = len(expect)
-    code = get_parsed_for_fn(fn_name)
     for name, offset in sorted(parsed.offsets.keys()):
         assert i+1 <= max_expect, (
             "%s: ran out if items in testing node" % fn_name)
         nodeInfo = parsed.offsets[name, offset]
         node = nodeInfo.node
-        nodeInfo2 = deparsed_find((name, offset), parsed, code)
         extractInfo = parsed.extract_node_info(node)
 
         assert expect[i] == extractInfo.selectedLine, \
@@ -319,5 +316,3 @@ for i in range(2): ...
 .
 """.split("\n")
     parsed = get_parsed_for_fn(for_range_stmt)
-    if not PYTHON3:
-        check_expect(expect, parsed, 'range_stmt')

pytest/test_disasm.py

@@ -1,7 +1,7 @@
 import os.path
 import pytest
 
-from uncompyle6.disas import disassemble_file
+from uncompyle6.code_fns import disassemble_file
 
 def get_srcdir():
     filename = os.path.normcase(os.path.dirname(__file__))

pytest/test_fjt.py

@@ -1,5 +1,5 @@
 #!/usr/bin/env python
-from uncompyle6 import PYTHON_VERSION, IS_PYPY
+from xdis.version_info import PYTHON_VERSION_TRIPLE, IS_PYPY, version_tuple_to_str
 from uncompyle6.scanner import get_scanner
 def bug(state, slotstate):
     if state:
@@ -21,8 +21,8 @@ def bug_loop(disassemble, tb=None):
 
 def test_if_in_for():
     code = bug.__code__
-    scan = get_scanner(PYTHON_VERSION)
-    if 2.7 <= PYTHON_VERSION <= 3.0 and not IS_PYPY:
+    scan = get_scanner(PYTHON_VERSION_TRIPLE)
+    if (2, 7) <= PYTHON_VERSION_TRIPLE < (3, 1) and not IS_PYPY:
         scan.build_instructions(code)
         fjt = scan.find_jump_targets(False)
 
@@ -51,7 +51,7 @@ def test_if_in_for():
             # previous bug was not mistaking while-loop for if-then
             {'start': 48, 'end': 67, 'type': 'while-loop'}]
 
-    elif 3.2 < PYTHON_VERSION <= 3.4:
+    elif (3, 2) < PYTHON_VERSION_TRIPLE <= (3, 4):
         scan.build_instructions(code)
         fjt  = scan.find_jump_targets(False)
         assert {69: [66], 63: [18]} == fjt
@@ -62,6 +62,6 @@ def test_if_in_for():
            {'end': 59, 'type': 'for-loop', 'start': 31},
            {'end': 63, 'type': 'for-else', 'start': 62}]
     else:
-        print("FIXME: should fix for %s" % PYTHON_VERSION)
+        print("FIXME: should fix for %s" % version_tuple_to_str())
         assert True
     return

pytest/test_fstring.py

@@ -1,158 +0,0 @@
-# std
-# test
-import sys
-from uncompyle6 import PYTHON_VERSION, code_deparse
-import pytest
-
-pytestmark = pytest.mark.skipif(
-    PYTHON_VERSION <= 2.6, reason="hypothesis needs 2.7 or later"
-)
-if PYTHON_VERSION > 2.6:
-
-    import hypothesis
-    from hypothesis import strategies as st
-
-    # uncompyle6
-
-    @st.composite
-    def expressions(draw):
-        # todo : would be nice to generate expressions using hypothesis however
-        # this is pretty involved so for now just use a corpus of expressions
-        # from which to select.
-        return draw(
-            st.sampled_from(
-                (
-                    "abc",
-                    "len(items)",
-                    "x + 1",
-                    "lineno",
-                    "container",
-                    "self.attribute",
-                    "self.method()",
-                    # These expressions are failing, I think these are control
-                    # flow problems rather than problems with FORMAT_VALUE,
-                    # however I need to confirm this...
-                    #'sorted(items, key=lambda x: x.name)',
-                    #'func(*args, **kwargs)',
-                    #'text or default',
-                    #'43 if life_the_universe and everything else None'
-                )
-            )
-        )
-
-    @st.composite
-    def format_specifiers(draw):
-        """
-        Generate a valid format specifier using the rules:
-
-        format_spec ::=  [[fill]align][sign][#][0][width][,][.precision][type]
-        fill        ::=  <any character>
-        align       ::=  "<" | ">" | "=" | "^"
-        sign        ::=  "+" | "-" | " "
-        width       ::=  integer
-        precision   ::=  integer
-        type        ::=  "b" | "c" | "d" | "e" | "E" | "f" | "F" | "g" | "G" | "n" | "o" | "s" | "x" | "X" | "%"
-
-        See https://docs.python.org/2/library/string.html
-
-        :param draw: Let hypothesis draw from other strategies.
-
-        :return: An example format_specifier.
-        """
-        alphabet_strategy = st.characters(
-            min_codepoint=ord("a"), max_codepoint=ord("z")
-        )
-        fill = draw(st.one_of(alphabet_strategy, st.none()))
-        align = draw(st.sampled_from(list("<>=^")))
-        fill_align = (fill + align or "") if fill else ""
-
-        type_ = draw(st.sampled_from("bcdeEfFgGnosxX%"))
-        can_have_sign = type_ in "deEfFgGnoxX%"
-        can_have_comma = type_ in "deEfFgG%"
-        can_have_precision = type_ in "fFgG"
-        can_have_pound = type_ in "boxX%"
-        can_have_zero = type_ in "oxX"
-
-        sign = draw(st.sampled_from(list("+- ") + [""])) if can_have_sign else ""
-        pound = draw(st.sampled_from(("#", ""))) if can_have_pound else ""
-        zero = draw(st.sampled_from(("0", ""))) if can_have_zero else ""
-
-        int_strategy = st.integers(min_value=1, max_value=1000)
-
-        width = draw(st.one_of(int_strategy, st.none()))
-        width = str(width) if width is not None else ""
-
-        comma = draw(st.sampled_from((",", ""))) if can_have_comma else ""
-        if can_have_precision:
-            precision = draw(st.one_of(int_strategy, st.none()))
-            precision = "." + str(precision) if precision else ""
-        else:
-            precision = ""
-
-        return "".join((fill_align, sign, pound, zero, width, comma, precision, type_))
-
-    @st.composite
-    def fstrings(draw):
-        """
-        Generate a valid f-string.
-        See https://www.python.org/dev/peps/pep-0498/#specification
-
-        :param draw: Let hypothsis draw from other strategies.
-
-        :return: A valid f-string.
-        """
-        character_strategy = st.characters(
-            blacklist_characters="\r\n'\\s{}", min_codepoint=1, max_codepoint=1000
-        )
-        is_raw = draw(st.booleans())
-        integer_strategy = st.integers(min_value=0, max_value=3)
-        expression_count = draw(integer_strategy)
-        content = []
-        for _ in range(expression_count):
-            expression = draw(expressions())
-            conversion = draw(st.sampled_from(("", "!s", "!r", "!a")))
-            has_specifier = draw(st.booleans())
-            specifier = ":" + draw(format_specifiers()) if has_specifier else ""
-            content.append("{{{}{}}}".format(expression, conversion, specifier))
-            content.append(draw(st.text(character_strategy)))
-        content = "".join(content)
-        return "f{}'{}'".format("r" if is_raw else "", content)
-
-    @pytest.mark.skipif(PYTHON_VERSION != 3.6, reason="need Python 3.6")
-    @hypothesis.given(format_specifiers())
-    def test_format_specifiers(format_specifier):
-        """Verify that format_specifiers generates valid specifiers"""
-        try:
-            exec('"{:' + format_specifier + '}".format(0)')
-        except ValueError as e:
-            if "Unknown format code" not in str(e):
-                raise
-
-    def run_test(text):
-        hypothesis.assume(len(text))
-        hypothesis.assume("f'{" in text)
-        expr = text + "\n"
-        code = compile(expr, "<string>", "single")
-        deparsed = code_deparse(code, sys.stdout, PYTHON_VERSION, compile_mode="single")
-        recompiled = compile(deparsed.text, "<string>", "single")
-        if recompiled != code:
-            print(recompiled)
-            print("================")
-            print(code)
-            print("----------------")
-            assert (
-                "dis(" + deparsed.text.strip("\n") + ")"
-                == "dis(" + expr.strip("\n") + ")"
-            )
-
-    @pytest.mark.skipif(PYTHON_VERSION != 3.6, reason="need Python 3.6")
-    @hypothesis.given(fstrings())
-    def test_uncompyle_fstring(fstring):
-        """Verify uncompyling fstring bytecode"""
-        run_test(fstring)
-
-    @pytest.mark.skipif(PYTHON_VERSION != 3.6, reason="need Python 3.6+")
-    @pytest.mark.parametrize("fstring", ["f'{abc}{abc!s}'", "f'{abc}0'"])
-    def test_uncompyle_direct(fstring):
-        """useful for debugging"""
-        run_test(fstring)

pytest/test_function_call.py

@@ -1,185 +0,0 @@
-import string
-from uncompyle6 import PYTHON_VERSION
-import pytest
-pytestmark = pytest.mark.skip(PYTHON_VERSION < 2.7,
-                              reason="need at least Python 2.7")
-
-if PYTHON_VERSION > 2.6:
-    from hypothesis import given, assume, example, settings, strategies as st
-    from validate import validate_uncompyle
-    from test_fstring import expressions
-
-    alpha = st.sampled_from(string.ascii_lowercase)
-    numbers = st.sampled_from(string.digits)
-    alphanum = st.sampled_from(string.ascii_lowercase + string.digits)
-
-
-    @st.composite
-    def function_calls(draw,
-                       min_keyword_args=0, max_keyword_args=5,
-                       min_positional_args=0, max_positional_args=5,
-                       min_star_args=0, max_star_args=1,
-                       min_double_star_args=0, max_double_star_args=1):
-        """
-        Strategy factory for generating function calls.
-
-        :param draw: Callable which draws examples from other strategies.
-
-        :return: The function call text.
-        """
-        st_positional_args = st.lists(
-            alpha,
-            min_size=min_positional_args,
-            max_size=max_positional_args
-        )
-        st_keyword_args = st.lists(
-            alpha,
-            min_size=min_keyword_args,
-            max_size=max_keyword_args
-        )
-        st_star_args = st.lists(
-            alpha,
-            min_size=min_star_args,
-            max_size=max_star_args
-        )
-        st_double_star_args = st.lists(
-            alpha,
-            min_size=min_double_star_args,
-            max_size=max_double_star_args
-        )
-
-        positional_args = draw(st_positional_args)
-        keyword_args = draw(st_keyword_args)
-        st_values = st.lists(
-            expressions(),
-            min_size=len(keyword_args),
-            max_size=len(keyword_args)
-        )
-        keyword_args = [
-            x + '=' + e
-            for x, e in
-            zip(keyword_args, draw(st_values))
-        ]
-        star_args = ['*' + x for x in draw(st_star_args)]
-        double_star_args = ['**' + x for x in draw(st_double_star_args)]
-
-        arguments = positional_args + keyword_args + star_args + double_star_args
-        draw(st.randoms()).shuffle(arguments)
-        arguments = ','.join(arguments)
-
-        function_call = 'fn({arguments})'.format(arguments=arguments)
-        try:
-            # TODO: Figure out the exact rules for ordering of positional, keyword,
-            # star args, double star args and in which versions the various
-            # types of arguments are supported so we don't need to check that the
-            # expression compiles like this.
-            compile(function_call, '<string>', 'single')
-        except:
-            assume(False)
-        return function_call
-
-
-    def test_function_no_args():
-        validate_uncompyle("fn()")
-
-    @pytest.mark.skipif(PYTHON_VERSION < 2.7,
-                        reason="need at least Python 2.7")
-    def isolated_function_calls(which):
-        """
-        Returns a strategy for generating function calls, but isolated to
-        particular types of arguments, for example only positional arguments.
-
-        This can help reason about debugging errors in specific types of function
-        calls.
-
-        :param which: One of 'keyword', 'positional', 'star', 'double_star'
-
-        :return: Strategy for generating an function call isolated to specific
-                 argument types.
-        """
-        kwargs = dict(
-            max_keyword_args=0,
-            max_positional_args=0,
-            max_star_args=0,
-            max_double_star_args=0,
-        )
-        kwargs['_'.join(('min', which, 'args'))] = 1
-        kwargs['_'.join(('max', which, 'args'))] = 5 if 'star' not in which else 1
-        return function_calls(**kwargs)
-
-
-    with settings(max_examples=25):
-
-        @pytest.mark.skipif(PYTHON_VERSION < 2.7,
-                            reason="need at least Python 2.7")
-        @given(isolated_function_calls('positional'))
-        @example("fn(0)")
-        def test_function_positional_only(expr):
-            validate_uncompyle(expr)
-
-        @pytest.mark.skipif(PYTHON_VERSION < 2.7,
-                            reason="need at least Python 2.7")
-        @given(isolated_function_calls('keyword'))
-        @example("fn(a=0)")
-        def test_function_call_keyword_only(expr):
-            validate_uncompyle(expr)
-
-        @pytest.mark.skipif(PYTHON_VERSION < 2.7,
-                            reason="need at least Python 2.7")
-        @given(isolated_function_calls('star'))
-        @example("fn(*items)")
-        def test_function_call_star_only(expr):
-            validate_uncompyle(expr)
-
-        @pytest.mark.skipif(PYTHON_VERSION < 2.7,
-                            reason="need at least Python 2.7")
-        @given(isolated_function_calls('double_star'))
-        @example("fn(**{})")
-        def test_function_call_double_star_only(expr):
-            validate_uncompyle(expr)
-
-
-    @pytest.mark.xfail()
-    def test_BUILD_CONST_KEY_MAP_BUILD_MAP_UNPACK_WITH_CALL_BUILD_TUPLE_CALL_FUNCTION_EX():
-        validate_uncompyle("fn(w=0,m=0,**v)")
-
-
-    @pytest.mark.xfail()
-    def test_BUILD_MAP_BUILD_MAP_UNPACK_WITH_CALL_BUILD_TUPLE_CALL_FUNCTION_EX():
-        validate_uncompyle("fn(a=0,**g)")
-
-
-    @pytest.mark.xfail()
-    def test_CALL_FUNCTION_EX():
-        validate_uncompyle("fn(*g,**j)")
-
-
-    @pytest.mark.xfail()
-    def test_BUILD_MAP_CALL_FUNCTION_EX():
-        validate_uncompyle("fn(*z,u=0)")
-
-
-    @pytest.mark.xfail()
-    def test_BUILD_TUPLE_CALL_FUNCTION_EX():
-        validate_uncompyle("fn(**a)")
-
-
-    @pytest.mark.xfail()
-    def test_BUILD_MAP_BUILD_TUPLE_BUILD_TUPLE_UNPACK_WITH_CALL_CALL_FUNCTION_EX():
-        validate_uncompyle("fn(b,b,b=0,*a)")
-
-
-    @pytest.mark.xfail()
-    def test_BUILD_TUPLE_BUILD_TUPLE_UNPACK_WITH_CALL_CALL_FUNCTION_EX():
-        validate_uncompyle("fn(*c,v)")
-
-
-    @pytest.mark.xfail()
-    def test_BUILD_CONST_KEY_MAP_CALL_FUNCTION_EX():
-        validate_uncompyle("fn(i=0,y=0,*p)")
-
-
-    @pytest.mark.skip(reason='skipping property based test until all individual tests are passing')
-    @given(function_calls())
-    def test_function_call(function_call):
-        validate_uncompyle(function_call)

pytest/test_grammar.py

@@ -1,89 +1,111 @@
 import re
-from uncompyle6 import PYTHON_VERSION, PYTHON3, IS_PYPY # , PYTHON_VERSION
 from uncompyle6.parser import get_python_parser, python_parser
 from uncompyle6.scanner import get_scanner
+from xdis.version_info import PYTHON_VERSION_TRIPLE, IS_PYPY
+
 
 def test_grammar():
-
     def check_tokens(tokens, opcode_set):
         remain_tokens = set(tokens) - opcode_set
-        remain_tokens = set([re.sub(r'_\d+$','', t) for t in remain_tokens])
-        remain_tokens = set([re.sub('_CONT$','', t) for t in remain_tokens])
-        remain_tokens = set([re.sub('LOAD_CODE$','', t) for t in remain_tokens])
+        remain_tokens = set([re.sub(r"_\d+$", "", t) for t in remain_tokens])
+        remain_tokens = set([re.sub("_CONT$", "", t) for t in remain_tokens])
+        remain_tokens = set([re.sub("LOAD_CODE$", "", t) for t in remain_tokens])
         remain_tokens = set(remain_tokens) - opcode_set
-        assert remain_tokens == set([]), \
-            "Remaining tokens %s\n====\n%s" % (remain_tokens, p.dump_grammar())
+        assert remain_tokens == set([]), "Remaining tokens %s\n====\n%s" % (
+            remain_tokens,
+            p.dump_grammar(),
+        )
 
-    p = get_python_parser(PYTHON_VERSION, is_pypy=IS_PYPY)
-    (lhs, rhs, tokens,
-     right_recursive, dup_rhs) = p.check_sets()
+    p = get_python_parser(PYTHON_VERSION_TRIPLE, is_pypy=IS_PYPY)
+    (lhs, rhs, tokens, right_recursive, dup_rhs) = p.check_sets()
 
     # We have custom rules that create the below
-    expect_lhs = set(['pos_arg', 'attribute'])
-    if PYTHON_VERSION < 3.8:
-        expect_lhs.add('get_iter')
+    expect_lhs = set(["pos_arg"])
 
+    if PYTHON_VERSION_TRIPLE < (3, 8):
+        if PYTHON_VERSION_TRIPLE < (3, 7):
+            expect_lhs.add("attribute")
 
-    unused_rhs = set(['list', 'mkfunc',
-                      'mklambda',
-                      'unpack',])
+        expect_lhs.add("get_iter")
 
-    expect_right_recursive = set([('designList',
-                                   ('store', 'DUP_TOP', 'designList'))])
-
-    if PYTHON_VERSION < 3.7:
-        unused_rhs.add('call')
-
-    if PYTHON_VERSION > 2.6:
-        expect_lhs.add('kvlist')
-        expect_lhs.add('kv3')
-        unused_rhs.add('dict')
-
-    if PYTHON3:
-        expect_lhs.add('load_genexpr')
-
-        unused_rhs = unused_rhs.union(set("""
-        except_pop_except generator_exp
-        """.split()))
-        if PYTHON_VERSION >= 3.0:
-            expect_lhs.add("annotate_arg")
-            expect_lhs.add("annotate_tuple")
-            unused_rhs.add("mkfunc_annotate")
-            unused_rhs.add("dict_comp")
-            unused_rhs.add("classdefdeco1")
-            unused_rhs.add("tryelsestmtl")
-            if PYTHON_VERSION >= 3.5:
-                expect_right_recursive.add((('l_stmts',
-                                             ('lastl_stmt', 'come_froms', 'l_stmts'))))
-                pass
-            elif 3.0 < PYTHON_VERSION < 3.3:
-                expect_right_recursive.add((('l_stmts',
-                                             ('lastl_stmt', 'COME_FROM', 'l_stmts'))))
-                pass
-            pass
-        pass
+        if PYTHON_VERSION_TRIPLE >= (3, 8) or PYTHON_VERSION_TRIPLE < (3, 0):
+            expect_lhs.add("stmts_opt")
     else:
-        expect_lhs.add('kwarg')
+        expect_lhs.add("async_with_as_stmt")
+        expect_lhs.add("async_with_stmt")
 
-    assert expect_lhs == set(lhs)
+    unused_rhs = set(["list", "mkfunc", "lambda_body", "unpack"])
 
+    expect_right_recursive = set([("designList", ("store", "DUP_TOP", "designList"))])
+
+    if PYTHON_VERSION_TRIPLE[:2] <= (3, 6):
+        unused_rhs.add("call")
+
+    if PYTHON_VERSION_TRIPLE >= (2, 7):
+        expect_lhs.add("kvlist")
+        expect_lhs.add("kv3")
+        unused_rhs.add("dict")
+
+        if PYTHON_VERSION_TRIPLE < (3, 7) and PYTHON_VERSION_TRIPLE[:2] != (2, 7):
+            # NOTE: this may disappear
+            expect_lhs.add("except_handler_else")
+
+    expect_lhs.add("load_genexpr")
+
+    unused_rhs = unused_rhs.union(
+        set(
+            """
+    except_pop_except generator_exp
+    """.split()
+        )
+    )
+    if PYTHON_VERSION_TRIPLE < (3, 7):
+        expect_lhs.add("annotate_arg")
+        expect_lhs.add("annotate_tuple")
+        unused_rhs.add("mkfunc_annotate")
+
+    unused_rhs.add("dict_comp")
+    unused_rhs.add("classdefdeco1")
+    unused_rhs.add("tryelsestmtl")
+    if PYTHON_VERSION_TRIPLE >= (3, 5):
+        expect_right_recursive.add(
+            (("l_stmts", ("lastl_stmt", "come_froms", "l_stmts")))
+        )
+        pass
+    pass
+
+    if PYTHON_VERSION_TRIPLE >= (3, 7):
+        expect_lhs.add("set_for")
+        unused_rhs.add("set_iter")
+        pass
+    pass
     # FIXME
-    if PYTHON_VERSION != 3.8:
+    if PYTHON_VERSION_TRIPLE < (3, 8):
+        assert expect_lhs == set(lhs)
         assert unused_rhs == set(rhs)
 
     assert expect_right_recursive == right_recursive
 
-    expect_dup_rhs = frozenset([('COME_FROM',), ('CONTINUE',), ('JUMP_ABSOLUTE',),
-                                ('LOAD_CONST',),
-                                ('JUMP_BACK',), ('JUMP_FORWARD',)])
+    expect_dup_rhs = frozenset(
+        [
+            ("COME_FROM",),
+            ("CONTINUE",),
+            ("JUMP_ABSOLUTE",),
+            ("LOAD_CONST",),
+            ("JUMP_BACK",),
+            ("JUMP_FORWARD",),
+        ]
+    )
     reduced_dup_rhs = dict((k, dup_rhs[k]) for k in dup_rhs if k not in expect_dup_rhs)
-    for k in reduced_dup_rhs:
-        print(k, reduced_dup_rhs[k])
+    if reduced_dup_rhs:
+        print("\nPossible duplicate RHS that might be folded, into one of the LHS symbols")
+        for k in reduced_dup_rhs:
+            print(k, reduced_dup_rhs[k])
     # assert not reduced_dup_rhs, reduced_dup_rhs
 
-    s = get_scanner(PYTHON_VERSION, IS_PYPY)
+    s = get_scanner(PYTHON_VERSION_TRIPLE, IS_PYPY)
     ignore_set = set(
-            """
+        """
             JUMP_BACK CONTINUE
             COME_FROM COME_FROM_EXCEPT
             COME_FROM_EXCEPT_CLAUSE
@@ -92,22 +114,34 @@ def test_grammar():
             LOAD_GENEXPR LOAD_ASSERT LOAD_SETCOMP LOAD_DICTCOMP LOAD_STR LOAD_CODE
             LAMBDA_MARKER
             RETURN_END_IF RETURN_END_IF_LAMBDA RETURN_VALUE_LAMBDA RETURN_LAST
-            """.split())
-    if 2.6 <= PYTHON_VERSION <= 2.7:
+            """.split()
+    )
+
+    if (2, 6) <= PYTHON_VERSION_TRIPLE <= (2, 7):
         opcode_set = set(s.opc.opname).union(ignore_set)
-        if PYTHON_VERSION == 2.6:
+        if PYTHON_VERSION_TRIPLE[:2] == (2, 6):
             opcode_set.add("THEN")
         check_tokens(tokens, opcode_set)
-    elif PYTHON_VERSION == 3.4:
-        ignore_set.add('LOAD_CLASSNAME')
-        ignore_set.add('STORE_LOCALS')
+    elif PYTHON_VERSION_TRIPLE[:2] == (3, 4):
+        ignore_set.add("LOAD_CLASSNAME")
+        ignore_set.add("STORE_LOCALS")
         opcode_set = set(s.opc.opname).union(ignore_set)
         check_tokens(tokens, opcode_set)
 
+
 def test_dup_rule():
     import inspect
-    python_parser(PYTHON_VERSION, inspect.currentframe().f_code,
-                  is_pypy=IS_PYPY,
-                  parser_debug={
-                      'dups': True, 'transition': False, 'reduce': False,
-                      'rules': False, 'errorstack': None, 'context': True})
+
+    python_parser(
+        PYTHON_VERSION_TRIPLE,
+        inspect.currentframe().f_code,
+        is_pypy=IS_PYPY,
+        parser_debug={
+            "dups": True,
+            "transition": False,
+            "reduce": False,
+            "rules": False,
+            "errorstack": None,
+            "context": True,
+        },
+    )

pytest/test_pysource.py-notyet

@@ -1,19 +1,13 @@
 import sys
-from uncompyle6 import PYTHON3
 from uncompyle6.scanner import get_scanner
 from uncompyle6.semantics.consts import (
     escape, NONE,
     # RETURN_NONE, PASS, RETURN_LOCALS
 )
 
-if PYTHON3:
-    from io import StringIO
-    def iteritems(d):
-        return d.items()
-else:
-    from StringIO import StringIO
-    def iteritems(d):
-        return d.iteritems()
+from io import StringIO
+def iteritems(d):
+    return d.items()
 
 from uncompyle6.semantics.pysource import (SourceWalker, deparse_code2str)
 

pytest/test_single_compile.py

@@ -1,22 +1,24 @@
 import pytest
-from uncompyle6 import PYTHON_VERSION, code_deparse
-pytestmark = pytest.mark.skip(PYTHON_VERSION < 2.7,
-                              reason="need at least Python 2.7")
+from uncompyle6 import code_deparse
+from xdis.version_info import PYTHON_VERSION_TRIPLE
 
-if PYTHON_VERSION > 2.6:
-    def test_single_mode():
-        single_expressions = (
-            'i = 1',
-            'i and (j or k)',
-            'i += 1',
-            'i = j % 4',
-            'i = {}',
-            'i = []',
-            'for i in range(10):\n    i\n',
-            'for i in range(10):\n    for j in range(10):\n        i + j\n',
-            'try:\n    i\nexcept Exception:\n    j\nelse:\n    k\n'
-        )
+pytest.mark.skip(PYTHON_VERSION_TRIPLE < (2, 7), reason="need Python < 2.7")
 
-        for expr in single_expressions:
-            code = compile(expr + '\n', '<string>', 'single')
-            assert code_deparse(code, compile_mode='single').text == expr + '\n'
+
+def test_single_mode():
+    single_expressions = (
+        "i = 1",
+        "i and (j or k)",
+        "i += 1",
+        "i = j % 4",
+        "i = {}",
+        "i = []",
+        "for i in range(10):\n    i\n",
+        "for i in range(10):\n    for j in range(10):\n        i + j\n",
+        # 'try:\n    i\nexcept Exception:\n    j\nelse:\n    k\n'
+    )
+
+    for expr in single_expressions:
+        code = compile(expr + "\n", "<string>", "single")
+        got = code_deparse(code, compile_mode="single").text
+        assert got == expr + "\n"

pytest/test_token.py

@@ -1,24 +1,24 @@
-from uncompyle6 import PYTHON_VERSION
 from uncompyle6.scanners.tok import Token
 
+
 def test_token():
     # Test token formatting of: LOAD_CONST None
-    t = Token('LOAD_CONST', offset=0, attr=None, pattr=None, has_arg=True)
-    expect = '           0  LOAD_CONST               None'
+    t = Token("LOAD_CONST", offset=0, attr=None, pattr=None, has_arg=True)
+    expect = "0  LOAD_CONST               None"
     # print(t.format())
     assert t
-    assert t.format() == expect
+    assert t.format().strip() == expect.strip()
 
     # Make sure equality testing of tokens ignores offset
-    t2 = Token('LOAD_CONST', offset=2, attr=None, pattr=None, has_arg=True)
+    t2 = Token("LOAD_CONST", offset=2, attr=None, pattr=None, has_arg=True)
     assert t2 == t
 
-
     # Make sure formatting of: LOAD_CONST False. We assume False is the 0th index
     # of co_consts.
-    t = Token('LOAD_CONST', offset=1, attr=False, pattr=False, has_arg=True)
-    expect = '           1  LOAD_CONST               False'
-    assert t.format() == expect
+    t = Token("LOAD_CONST", offset=1, attr=False, pattr=False, has_arg=True)
+    expect = "1  LOAD_CONST               False"
+    assert t.format().strip() == expect.strip()
 
-if __name__ == '__main__':
+
+if __name__ == "__main__":
     test_token()

pytest/testdata/if-2.7.right

@@ -1,12 +1,12 @@
 # Python 2.7
 # Embedded file name: simple_source/branching/05_if.py
 
-   6       0  LOAD_NAME             0  'True'
-           3  POP_JUMP_IF_FALSE    15  'to 15'
+ L.   6         0  LOAD_NAME             0  'True'
+                3  POP_JUMP_IF_FALSE    15  'to 15'
 
-   7       6  LOAD_NAME             1  'False'
-           9  STORE_NAME            2  'b'
-          12  JUMP_FORWARD          0  'to 15'
-        15_0  COME_FROM            12  '12'
-          15  LOAD_CONST               None
-          18  RETURN_VALUE     
+ L.   7         6  LOAD_NAME             1  'False'
+                9  STORE_NAME            2  'b'
+               12  JUMP_FORWARD          0  'to 15'
+             15_0  COME_FROM            12  '12'
+               15  LOAD_CONST               None
+               18  RETURN_VALUE     

pytest/testdata/ifelse-2.7.right

@@ -1,15 +1,15 @@
 # Python 2.7
 # Embedded file name: simple_source/branching/05_ifelse.py
 
-   3       0  LOAD_NAME             0  'True'
-           3  POP_JUMP_IF_FALSE    15  'to 15'
+ L.   3         0  LOAD_NAME             0  'True'
+                3  POP_JUMP_IF_FALSE    15  'to 15'
 
-   4       6  LOAD_CONST               1
-           9  STORE_NAME            1  'b'
-          12  JUMP_FORWARD          6  'to 21'
+ L.   4         6  LOAD_CONST               1
+                9  STORE_NAME            1  'b'
+               12  JUMP_FORWARD          6  'to 21'
 
-   6      15  LOAD_CONST               2
-          18  STORE_NAME            2  'd'
-        21_0  COME_FROM            12  '12'
-          21  LOAD_CONST               None
-          24  RETURN_VALUE     
+ L.   6        15  LOAD_CONST               2
+               18  STORE_NAME            2  'd'
+             21_0  COME_FROM            12  '12'
+               21  LOAD_CONST               None
+               24  RETURN_VALUE     

pytest/validate.py

@@ -9,22 +9,16 @@ import tempfile
 import functools
 
 # uncompyle6 / xdis
-from uncompyle6 import PYTHON_VERSION, PYTHON3, IS_PYPY, code_deparse
+from uncompyle6 import code_deparse
+from xdis.version_info import PYTHON_VERSION_TRIPLE, IS_PYPY
 
 # TODO : I think we can get xdis to support the dis api (python 3 version) by doing something like this there
-from xdis.bytecode import Bytecode
-from xdis.main import get_opcode
+from xdis import Bytecode, get_opcode
 
-opc = get_opcode(PYTHON_VERSION, IS_PYPY)
+opc = get_opcode(PYTHON_VERSION_TRIPLE, IS_PYPY)
 Bytecode = functools.partial(Bytecode, opc=opc)
 import six
 
-if PYTHON3:
-    from io import StringIO
-else:
-    from StringIO import StringIO
-
-
 def _dis_to_text(co):
     return Bytecode(co).dis()
 
@@ -128,7 +122,7 @@ def validate_uncompyle(text, mode="exec"):
     original_text = text
 
     deparsed = code_deparse(
-        original_code, out=six.StringIO(), version=PYTHON_VERSION, compile_mode=mode
+        original_code, out=six.StringIO(), version=PYTHON_VERSION_TRIPLE, compile_mode=mode
     )
     uncompyled_text = deparsed.text
     uncompyled_code = compile(uncompyled_text, "<string>", "exec")

requirements-dev.txt

@@ -1,4 +1,3 @@
 flake8
-hypothesis<=3.0.0
 six
-pytest==3.2.5
+pytest==3.2.5 # for 2.7 < PYTHON_VERSION <= 3.2 use pytest 2.9.2; for 3.1 2.10

requirements.txt

@@ -2,3 +2,8 @@
 hypothesis==2.0.0
 pytest
 -e .
+
+Click~=7.0
+xdis>=6.0.4
+configobj~=5.0.6
+setuptools~=65.5.1

setup.cfg

@@ -1,11 +1,57 @@
 [bdist_rpm]
-release = 1
-packager = Mysterie <kajusska@gmail.com>
-doc_files = README
-#		CHANGES.txt
-#		USAGE.txt
+release = 0
+packager = rocky <rb@dustyfeet.com
+doc_files = README.rst
+		ChangeLog
+		COPYING
+		DECOMPYLE-2.4-CHANGELOG.txt
+		HISTORY.md
+		HOW-TO_REPORT-A-BUG.md
+		NEWS.md
 #		doc/
 #		examples/
 
 [bdist_wheel]
 universal=1
+
+[metadata]
+description_file = README.rst
+
+[flake8]
+# max-line-length setting: NO we do not want everyone writing 120-character lines!
+# We are setting the maximum line length big here because there are longer
+# lines allowed by black in some cases that are forbidden by flake8. Since
+# black has the final say about code formatting issues, this setting is here to
+# make sure that flake8 doesn't fail the build on longer lines allowed by
+# black.
+max-line-length = 120
+max-complexity = 12
+select = E,F,W,C,B,B9
+ignore =
+    # E123 closing bracket does not match indentation of opening bracket's line
+    E123
+    # E203 whitespace before ':' (Not PEP8 compliant, Python Black)
+    E203
+    # E501 line too long (82 > 79 characters) (replaced by B950 from flake8-bugbear,
+    # https://github.com/PyCQA/flake8-bugbear)
+    E501
+    # W503 line break before binary operator (Not PEP8 compliant, Python Black)
+    W503
+    # W504 line break after binary operator (Not PEP8 compliant, Python Black)
+    W504
+    # C901 function too complex - since many of zz9 functions are too complex with a lot
+    # of if branching
+    C901
+    # module level import not at top of file. This is too restrictive. Can't even have a
+    # docstring higher.
+    E402
+per-file-ignores =
+    # These are config files. The `c` variable them is injected not defined.
+    pow/ansible/roles/jupyterhub/templates/jupyterhub_config*.py:F821
+    # Ignore some errors in files that are stolen from other projects to avoid lots
+    # of merge problems later .
+    pow/ansible/roles/webtier/files/supervisor_httpgroupok.py:E126,E128,E222,E225,E226,E261,E301,E302,E305,F841,E201,E202
+    silhouette/src/silhouette/gprof2dot.py:E711,E713,E741,F401
+    # Ignore undefined name errors in "expectation" test Python code.
+    # These files get exec'd in an environment that defines the variables.
+    server/tests/files/expectations/*.py:F821

setup.py

@@ -1,42 +1,61 @@
 #!/usr/bin/env python
+import setuptools
 import sys
 
 """Setup script for the 'uncompyle6' distribution."""
 
 SYS_VERSION = sys.version_info[0:2]
-if not ((2, 6) <= SYS_VERSION  <= (3, 8)):
-    mess = "Python Release 2.6 .. 3.8 are supported in this code branch."
-    if ((2, 4) <= SYS_VERSION <= (2, 7)):
-        mess += ("\nFor your Python, version %s, use the python-2.4 code/branch." %
-                 sys.version[0:3])
+if not ((2, 4) <= SYS_VERSION < (3, 12)):
+    mess = "Python Release 2.6 .. 3.11 are supported in this code branch."
+    if (2, 4) <= SYS_VERSION <= (2, 7):
+        mess += (
+            "\nFor your Python, version %s, use the python-2.4 code/branch."
+            % sys.version[0:3]
+        )
+    if (3, 3) <= SYS_VERSION < (3, 6):
+        mess += (
+            "\nFor your Python, version %s, use the python-3.3-to-3.5 code/branch."
+            % sys.version[0:3]
+        )
     elif SYS_VERSION < (2, 4):
-        mess += ("\nThis package is not supported for Python version %s."
-                 % sys.version[0:3])
+        mess += (
+            "\nThis package is not supported for Python version %s." % sys.version[0:3]
+        )
     print(mess)
     raise Exception(mess)
 
-from __pkginfo__ import \
-    author,           author_email,       install_requires,          \
-    license,          long_description,   classifiers,               \
-    entry_points,     modname,            py_modules,                \
-    short_desc,       VERSION,            web,                       \
-    zip_safe
+from __pkginfo__ import (
+    author,
+    author_email,
+    install_requires,
+    license,
+    long_description,
+    classifiers,
+    entry_points,
+    modname,
+    py_modules,
+    short_desc,
+    __version__,
+    web,
+    zip_safe,
+)
 
-from setuptools import setup, find_packages
-setup(
-       author             = author,
-       author_email       = author_email,
-       classifiers        = classifiers,
-       description        = short_desc,
-       entry_points       = entry_points,
-       install_requires   = install_requires,
-       license            = license,
-       long_description   = long_description,
-       name               = modname,
-       packages           = find_packages(),
-       py_modules         = py_modules,
-       test_suite         = 'nose.collector',
-       url                = web,
-       tests_require      = ['nose>=1.0'],
-       version            = VERSION,
-       zip_safe           = zip_safe)
+setuptools.setup(
+    author=author,
+    author_email=author_email,
+    classifiers=classifiers,
+    description=short_desc,
+    entry_points=entry_points,
+    install_requires=install_requires,
+    license=license,
+    long_description=long_description,
+    long_description_content_type="text/x-rst",
+    name=modname,
+    packages=setuptools.find_packages(),
+    py_modules=py_modules,
+    test_suite="nose.collector",
+    url=web,
+    tests_require=["nose>=1.0"],
+    version=__version__,
+    zip_safe=zip_safe,
+)

test/.gitignore

@@ -1 +1,8 @@
+/.coverage
+/.python-version
 /nohup.out
+/pycdc
+/test_pycdc_tests.sh
+/test_uncompyle2.py
+/test_unpy33.py
+/test_unpy37.py

test/Makefile

@@ -1,5 +1,6 @@
 PHONY=check clean dist distclean test test-unit test-functional rmChangeLog clean_pyc nosetests \
-      check-bytecode-1  check-bytecode-1.3 check-bytecode-1.4 check-bytecode-1.5 \
+      check-bytecode-1.0 check-bytecode-1.1 check-bytecode-1.2 check-bytecode-1.3 \
+      check-bytecode-1 check-bytecode-1.4 check-bytecode-1.5 check-bytecode-1.6 \
       check-bytecode-2 check-bytecode-3 check-bytecode-3-short \
       check-bytecode-2.2 check-byteocde-2.3 check-bytecode-2.4 \
       check-short check-2.6 check-2.7 check-3.0 check-3.1 check-3.2 check-3.3 \
@@ -12,7 +13,7 @@ PHONY=check clean dist distclean test test-unit test-functional rmChangeLog clea
 GIT2CL ?= git2cl
 PYTHON ?= python
 
-PYTHON_VERSION = $(shell $(PYTHON) -V 2>&1 | cut -d ' ' -f 2 | cut -d'.' -f1,2)
+PYTHON_VERSION = $(shell $(PYTHON) -V 2>&1 | cut -d ' ' -f 2 | cut -d'.' -f1,2 | head -1)
 NATIVE_CHECK = check-$(PYTHON_VERSION)
 
 # Set COMPILE='--compile' to force compilation before check
@@ -21,8 +22,8 @@ COVER_DIR=../tmp/grammar-cover
 
 # Run short tests
 check-short:
-	@$(PYTHON) -V && PYTHON_VERSION=`$(PYTHON) -V 2>&1 | cut -d ' ' -f 2 | cut -d'.' -f1,2`; \
-	$(MAKE) check-bytecode-short
+	@$(PYTHON) -V && PYTHON_VERSION=`$(PYTHON) -V 2>&1 | cut -d ' ' -f 2 | cut -d'.' -f1,2` | head -1; \
+	$(MAKE) check-bytecode-${PYTHON_VERSION}
 
 # Run all tests
 check:
@@ -71,21 +72,30 @@ check-3.7: check-bytecode
 	$(PYTHON) test_pythonlib.py --bytecode-3.7-run --verify-run
 	$(PYTHON) test_pythonlib.py --bytecode-3.7 --syntax-verify $(COMPILE)
 
+check-pypy37: check-bytecode
+	$(PYTHON) test_pythonlib.py --bytecode-pypy37 --verify-run
+
 #: Run working tests from Python 3.8
 check-3.8: check-bytecode
 	$(PYTHON) test_pythonlib.py --bytecode-3.8-run --verify-run
 	$(PYTHON) test_pythonlib.py --bytecode-3.8 --syntax-verify $(COMPILE)
 
+check-3.9: check-bytecode
+	@echo "Note that we do not support decompiling Python 3.9 bytecode - no 3.9 tests run"
+
+check-3.10: check-bytecode
+	@echo "Note that we do not support decompiling Python 3.10 bytecode - no 3.10 tests run"
+
 # FIXME
-#: this is called when running under pypy3.5-5.8.0 or pypy2-5.6.0
-5.8 5.6:
+#: this is called when running under pypy3.5-5.8.0, pypy2-5.6.0, pypy3.6-7.3.0 or pypy3.8-7.3.7
+5.8 5.6 7.3:
 
 #: Check deparsing only, but from a different Python version
 check-disasm:
 	$(PYTHON) dis-compare.py
 
 #: Check deparsing bytecode 1.x only
-check-bytecode-1: check-bytecode-1.4 check-bytecode-1.5
+check-bytecode-1: check-bytecode-1.0 check-bytecode-1.1 check-bytecode-1.2 check-bytecode-1.3 check-bytecode-1.4 check-bytecode-1.5 check-bytecode-1.6
 
 #: Check deparsing bytecode 2.x only
 check-bytecode-2:
@@ -94,12 +104,19 @@ check-bytecode-2:
         --bytecode-2.5 --bytecode-2.6 --bytecode-2.7 --bytecode-pypy2.7
 
 #: Check deparsing bytecode 3.x only
+# check-bytecode-3:
+# 	$(PYTHON) test_pythonlib.py \
+#         --bytecode-3.0 \
+# 	--bytecode-3.1 --bytecode-3.2 --bytecode-3.3 \
+# 	--bytecode-3.4 --bytecode-3.5 --bytecode-3.6 \
+#         --bytecode-3.7  \
+# 	--bytecode-pypy3.2 --bytecode-pypy3.6 --bytecode-3.8
+
+# FIXME: Until we shaked out problems with xdis...
 check-bytecode-3:
-	$(PYTHON) test_pythonlib.py --bytecode-3.0 \
-	--bytecode-3.1 --bytecode-3.2 --bytecode-3.3 \
+	$(PYTHON) test_pythonlib.py \
 	--bytecode-3.4 --bytecode-3.5 --bytecode-3.6 \
-        --bytecode-3.7 --bytecode-3.8 \
-	--bytecode-pypy3.2
+        --bytecode-3.7  --bytecode-3.8
 
 #: Check deparsing on selected bytecode 3.x
 check-bytecode-3-short:
@@ -109,6 +126,7 @@ check-bytecode-3-short:
 #: Check deparsing bytecode on all Python 2 and Python 3 versions
 check-bytecode: check-bytecode-3
 	$(PYTHON) test_pythonlib.py \
+	--bytecode-1.0 --bytecode-1.1 --bytecode-1.2 \
 	--bytecode-1.3 --bytecode-1.4 --bytecode-1.5 \
         --bytecode-2.2 --bytecode-2.3 --bytecode-2.4 \
 	--bytecode-2.1 --bytecode-2.2 --bytecode-2.3 --bytecode-2.4 \
@@ -122,6 +140,18 @@ check-bytecode-short: check-bytecode-3-short
         --bytecode-2.6 --bytecode-2.7 --bytecode-pypy2.7
 
 
+#: Check deparsing bytecode 1.0 only
+check-bytecode-1.0:
+	$(PYTHON) test_pythonlib.py --bytecode-1.0
+
+#: Check deparsing bytecode 1.1 only
+check-bytecode-1.1:
+	$(PYTHON) test_pythonlib.py --bytecode-1.1
+
+#: Check deparsing bytecode 1.2 only
+check-bytecode-1.2:
+	$(PYTHON) test_pythonlib.py --bytecode-1.2
+
 #: Check deparsing bytecode 1.3 only
 check-bytecode-1.3:
 	$(PYTHON) test_pythonlib.py --bytecode-1.3
@@ -134,6 +164,10 @@ check-bytecode-1.4:
 check-bytecode-1.5:
 	$(PYTHON) test_pythonlib.py --bytecode-1.5
 
+#: Check deparsing bytecode 1.6 only
+check-bytecode-1.6:
+	$(PYTHON) test_pythonlib.py --bytecode-1.6
+
 #: Check deparsing Python 2.1
 check-bytecode-2.1:
 	$(PYTHON) test_pythonlib.py --bytecode-2.1
@@ -148,12 +182,24 @@ check-bytecode-2.3:
 
 #: Check deparsing Python 2.4
 check-bytecode-2.4:
+	$(PYTHON) test_pythonlib.py --bytecode-2.4-run --verify-run
 	$(PYTHON) test_pythonlib.py --bytecode-2.4
 
 #: Check deparsing Python 2.5
 check-bytecode-2.5:
+	$(PYTHON) test_pythonlib.py --bytecode-2.5-run --verify-run
 	$(PYTHON) test_pythonlib.py --bytecode-2.5
 
+#: Check deparsing Python 2.6
+check-bytecode-2.6:
+	$(PYTHON) test_pythonlib.py --bytecode-2.6-run --verify-run
+	$(PYTHON) test_pythonlib.py --bytecode-2.6 --syntax-verify
+
+#: Check deparsing Python 2.7
+check-bytecode-2.7:
+	$(PYTHON) test_pythonlib.py --bytecode-2.7-run --verify-run
+	$(PYTHON) test_pythonlib.py --bytecode-2.7 --syntax-verify
+
 #: Get grammar coverage for Python 2.4
 grammar-coverage-2.4:
 	-rm $(COVER_DIR)/spark-grammar-24.cover
@@ -226,16 +272,6 @@ grammar-coverage-3.7:
 	rm $(COVER_DIR)/spark-grammar-3.7.cover || /bin/true
 	SPARK_PARSER_COVERAGE=$(COVER_DIR)/spark-grammar-3.7.cover $(PYTHON) test_pyenvlib.py --3.7.3 --max=500
 
-#: Check deparsing Python 2.6
-check-bytecode-2.6:
-	$(PYTHON) test_pythonlib.py --bytecode-2.6-run --verify-run
-	$(PYTHON) test_pythonlib.py --bytecode-2.6 --syntax-verify
-
-#: Check deparsing Python 2.7
-check-bytecode-2.7:
-	$(PYTHON) test_pythonlib.py --bytecode-2.7-run --verify-run
-	$(PYTHON) test_pythonlib.py --bytecode-2.7 --syntax-verify
-
 #: Check deparsing Python 3.0
 check-bytecode-3.0:
 	$(PYTHON) test_pythonlib.py --bytecode-3.0-run --verify-run
@@ -314,11 +350,24 @@ pypy-2.7 5.0 5.3 6.0:
 pypy-3.2 2.4:
 	$(PYTHON) test_pythonlib.py --bytecode-pypy3.2 --verify
 
-#: PyPy 5.0.x with Python 3.6 ...
+#: PyPy 5.0.x with Python 3.6.1 ...
+check-bytecode-pypy3.6: 7.1
 7.1:
+	$(PYTHON) test_pythonlib.py --bytecode-pypy3.6-run --verify-run
 	$(PYTHON) test_pythonlib.py --bytecode-pypy3.6 --verify
 
+#: PyPy 5.0.x with Python 3.6.9
+check-bytecode-pypy3.6: 7.2 7.3
 
+7.2:
+	$(PYTHON) test_pythonlib.py --bytecode-pypy3.6-run --verify-run
+	$(PYTHON) test_pythonlib.py --bytecode-pypy3.6 --verify
+
+7.3:
+	$(PYTHON) test_pythonlib.py --bytecode-pypy3.7 --verify-run
+
+# Pyston 2.3
+2.3: check-3.8
 
 clean: clean-py-dis clean-dis clean-unverified
 

test/add-test.py

@@ -1,23 +1,49 @@
 #!/usr/bin/env python
-""" Trivial helper program to bytecompile and run an uncompile
+""" Trivial helper program to byte compile and uncompile the bytecode file.
 """
 import os, sys, py_compile
-assert len(sys.argv) >= 2
+from xdis.version_info import version_tuple_to_str, PYTHON_VERSION_TRIPLE
+
+if len(sys.argv) < 2:
+    print("Usage: add-test.py [--run] *python-source*... [optimize-level]")
+    sys.exit(1)
+
+assert 2 <= len(sys.argv) <= 4
 version = sys.version[0:3]
-if sys.argv[1] == '--run':
-    suffix = '_run'
+vers = sys.version_info[:2]
+if sys.argv[1] in ("--run", "-r"):
+    suffix = "_run"
+    assert len(sys.argv) >= 3
     py_source = sys.argv[2:]
+    i = 2
 else:
-    suffix = ''
+    suffix = ""
     py_source = sys.argv[1:]
+    i = 1
+try:
+    optimize = int(sys.argv[-1])
+    assert sys.argv >= i + 2
+    py_source = sys.argv[i:-1]
+    i = 2
+
+except:
+    optimize = 2
 
 for path in py_source:
     short = os.path.basename(path)
-    if hasattr(sys, 'pypy_version_info'):
-        cfile =  "bytecode_pypy%s%s/%s" % (version, suffix, short) + 'c'
+    if short.endswith(".py"):
+        short = short[: -len(".py")]
+    if hasattr(sys, "pypy_version_info"):
+        version = version_tuple_to_str(end=2, delimiter="")
+        bytecode = "bytecode_pypy%s%s/%spy%s.pyc" % (version, suffix, short, version)
     else:
-        cfile =  "bytecode_%s%s/%s" % (version, suffix, short) + 'c'
-    print("byte-compiling %s to %s" % (path, cfile))
-    py_compile.compile(path, cfile)
-    if isinstance(version, str) or version >= (2, 6, 0):
-        os.system("../bin/uncompyle6 -a -t %s" % cfile)
+        version = version_tuple_to_str(end=2)
+        bytecode = "bytecode_%s%s/%s.pyc" % (version, suffix, short)
+
+    print("byte-compiling %s to %s" % (path, bytecode))
+    if PYTHON_VERSION_TRIPLE >= (3, 2):
+        py_compile.compile(path, bytecode, optimize=optimize)
+    else:
+        py_compile.compile(path, bytecode)
+    if PYTHON_VERSION_TRIPLE >= (2, 6):
+        os.system("../bin/uncompyle6 -a -t %s" % bytecode)