Reinstate pos_args in CALL_METHOD

The next release of xdis will no longer support self.opc.version (a
float value which doesn't work in the presense of 3.10 and above)

.circleci/config.yml

@@ -0,0 +1,77 @@
+version: 2
+filters:
+  branches:
+    only: master
+jobs:
+  build:
+    working_directory: ~/rocky/python-uncompyle6
+    parallelism: 1
+    shell: /bin/bash --login
+    # CircleCI 2.0 does not support environment variables that refer to each other the same way as 1.0 did.
+    # If any of these refer to each other, rewrite them so that they don't or see https://circleci.com/docs/2.0/env-vars/#interpolating-environment-variables-to-set-other-environment-variables .
+    environment:
+      CIRCLE_ARTIFACTS: /tmp/circleci-artifacts
+      CIRCLE_TEST_REPORTS: /tmp/circleci-test-results
+      COMPILE: --compile
+    # To see the list of pre-built images that CircleCI provides for most common languages see
+    # https://circleci.com/docs/2.0/circleci-images/
+    docker:
+      - image: circleci/python:3.6.9
+    steps:
+    # Machine Setup
+    #   If you break your build into multiple jobs with workflows, you will probably want to do the parts of this that are relevant in each
+    # The following `checkout` command checks out your code to your working directory. In 1.0 we did this implicitly. In 2.0 you can choose where in the course of a job your code should be checked out.
+    - checkout
+    # Prepare for artifact and test results  collection equivalent to how it was done on 1.0.
+    # In many cases you can simplify this from what is generated here.
+    # 'See docs on artifact collection here https://circleci.com/docs/2.0/artifacts/'
+    - run: mkdir -p $CIRCLE_ARTIFACTS $CIRCLE_TEST_REPORTS
+    # This is based on your 1.0 configuration file or project settings
+    - run:
+        working_directory: ~/rocky/python-uncompyle6
+        command: pip install --user virtualenv && pip install --user nose && pip install --user pep8
+    # Dependencies
+    #   This would typically go in either a build or a build-and-test job when using workflows
+    # Restore the dependency cache
+    - restore_cache:
+        keys:
+          - v2-dependencies-{{ .Branch }}-
+          # fallback to using the latest cache if no exact match is found
+          - v2-dependencies-
+
+    - run:
+        command: | # Use pip to install dependengcies
+          pip install --user --upgrade setuptools
+          # Until the next release
+          pip install git+https://github.com/rocky/python-xdis#egg=xdis
+          pip install --user -e .
+          pip install --user -r requirements-dev.txt
+
+    # Save dependency cache
+    - save_cache:
+        key: v2-dependencies-{{ .Branch }}-{{ epoch }}
+        paths:
+        # This is a broad list of cache paths to include many possible development environments
+        # You can probably delete some of these entries
+        - vendor/bundle
+        - ~/virtualenvs
+        - ~/.m2
+        - ~/.ivy2
+        - ~/.bundle
+        - ~/.cache/bower
+
+    # Test
+    #   This would typically be a build job when using workflows, possibly combined with build
+    # This is based on your 1.0 configuration file or project settings
+    - run: sudo python ./setup.py develop && make check-3.6
+    - run: cd ./test/stdlib && bash ./runtests.sh 'test_[p-z]*.py'
+    # Teardown
+    #   If you break your build into multiple jobs with workflows, you will probably want to do the parts of this that are relevant in each
+    # Save test results
+    - store_test_results:
+        path: /tmp/circleci-test-results
+    # Save artifacts
+    - store_artifacts:
+        path: /tmp/circleci-artifacts
+    - store_artifacts:
+        path: /tmp/circleci-test-results

.editorconfig

@@ -0,0 +1,28 @@
+# THis is an EditorConfig file
+# https://EditorConfig.org
+
+root = true
+
+[*]
+end_of_line = lf
+insert_final_newline = true
+charset = utf-8
+indent_style = tab
+indent_size = 4
+insert_final_newline = true
+
+[*.yml]
+indent_style = space
+indent_size = 2
+end_of_line = lf
+insert_final_newline = true
+
+[*.py]
+indent_style = space
+indent_size = 4
+end_of_line = lf
+insert_final_newline = true
+
+# Tab indentation (no size specified)
+[Makefile]
+indent_style = tab

.github/FUNDING.yml

@@ -0,0 +1,12 @@
+# These are supported funding model platforms
+
+github: [rocky]
+patreon: # Replace with a single Patreon username
+open_collective: # Replace with a single Open Collective username
+ko_fi: # Replace with a single Ko-fi username
+tidelift: # Replace with a single Tidelift platform-name/package-name e.g., npm/babel
+community_bridge: # Replace with a single Community Bridge project-name e.g., cloud-foundry
+liberapay: # Replace with a single Liberapay username
+issuehunt: # Replace with a single IssueHunt username
+otechie: # Replace with a single Otechie username
+custom: # Replace with up to 4 custom sponsorship URLs e.g., ['link1', 'link2']

.github/ISSUE_TEMPLATE/bug-report.md

@@ -0,0 +1,129 @@
+---
+name: Bug report
+about: Tell us about uncompyle6 bugs
+
+---
+
+<!-- __Note:__ If you are using this program to do something illegal - don't.
+The issue may be flagged to make it easier for those looking for illegal activity.
+
+Bugs are not for asking questions about a problem you
+are trying to solve that involve the use of uncompyle6 along the way,
+although I may be more tolerant of this if you sponsor the project.
+
+Bugs are also not for general or novice kind help on how to install
+this Python program in your environment in the way you would like to
+have it set up, or how to interpret a Python traceback e.g. that winds
+up saying Python X.Y.Z is not supported.
+
+For these kinds of things, you will save yourself time by asking
+instead on forums like StackOverflow that are geared to helping people
+for such general or novice kinds questions and tasks. And unless you
+are a sponsor of the project, if your question seems to be of this
+category, the issue may just be closed.
+
+Also, the unless you are a sponsor of the project, it may take a
+while, maybe a week or so, before the bug report is noticed, let alone
+acted upon.
+
+To set expectations, some legitimate bugs can take years to fix, but
+they eventually do get fixed.
+
+Funding the project was added to partially address the problem that there are
+lots of people seeking help and reporting bugs, but few people who are
+willing or capable of providing help or fixing bugs.
+
+Tasks or the kinds of things others can do, but you can't do or don't
+want to do yourself are typically the kind of thing that you pay
+someone to do, especially when you are the primary beneficiary of the
+work, or the task is complex, long, or tedious. If your code is over
+30 lines long, it fits into this category.
+
+
+See also https://github.com/rocky/python-uncomp[yle6/blob/master/HOW-TO-REPORT-A-BUG.md ?
+-->
+
+<!--
+Please remove any of the optional sections if they are not applicable.
+
+Prerequisites/Caveats
+
+* Make sure the bytecode you have can be disassembled with a
+  disassembler and produces valid results.
+* Don't put bytecode and corresponding source code on any service that
+  requires registration to download.
+* When you open a bug report there is no privacy. If you need privacy, then
+  contact me by email and explain who you are and the need for privacy.
+  But be mindful that you may be asked to sponsor the project for the
+  personal and private help that you are requesting.
+* If the legitimacy of the activity is deemed suspicious, I may flag it as suspicious,
+  making the issue even more easy to detect.
+
+Bug reports that violate the above may be discarded.
+
+-->
+
+## Description
+
+<!-- Please add a clear and concise description of the bug. Try to narrow the problem down to the smallest that exhibits the bug.-->
+
+## How to Reproduce
+
+<!-- Please show both the *input* you gave and the
+output you got in describing how to reproduce the bug:
+
+or give a complete console log with input and output
+
+```console
+$ uncompyle6 <command-line-options>
+...
+$
+```
+
+Provide links to the Python bytecode. For example, you can create a
+gist with the information. If you have the correct source code, you
+can add that too.
+
+-->
+
+## Output Given
+
+<!--
+Please include not just the error message but all output leading to the message which includes echoing input and messages up to the error.
+For a command-line environment include command invocation and all the output produced.
+
+If this is too long, then try narrowing the problem to something short.
+-->
+
+
+## Expected behavior
+
+<!-- Add a clear and concise description of what you expected to happen. -->
+
+## Environment
+
+<!-- _This section sometimes is optional but helpful to us._
+
+Please modify for your setup
+
+- Uncompyle6 version: output from  `uncompyle6 --version` or `pip show uncompyle6`
+- Python version for the version of Python the byte-compiled the file: `python -c "import sys; print(sys.version)"` where `python` is the correct CPython or PyPy binary.
+- OS and Version: [e.g. Ubuntu bionic]
+
+-->
+
+## Workarounds
+
+<!-- If there is a workaround for the problem, describe that here. -->
+
+## Priority
+
+<!-- If this is blocking some important activity let us know what activity it blocks. -->
+
+## Additional Context
+
+<!-- _This section is optional._
+
+Add any other context about the problem here or special environment setup.
+
+-->

.github/ISSUE_TEMPLATE/feature-request.md

@@ -0,0 +1,22 @@
+---
+name: Feature Request
+about: Tell us about a new feature that you would like to see in uncompyle6
+
+---
+
+## Description
+
+<!-- Add a short description of the feature. This might
+include same input and output. -->
+
+## Background
+
+<!-- Add any additional background for the
+feature, for example: user scenarios, or the value of the feature. -->
+
+## Tests
+<!-- _This section is optional._
+
+Add text with suggestions on how to test the feature,
+if it is not obvious.
+-->

.github/workflows/osx.yml

@@ -0,0 +1,31 @@
+name: uncompyle6 (osx)
+
+on:
+  push:
+    branches: [ master ]
+  pull_request:
+    branches: [ master ]
+
+jobs:
+  build:
+    runs-on: macos-latest
+    strategy:
+      matrix:
+        os: [macOS]
+        python-version: [3.7, 3.8]
+    steps:
+    - uses: actions/checkout@v2
+    - name: Set up Python ${{ matrix.python-version }}
+      uses: actions/setup-python@v2
+      with:
+        python-version: ${{ matrix.python-version }}
+    - name: Install dependencies
+      run: |
+        python -m pip install --upgrade pip
+        # Until the next xdis release
+        pip install git+https://github.com/rocky/python-xdis#egg=xdis
+        pip install -e .
+        pip install -r requirements-dev.txt
+    - name: Test uncompyle6
+      run: |
+        make check

.github/workflows/ubuntu.yml

@@ -0,0 +1,30 @@
+name: uncompyle6 (ubuntu)
+
+on:
+  push:
+    branches: [ master ]
+  pull_request:
+    branches: [ master ]
+
+jobs:
+  build:
+    runs-on: ubuntu-latest
+    strategy:
+      matrix:
+        python-version: [3.7, 3.8]
+    steps:
+    - uses: actions/checkout@v2
+    - name: Set up Python ${{ matrix.python-version }}
+      uses: actions/setup-python@v2
+      with:
+        python-version: ${{ matrix.python-version }}
+    - name: Install dependencies
+      run: |
+        python -m pip install --upgrade pip
+        # Until the next xdis release
+        pip install git+https://github.com/rocky/python-xdis#egg=xdis
+        pip install -e .
+        pip install -r requirements-dev.txt
+    - name: Test uncompyle6
+      run: |
+        make check

.github/workflows/windows.yml

@@ -0,0 +1,31 @@
+name: uncompyle6 (windows)
+
+on:
+  push:
+    branches: [ master ]
+  pull_request:
+    branches: [ master ]
+
+jobs:
+  build:
+    runs-on: macos-latest
+    strategy:
+      matrix:
+        os: [windows]
+        python-version: [3.7, 3.8]
+    steps:
+    - uses: actions/checkout@v2
+    - name: Set up Python ${{ matrix.python-version }}
+      uses: actions/setup-python@v2
+      with:
+        python-version: ${{ matrix.python-version }}
+    - name: Install dependencies
+      run: |
+        python -m pip install --upgrade pip
+        # Until the next xdis release
+        pip install git+https://github.com/rocky/python-xdis#egg=xdis
+        pip install -e .
+        pip install -r requirements-dev.txt
+    - name: Test uncompyle6
+      run: |
+        make check

.gitignore

@@ -6,9 +6,11 @@
 /.eggs
 /.hypothesis
 /.idea
+/.mypy_cache
 /.pytest_cache
 /.python-version
 /.tox
+.mypy_cache
 /.venv*
 /README
 /__pkginfo__.pyc

.isort.cfg

@@ -0,0 +1,11 @@
+[settings]
+multi_line_output = 3
+include_trailing_comma = True
+force_grid_wrap = 0
+use_parentheses = True
+line_length = 88
+known_crunch = cr, zz9d, zz9lib, pycrunch, silhouette
+sections = FUTURE,STDLIB,THIRDPARTY,FIRSTPARTY,CRUNCH,LOCALFOLDER
+default_section = THIRDPARTY
+combine_as_imports = 1
+profile = black

.travis.yml

@@ -1,17 +1,20 @@
 language: python
 
-sudo: false
-
 python:
-  - '3.5'
-  - '2.7'
-  - '2.6'
-  - '3.3'
-  - '3.4'
-  - '3.2'
+#  - '3.5'
+#  - '2.7'
+#  - '3.4'
   - '3.6'
+  - '3.8'
+
+matrix:
+  include:
+    - python: '3.7'
+      dist: xenial  # required for Python >= 3.7 (travis-ci/travis-ci#9069)
 
 install:
+# Remove the next line when xdis 6.0.0 is released
+# - pip install git://github.com/rocky/python-xdis.git#egg=xdis
 - pip install -e .
 - pip install -r requirements-dev.txt
 

HISTORY.md

@@ -1,213 +1,93 @@
-This project has history of over 18 years spanning back to Python 1.5
+# Introduction
 
-There have been a number of people who have worked on this. I am awed
-by the amount of work, number of people who have contributed to this,
-and the cleverness in the code.
+This project started around 1999 spanning back to Python 1.5
 
-The below is an annotated history from talking to participants
-involved and my reading of the code and sources cited.
+In the interest of shortening what is written here, I am going to start where we left off where [decompyle 2.4's history](https://github.com/rocky/decompile-2.4/blob/master/HISTORY.md) ends.
 
-In 1998, John Aycock first wrote a grammar parser in Python,
-eventually called SPARK, that was usable inside a Python program. This
-code was described in the
-[7th International Python Conference](http://legacy.python.org/workshops/1998-11/proceedings/papers/aycock-little/aycock-little.html). That
-paper doesn't talk about decompilation, nor did John have that in mind
-at that time. It does mention that a full parser for Python (rather
-than the simple languages in the paper) was being considered.
+For the earlier history up to 2006 and the code up until Python 2.4, which I find interesting, look at that link.
 
-[This](http://pages.cpsc.ucalgary.ca/~aycock/spark/content.html#contributors)
-contains a of people acknowledged in developing SPARK. What's amazing
-about this code is that it is reasonably fast and has survived up to
-Python 3 with relatively little change. This work was done in
-conjunction with his Ph.D Thesis. This was finished around 2001. In
-working on his thesis, John realized SPARK could be used to deparse
-Python bytecode. In the fall of 1999, he started writing the Python
-program, "decompyle", to do this.
+Sometime around 2014 was the dawn of ["uncompyle" and PyPI](https://pypi.python.org/pypi/uncompyle/1.1) — the era of
+public version control. Dan Pascu's code although not public used [darcs](http://darcs.net/) for version control. I converted the darcs to to git and put this at [decompyle-2.4](https://github.com/rocky/decompile-2.4).
 
-To help with control structure deparsing the instruction sequence was
-augmented with pseudo instruction COME_FROM. This code introduced
-another clever idea: using table-driven semantics routines, using
-format specifiers.
+# uncompyle, unpyc
 
-The last mention of a release of SPARK from John is around 2002. As
-released, although the Earley Algorithm parser was in good shape, this
-code was woefully lacking as serious Python deparser.
+In contrast to _decompyle_ that went up to Python 2.4, _uncompyle_, at least in its final versions, runs only on Python 2.7. However it accepts bytecode back to Python 2.5. Thomas Grainger is the package owner of this, although Hartmut is still listed as the author.
 
-In the fall of 2000, Hartmut Goebel
-[took over maintaining the code](https://groups.google.com/forum/#!searchin/comp.lang.python/hartmut$20goebel/comp.lang.python/35s3mp4-nuY/UZALti6ujnQJ). The
-first subsequent public release announcement that I can find is
-["decompyle - A byte-code-decompiler version 2.2 beta 1"](https://mail.python.org/pipermail/python-announce-list/2002-February/001272.html).
+The project exists not only on [github](https://github.com/gstarnberger/uncompyle) but also on
+[bitbucket](https://bitbucket.org/gstarnberger/uncompyle) and later the defunct [google
+code](https://code.google.com/archive/p/unpyc/) under the name _unpyc_. The git/svn history goes back to 2009. Somewhere in there the name was changed from "decompyle" to "unpyc" by Keknehv, and then to "uncompyle" by Guenther Starnberger.
 
-From the CHANGES file found in
-[the tarball for that release](http://old-releases.ubuntu.com/ubuntu/pool/universe/d/decompyle2.2/decompyle2.2_2.2beta1.orig.tar.gz),
-it appears that Hartmut did most of the work to get this code to
-accept the full Python language. He added precedence to the table
-specifiers, support for multiple versions of Python, the
-pretty-printing of docstrings, lists, and hashes. He also wrote test and verification routines of
-deparsed bytecode, and used this in an extensive set of tests that he also wrote. He says he could verify against the
-entire Python library. However I have subsequently found small and relatively obscure bugs in the decompilation code.
+The name Thomas Grainger isn't found in (m)any of the commits in the several years of active development. First Keknehv worked on this up to Python 2.5 or so while acceping Python bytecode back to 2.0 or so. Then "hamled" made a few commits earler on, while Eike Siewertsen made a few commits later on. But mostly "wibiti", and Guenther Starnberger got the code to where uncompyle2 was around 2012.
 
-decompyle2.2 was packaged for Debian (sarge) by
-[Ben Burton around 2002](https://packages.qa.debian.org/d/decompyle.html). As
-it worked on Python 2.2 only long after Python 2.3 and 2.4 were in
-widespread use, it was removed.
+While John Aycock and Hartmut Goebel were well versed in compiler technology, those that have come afterwards don't seem to have been as facile in it.  Furthermore, documentation or guidance on how the decompiler code worked, comparison to a conventional compiler pipeline, how to add new constructs, or debug grammars was weak. Some of the grammar tracing and error reporting was a bit weak as well.
 
-[Crazy Compilers](http://www.crazy-compilers.com/decompyle/) offers a
-byte-code decompiler service for versions of Python up to 2.6. As
-someone who worked in compilers, it is tough to make a living by
-working on compilers. (For example, based on
-[John Aycock's recent papers](http://pages.cpsc.ucalgary.ca/~aycock/)
-it doesn't look like he's done anything compiler-wise since SPARK). So
-I hope people will use the crazy-compilers service. I wish them the
-success that his good work deserves.
+Given this, perhaps it is not surprising that subsequent changes tended to shy away from using the built-in compiler technology mechanisms and addressed problems and extensions by some other means.
 
-Dan Pascu did a bit of work from late 2004 to early 2006 to get this
-code to handle first Python 2.3 and then 2.4 bytecodes. Because of
-jump optimization introduced in the CPython bytecode compiler at that
-time, various JUMP instructions were classified to assist parsing For
-example, due to the way that code generation and line number table
-work, jump instructions to an earlier offset must be looping jumps,
-such as those found in a "continue" statement; "COME FROM"
-instructions were reintroduced.  See
-[RELEASE-2.4-CHANGELOG.txt](https://github.com/rocky/python-uncompyle6/blob/master/DECOMPYLE-2.4-CHANGELOG.txt)
-for more details here. There wasn't a public release of RELEASE-2.4
-and bytecodes other than Python 2.4 weren't supported. Dan says the
-Python 2.3 version could verify the entire Python library. But given
-subsequent bugs found like simply recognizing complex-number constants
-in bytecode, decompilation wasn't perfect.
+Specifically, in `uncompyle`, decompilation of python bytecode 2.5 & 2.6 is done by transforming the byte code into a pseudo-2.7 Python bytecode and is based on code from Eloi Vanderbeken. A bit of this could have been easily added by modifying grammar rules.
 
-Next we get to ["uncompyle" and
-PyPI](https://pypi.python.org/pypi/uncompyle/1.1) and the era of
-public version control. (Dan's code although not public used
-[darcs](http://darcs.net/) for version control.)
 
-In contrast to _decompyle_, _uncompyle_ at least in its final versions,
-runs only on Python 2.7. However it accepts bytecode back to Python
-2.5. Thomas Grainger is the package owner of this, although Hartmut is
-still listed as the author.
+# uncompyle2, uncompyle3, uncompyle6
 
-The project exists not only on
-[github](https://github.com/gstarnberger/uncompyle) but also on
-[bitbucket](https://bitbucket.org/gstarnberger/uncompyle) and later
-the defunct [google
-code](https://code.google.com/archive/p/unpyc/). The git/svn history
-goes back to 2009. Somewhere in there the name was changed from
-"decompyle" to "unpyc" by Keknehv, and then to "uncompyle" by Guenther Starnberger.
+`Uncompyle6`, which I started in 2015, owes its existence to the fork of [uncompyle2](https://github.com/Mysterie/uncompyle2) by Myst herie (Mysterie) whose first commit picks up at 2012. I chose this since it seemed to have been at that time the most actively, if briefly, worked on. Also starting around 2012 is Dark Fenx's [uncompyle3](https://github.com/DarkFenX/uncompyle3) which I used for inspiration for Python3 support.
 
-The name Thomas Grainger isn't found in (m)any of the commits in the
-several years of active development. First Keknehv worked on this up
-to Python 2.5 or so while acceping Python bytecode back to 2.0 or
-so. Then hamled made a few commits earler on, while Eike Siewertsen
-made a few commits later on. But mostly wibiti, and Guenther
-Starnberger got the code to where uncompyle2 was around 2012.
-
-While John Aycock and Hartmut Goebel were well versed in compiler
-technology, those that have come afterwards don't seem to have been as
-facile in it.  Furthermore, documentation or guidance on how the
-decompiler code worked, comparison to a conventional compiler
-pipeline, how to add new constructs, or debug grammars was weak. Some
-of the grammar tracing and error reporting was a bit weak as well.
-
-Given this, perhaps it is not surprising that subsequent changes
-tended to shy away from using the built-in compiler technology
-mechanisms and addressed problems and extensions by some other means.
-
-Specifically, in `uncompyle`, decompilation of python bytecode 2.5 &
-2.6 is done by transforming the byte code into a pseudo-2.7 Python
-bytecode and is based on code from Eloi Vanderbeken. A bit of this
-could have bene easily added by modifying grammar rules.
-
-This project, `uncompyle6`, abandons that approach for various
-reasons. Having a grammar per Python version is much cleaner and it
-scales indefinitely. That said, we don't have entire copies of the
-grammar, but work off of differences from some neighboring version.
-
-Should there be a desire to rebase or start a new base version to work
-off of, say for some future Python version, that can be done by
-dumping a grammar for a specific version after it has been loaded
-incrementally. You can get a full dump of the grammar by profiling the
-grammar on a large body of Python source code.
-
-Another problem with pseudo-2.7 bytecode is that that we need offsets
-in fragment deparsing to be exactly the same as the bytecode; the
-transformation process can remove instructions.  _Adding_ instructions
-with psuedo offsets is however okay.
-
-`Uncompyle6` however owes its existence to the fork of `uncompyle2` by
-Myst herie (Mysterie) whose first commit picks up at
-2012. I chose this since it seemed to have been at that time the most
-actively, if briefly, worked on. Also starting around 2012 is Dark
-Fenx's uncompyle3 which I used for inspiration for Python3 support.
-
-I started working on this late 2015, mostly to add fragment support.
-In that, I decided to make this runnable on Python 3.2+ and Python 2.6+
-while, handling Python bytecodes from Python versions 2.5+ and
-3.2+. In doing so, it has been expedient to separate this into three
-projects:
+I started working on this late 2015, mostly to add fragment support. In that, I decided to make this runnable on Python 3.2+ and Python 2.6+ while handling Python bytecodes from Python versions 2.5+ and
+3.2+. In doing so, it was expedient to separate this into three projects:
 
 * marshaling/unmarshaling, bytecode loading and disassembly ([xdis](https://pypi.python.org/pypi/xdis)),
 * parsing and tree building ([spark_parser](https://pypi.python.org/pypi/spark_parser)),
 * this project - grammar and semantic actions for decompiling
   ([uncompyle6](https://pypi.python.org/pypi/uncompyle6)).
 
+ `uncompyle6`, abandons the idea found in some 2.7 version of `uncompyle` that support Python 2.6 and 2.5 by trying to rewite opcodes at the bytecode level.
 
-Over the many years, code styles and Python features have
-changed. However brilliant the code was and still is, it hasn't really
-had a single public active maintainer. And there have been many forks
-of the code.  I have spent a great deal of time trying to organize and
-modularize the code so that it can handle more Python versions more
-gracefully (with still only moderate success).
+Having a grammar per Python version is simpler to maintain, cleaner and it scales indefinitely.
 
-That it has been in need of an overhaul has been recognized by the
-Hartmut a decade an a half ago:
+Over the many years, code styles and Python features have changed. However brilliant the code was and still is, it hasn't really had a single public active maintainer. And there have been many forks of the code.
+
+That this code has been in need of an overhaul has been recognized by the Hartmut more than two decades ago.
 
 [decompyle/uncompile__init__.py](https://github.com/gstarnberger/uncompyle/blob/master/uncompyle/__init__.py#L25-L26)
 
     NB. This is not a masterpiece of software, but became more like a hack.
     Probably a complete rewrite would be sensefull. hG/2000-12-27
 
-This project deparses using an Earley-algorithm parse with lots of
-massaging of tokens and the grammar in the scanner
-phase. Earley-algorithm parsers are context free and tend to be linear
-if the grammar is LR or left recursive. There is a technique for
-improving LL right recursion, but our parser doesn't have that yet.
+In 2021, I created three git branches in order to allow the decompiler to run on a wide variety of Python versions from 2.4 up to 3.10. (Note this doesn't mean we decompile these versions. In fact we decompile starting from Python 1.0 up to Python 3.8 and no later than that.)
 
-Another approach to decompiling, and one that doesn't use grammars is
-to do something like simulate execution symbolically and build
-expression trees off of stack results. Control flow in that approach
-still needs to be handled somewhat ad hoc.  The two important projects
-that work this way are [unpyc3](https://code.google.com/p/unpyc3/) and
-most especially [pycdc](https://github.com/zrax/pycdc) The latter
-project is largely by Michael Hansen and Darryl Pogue. If they
-supported getting source-code fragments, did a better job in
-supporting Python more fully, and had a way I could call it from
-Python, I'd probably would have ditched this and used that. The code
-runs blindingly fast and spans all versions of Python, although more
-recently Python 3 support has been lagging. The code is impressive for
-its smallness given that it covers many versions of Python. However, I
-think it has reached a scalability issue, same as all the other
-efforts. To handle Python versions more accurately, I think that code
-base will need to have a lot more code specially which specializes for
-Python versions. And then it will run into a modularity problem.
+Using the separate git branches allows me to continually improve the coding style and add feature support while still supporting older Pythons. Supporting older Pythons is nice (but not strictly necessary) when you want to debug decompilation on older Pythons.
 
-Tests for the project have been, or are being, culled from all of the
-projects mentioned. Quite a few have been added to improve grammar
-coverage and to address the numerous bugs that have been encountered.
+I have spent a great deal of time trying to organize, modularize and even modernize the code so that it can handle more Python versions more gracefully (with still only moderate success).
 
-If you think, as I am sure will happen in the future, "hey, I can just
-write a decompiler from scratch and not have to deal with all all of
-the complexity here", think again. What is likely to happen is that
-you'll get at best a 90% solution working for a single Python release
-that will be obsolete in about a year, and more obsolete each
-subsequent year.  Writing a decompiler for Python gets harder as it
-Python progresses, so writing one for Python 3.7 isn't as easy as it
-was for Python 2.2.  That said, if you still feel you want to write a
-single version decompiler, look at the test cases in this project and
-talk to me. I may have some ideas.
+Tests for the project have been, or are being, culled from all of the projects mentioned above or below. Quite a few have been added to improve grammar coverage and to address the numerous bugs that have been encountered.
 
 
-For a little bit of the history of changes to the Earley-algorithm parser,
-see the file [NEW-FEATURES.rst](https://github.com/rocky/python-spark/blob/master/NEW-FEATURES.rst) in the [python-spark github repository](https://github.com/rocky/python-spark).
+# unpyc3 and pydc
 
-NB. If you find mistakes, want corrections, or want your name added
-(or removed), please contact me.
+Another approach to decompiling, and one that doesn't use grammars is to do something like simulate execution symbolically and build expression trees off of stack results. Control flow in that approach
+still needs to be handled somewhat ad hoc.  The two important projects that work this way are [unpyc3](https://code.google.com/p/unpyc3/) and most especially [pycdc](https://github.com/zrax/pycdc) The latter
+project is largely by Michael Hansen and Darryl Pogue. If they supported getting source-code fragments, did a better job in supporting Python more fully, and had a way I could call it from Python, I'd probably would have ditched this and used that. The code runs blindingly fast and spans all versions of Python, although more recently Python 3 support has been lagging. The code is impressive for its smallness given that it covers many versions of Python. However, I think it has reached a scalability issue, same as all the other efforts. To handle Python versions more accurately, I think that code base will need to have a lot more code specially which specializes for Python versions. And then it will run into a modularity problem.
+
+# So you want to write a decompiler for Python?
+
+If you think, as I am sure will happen in the future, "hey, I can just write a decompiler from scratch and not have to deal with all all of the complexity in uncompyle6", think again. What is likely to happen is that you'll get at best a 90% solution working for a single Python release that will be obsolete in about a year, and more obsolete each subsequent year.
+
+Writing a decompiler for Python gets harder as it Python progresses. Writing decompiler for Python 3.7 isn't as easy as it was for Python 2.2. For one thing, now that Python has a well-established AST, that opens another interface by which code can be improved.
+
+In Python 3.10 I am seeing (for the first time?) bytecode getting moved around so that it is no longer the case that line numbers have to be strictly increasing as bytecode offsets increase. And I am seeing dead code appear as well.
+
+That said, if you still feel you want to write a single version decompiler, look at the test cases in this project and talk to me. I may have some ideas that I haven't made public yet. See also what I've wrtten about the on how this code works and on [decompilation in dynamic runtime languages](http://rocky.github.io/Deparsing-Paper.pdf) in general.
+
+
+
+# Earley Algorithm Parser
+
+This project deparses using an Earley-algorithm parse. But in order to do this accurately, the process of tokenization is a bit more involved in the scanner. We don't just disassemble bytecode and use the opcode name. That aspect hasn't changed from the very first decompilers. However understanding _what_ information needs to be made explicit and what pseudo instructions to add that accomplish this has taken some time to understand.
+
+Earley-algorithm parsers have gotten negative press, most notably by the dragon book. Having used this a bit, I am convinced having a system that handles ambiguous grammars is the right thing to do and matches the problem well. Iin practice the speed of the parser isn't a problem when one understand what's up. And this has taken a little while to understand.
+Earley-algorim parsers for context free languages or languages that are to a large extent context free and tend to be linear and the grammar stears towards left recursive rules. There is a technique for improving LL right recursion, but our parser doesn't have that yet.
+
+The [decompiling paper](http://rocky.github.io/Deparsing-Paper.pdf) discusses these aspects in a more detail.
+
+
+For a little bit of the history of changes to the Earley-algorithm parser, see the file [NEW-FEATURES.rst](https://github.com/rocky/python-spark/blob/master/NEW-FEATURES.rst) in the [python-spark github repository](https://github.com/rocky/python-spark).
+
+NB. If you find mistakes, want corrections, or want your name added (or removed), please contact me.

HOW-TO-REPORT-A-BUG.md

@@ -1,53 +1,74 @@
-# How to report a Bug
+<!-- markdown-toc start - Don't edit this section. Run M-x markdown-toc-refresh-toc -->
+**Table of Contents**
 
-## The difficulty of the problem
+- [Ethics](#ethics)
+- [The importance of your bug report](#the-importance-of-your-bug-report)
+- [The difficulty of the problem and your bug](#the-difficulty-of-the-problem-and-your-bug)
+- [Is it really a bug?](#is-it-really-a-bug)
+    - [Do you have valid bytecode?](#do-you-have-valid-bytecode)
+    - [Semantic equivalence vs. exact source code](#semantic-equivalence-vs-exact-source-code)
+- [What to send (minimum requirements)](#what-to-send-minimum-requirements)
+- [What to send (additional helpful information)](#what-to-send-additional-helpful-information)
+    - [But I don't *have* the source code!](#but-i-dont-have-the-source-code)
+        - [But I don't *have* the source code and am incapable of figuring how to do a hand disassembly!](#but-i-dont-have-the-source-code-and-am-incapable-of-figuring-how-to-do-a-hand-disassembly)
+- [Narrowing the problem](#narrowing-the-problem)
+- [Karma](#karma)
+- [Confidentiality of Bug Reports](#confidentiality-of-bug-reports)
+
+<!-- markdown-toc end -->
+
+TL;DR (too long; didn't read)
+
+* Don't do something illegal. And don't ask me to do something illegal or help you do something illegal
+* We already have an infinite supply of decompilation bugs that need fixing, and an automated mechanism for finding more. Decompilation bugs get addressed by easiness to fix and by whim. If you expect yours to be fixed ahead of those, you need to justify why.
+* When asking for help, you may be asked for what you've tried on your own first. There are plenty of sources of information about this code.
+* If you are looking for *timely* help or support, well, that is typically known paid service. I don't really have a mechanism for that since I have a full-time job. But supporting the project is an approximation.
+* Submitting a bug or issue report that is likely to get acted upon may require a bit of effort on your part to make it easy for the problem solver. If you are not willing to do that, please don't waste our time. As indicated above, supporting the project will increase the likelihood of your issue getting noticed and acted upon.
+
+# Ethics
+
+I do not condone using this program for unethical or illegal purposes. More detestable, at least to me, is asking for help to assist you in something that might not legitimate.
+
+Don't use the issue tracker for such solicitations. To try to stave off illegitimate behavior, you should note that the issue tracker, the code, and bugs mentioned in that are in the open: there is no
+confidentiality. You may be asked about the authorship or claimed ownership of the bytecode. If I think something is not quite right, I may label the issue questionable which may make the it easier those who are looking for illegal activity.
+
+
+# The importance of your bug report
+
+For many open-source projects bugs where the expectation is that bugs are rare, reporting bugs in a *thoughtful* way can be helpful. See also [How to Ask Questions the Smart Way](http://www.catb.org/~esr/faqs/smart-questions.html).
+
+In this project though, most of the bug reports boil down to the something like: I have I am trying to reverse engineer some code that I am not the author/owner and that person doesn't want me to have access to. I am hitting a problem somewhere along the line which might have to do with decompilation, but it could be something else like how the bytecode was extracted, some problem in deliberately obfuscated code, or the use some kind of Python bytecode version that isn't supported by the decompiler.
+
+While you are free to report these, unless you sponsor the project, I may close them with about the same amount of effort spent that I think was used to open the report for them. And if you spent a considerable amount of time to create the bug report but didn't follow instructions given here and in the issue template, I am sorry in advance. Just go back, read, and follow instructions.
+
+This project already has an infinite supply of bugs that have been narrowed to the most minimal form and where I have source code to compare against. And in the unlikely event this supply runs out, I have automated means for generating *another* infinite supply.
+
+In this project the task of justifying why addressing your bug is of use to the community, and why it should be prioritized over the others, is the bug reporter's responsibility.
+
+While in the abstract, I have no problem answering questions about how to read a Python traceback or install Python software, or trying to understand what is going wrong in your particular setup, I am not a paid support person and there other things I'd rather be doing with my limited volunteer time. So save us both time, effort, and aggravation: use other avenues like StackOverflow. Again, justifying why you should receive unpaid help is the help requester's responsibility.
+
+
+# The difficulty of the problem and your bug
 
 This decompiler is a constant work in progress: Python keeps
 changing, and so does its code generation.
 
-There is no Python decompiler yet that I know about that will
-decompile everything. Overall, I think this one probably does the best
-job of *any* Python decompiler that handles such a wide range of
-versions.
+There is no Python decompiler yet that I know about that will decompile everything. Overall, I think this one probably does the best job of *any* Python decompiler that handles such a wide range of versions.
 
-But at any given time, there are a number of valid Python bytecode
-files that I know of that will cause problems. See, for example, the
-list in
+But at any given time, there are a number of valid Python bytecode files that I know of that will cause problems. See, for example, the list in
 [`test/stdlib/runtests.sh`](https://github.com/rocky/python-uncompyle6/blob/master/test/stdlib/runtests.sh).
 
-But I understand: you would the bugs _you_ encounter addressed before
-all the other known bugs.
+There are far more bug reporters than there are bug fixers.
 
-From my standpoint, the good thing about the bugs listed in
-`runtests.sh` is that each test case is small and isolated to a single
-kind of problem. And I'll tend to fix easier, more isolated cases than
-generic "something's wrong" kinds of bugs where I'd have to do a bit
-of work to figure out what's up, if not use some sort of mind reading,
-make some guesses, and perform some experiments to see if the guesses
-are correct. I can't read minds, nor am I into guessing games; I'd
-rather devote the effort spent instead towards fixing bugs that are
-precisely defined.
+Unless you are a sponsor of this project, it may take a while, maybe a week or so, before the bug report is noticed, let alone acted upon. Things eventually get fixed, but it may take years. And if your bug hasn't been narrowed, it might happen as a result of some other bug fix.
 
-And it often turns out that by just fixing the well-defined and
-prescribed cases, the ill-defined amorphous cases as well will get
-handled as well.
-
-In sum, you may need to do some work to have the bug you have found
-handled before the hundreds of other bugs, and things I could be
-doing.
-
-No one is getting paid to work to work on this project, let alone the
-bugs you may have an interest in. If you require decompiling bytecode
-immediately, consider using a decompilation service, listed further
-down in this document.
-
-## Is it really a bug?
+# Is it really a bug?
 
 
-### Do you have valid bytecode?
+## Do you have valid bytecode?
 
 As mentioned in README.rst, this project doesn't handle obfuscated
-code. See README.rst for suggestions for how to remove some kinds of
+code, release candidates, and the most recent versions of Python: version 3.9 and up. See README.rst for suggestions for how to remove some kinds of
 obfuscation.
 
 Checking if bytecode is valid is pretty simple: disassemble the code.
@@ -55,20 +76,9 @@ Python comes with a disassembly module called `dis`. A prerequisite
 module for this package, `xdis` has a cross-python version
 disassembler called `pydisasm`.
 
-### Semantic equivalence vs. exact source code
+## Semantic equivalence vs. exact source code
 
-Consider how Python compiles something like "(x*y) + 5". Early on
-Python creates an "abstract syntax tree" (AST) for this. And this is
-"abstract" in the sense that unimportant, redundant or unnecceary
-items have been removed. Here, this means that any notion that you
-wrote "x+y" in parenthesis is lost, since in this context they are
-unneeded. Also lost is the fact that the multiplication didn't have
-spaces around it while the addition did. It should not come as a
-surprise then that the bytecode which is derived from the AST also has
-no notion of such possible variation. Generally this kind of thing
-isn't noticed since the Python community has laid out a very rigid set
-of formatting guidelines; and it has largely beaten the community into
-compliance.
+Consider how Python compiles something like "(x*y) + 5". Early on Python creates an "abstract syntax tree" (AST) for this. And this is "abstract" in the sense that unimportant, redundant or unnecessary items have been removed. Here, this means that any notion that you wrote "x+y" in parenthesis is lost, since in this context they are unneeded. Also lost is the fact that the multiplication didn't have spaces around it while the addition did. It should not come as a surprise then that the bytecode which is derived from the AST also has no notion of such possible variation. Generally this kind of thing isn't noticed since the Python community has laid out a very rigid set of formatting guidelines; and it has largely beaten the community into compliance.
 
 Almost all versions of Python can perform some sort of code
 improvement that can't be undone. In earlier versions of Python it is
@@ -129,10 +139,9 @@ if False:
 
 Python will eliminate the entire "if" statement.
 
-So just because the text isn't the same, does not
-necessarily mean there's a bug.
+So just because the text isn't the same, this does not necessarily mean there's a bug.
 
-## What to send (minimum requirements)
+# What to send (minimum requirements)
 
 The basic requirement is pretty simple:
 
@@ -146,12 +155,12 @@ sending is too large.
 
 Also try to narrow the bug. See below.
 
-## What to send (additional helpful information)
+# What to send (additional helpful information)
 
 Some kind folks also give the invocation they used and the output
 which usually includes an error message produced. This is
 helpful. From this, I can figure out what OS you are running this on
-and what version of *uncomplye6* was used. Therefore, if you _don't_
+and what version of *uncompyle6* was used. Therefore, if you _don't_
 provide the input command and the output from that, please give:
 
 * _uncompyle6_ version used
@@ -159,32 +168,20 @@ provide the input command and the output from that, please give:
 * Python interpreter version used
 
 
-### But I don't *have* the source code!
+## But I don't *have* the source code!
 
-Sure, I get it. No problem. There is Python assembly code on parse
-errors, so simply by hand decompile that. To get a full disassembly,
-use `pydisasm` from the [xdis](https://pypi.python.org/pypi/xdis)
-package. Opcodes are described in the documentation for
-the [dis](https://docs.python.org/3.6/library/dis.html) module.
+There is Python assembly code on parse errors, so simply by hand decompile that. To get a full disassembly, use `pydisasm` from the [xdis](https://pypi.python.org/pypi/xdis) package. Opcodes are described in the documentation for the [dis](https://docs.python.org/3.6/library/dis.html) module.
 
-### But I don't *have* the source code and am incapable of figuring how how to do a hand disassembly!
+### But I don't *have* the source code and am incapable of figuring how to do a hand disassembly!
 
-Well, you could learn. No one is born into this world knowing how to
-disassemble Python bytecode. And as Richard Feynman once said, "What
-one fool can learn, so can another."
+Well, you could learn. No one is born into this world knowing how to disassemble Python bytecode. And as Richard Feynman once said, "What one fool can learn, so can another."
 
-If this is too difficult, or too time consuming, or not of interest to
-you, then perhaps what require is a decompilation service. [Crazy
-Compilers](http://www.crazy-compilers.com/decompyle/) offers a
-byte-code decompiler service for versions of Python up to 2.6. (If
-there are others around let me know and I'll list them here.)
+If this is too difficult, or too time consuming, or not of interest to you, then you might consider [sponsoring](https://github.com/sponsors/rocky) the project. [Crazy
+Compilers](http://www.crazy-compilers.com/decompyle/) offers a byte-code decompiler service for versions of Python up to 2.6. (If there are others around let me know and I'll list them here.) Don't be surprised if I ask you to pay for work (if I think the work is ethical) when you want me to work on your problem that I think isn't of interest or benefit to anyone but yourself or a small limited number of people, or I think the need is questionable.
 
-## Narrowing the problem
+# Narrowing the problem
 
-I don't need or want the entire source code base for the file(s) or
-module(s) can't be decompiled. I just need those file(s) or module(s).
-If there are problems in several files, file a bug report for each
-file.
+I don't need or want the entire source code base for the file(s) or module(s) can't be decompiled. I just need those file(s) or module(s). If there are problems in several files, file a bug report for each file.
 
 Python modules can get quite large, and usually decompilation problems
 occur in a single function or maybe the main-line code but not any of
@@ -197,28 +194,45 @@ what doesn't. That is useful. Or maybe the same file will decompile
 properly on a neighboring version of Python. That is helpful too.
 
 In sum, the more you can isolate or narrow the problem, the more
-likley the problem will be fixed and fixed sooner.
+likely the problem will be fixed and fixed sooner.
 
-## Confidentiality of Bug Reports
+# Karma
+
+I realize that following the instructions given herein puts a bit of
+burden on the bug reporter. This is justified since it attempts to balance
+the burden and effort needed to fix the bug with the amount of effort to report the problem. And it attempts
+to balance number of would-be bug reporters with the number of bug
+fixers. Better bug reporters are more likely to move in the category
+of bug fixers.
+
+The barrier to reporting a big is pretty small: all you really need is
+a github account, and the ability to type something after clicking
+some buttons. So the reality is that many people just don't bother to
+read these instructions, let alone follow it to any simulacrum.
+
+That said, bugs sometimes get fixed even though these instructions are not followed.
+
+I may take into consideration is the bug reporter's karma.
+
+* Have you demonstrably contributed to open source? I may look at your github profile to see what contributions you have made, how popular those contributions are, or how popular you are.
+* How appreciative are you? Have you starred this project that you are seeking help from? Have you starred _any_ github project? And the above two kind of feed into ...
+* Attitude. Some people feel that they are doing me and the world a
+  great favor by just pointing out that there is a problem whose
+  solution would greatly benefit them. (This might account partially
+  for the fact that those that have this attitude often don't read or
+  follow instructions such as those given here.)
+
+
+# Confidentiality of Bug Reports
 
 When you report a bug, you are giving up confidentiality to the source
 code and the byte code. However, I would imagine that if you have
 narrowed the problem sufficiently, confidentiality of the little that
 remains would not be an issue.
 
-However feel free to remove any commments, and modify variable names
+However feel free to remove any comments, and modify variable names
 or constants in the source code.
 
-## Ethics
+If there is some legitimate reason to keep confidentiality, you can contact me by email to explain the extenuating circumstances. However I tend to discard without reading anonymous email.
 
-I do not condone using this program for unethical or illegal purposes.
-More detestful, at least to me, is asking for help to assist you in
-something that might not legitimate.
-
-Don't use the issue tracker for such solicitations. To try to stave
-off illegitimate behavior, you should note that the issue tracker, the
-code, and bugs mentioned in that are in the open: there is no
-confidentiality. You may be asked about the authorship or claimed
-ownership of the bytecode. If I think something is not quite right, I
-may label the issue questionable which may make the it easier those
-who are looking for illegal activity.
+Private consulting available via https://calendly.com/rb3216 rates: $150 for 30 minutes; $250 for 60 minutes.

Makefile

@@ -11,7 +11,10 @@ RM      ?= rm
 LINT    = flake8
 
 #EXTRA_DIST=ipython/ipy_trepan.py trepan
-PHONY=all check clean distcheck pytest check-long dist distclean lint flake8 test rmChangeLog clean_pyc
+PHONY=all check check-2.7 check-3.4 \
+     clean distcheck pytest check-long check-short \
+     dist distclean lint flake8 test rmChangeLog clean_pyc \
+     2.6 5.0 5.3 5.6 5.8 7.2 7.3 check-short
 
 TEST_TYPES=check-long check-short check-2.7 check-3.4
 
@@ -40,12 +43,23 @@ check-3.0 check-3.1 check-3.2 check-3.6:
 check-3.7: pytest
 	$(MAKE) -C test check
 
+check-3.8:
+	$(MAKE) -C test check
+
 #:PyPy 2.6.1 PyPy 5.0.1, or PyPy 5.8.0-beta0
 # Skip for now
 2.6 5.0 5.3 5.6 5.8:
 
-#:PyPy pypy3-2.4.0 Python 3:
-pypy-3.2 2.4:
+#:PyPy pypy3-2.4.0 Python 3.6.1:
+7.1 pypy-3.2 2.4:
+	$(MAKE) -C test $@
+
+#:PyPy versions
+7.2 7.3:
+	$(MAKE) -C test $@
+
+#:pyston versions
+2.3:
 	$(MAKE) -C test $@
 
 #: Run py.test tests

NEWS

@@ -1,635 +0,0 @@
-uncompyle6 3.2.3 2018-06-04 Michael Cohen flips and Fleetwood Redux
-
-- Python 1.3 support 3.0 bug and
-- fix botched parameter ordering of 3.x in last release
-
-uncompyle6 3.2.2 2018-06-04 When I'm 64
-
-- Python 3.0 support and bug fixes
-
-uncompyle6 3.2.1 2018-06-04 MF
-
-- Python 1.4 and 1.5 bug fixes
-
-uncompyle6 3.2.0 2018-05-19 Rocket Scientist
-
-- Add rudimentary 1.4 support (still a bit buggy)
-- add --tree+ option to show formatting rule, when it is constant
-- Python 2.7.15candidate1 support (via xdis)
-- bug fixes, especially for 3.7 (but 2.7 and 3.6 and others as well)
-
-uncompyle6 3.1.3 2018-04-16
-
-- Add some Python 3.7 rules, such as for handling LOAD_METHOD (not complete)
-- Fix some fragment bugs
-- small doc changes
-
-uncompyle6 3.1.2 2018-04-08 Eastern Orthodox Easter
-
-- Python 3.x subclass and call parsing fixes
-- Allow/note running on Python 3.1
-- improve 3.5+ BUILD_MAP_UNPACK
-- DRY instruction building code between 2.x and 3.x
-- expand testing
-
-uncompyle6 3.1.1 2018-04-01 Easter April Fool's
-
-Jesus on Friday's New York Times puzzle: "I'm stuck on 2A"
-
-- fill out 3.5+ BUILD_MAP_UNPACK (more work is needed)
-- fill out 3.4+ CALL_FUNCTION_... (more work is needed)
-- fill out 3.5 MAKE_FUNCTION  (more work is needed)
-- reduce 3.5, 3.6 control-flow bugs
-- reduce ambiguity in rules that lead to long (exponential?) parses
-- limit/isolate some 2.6/2.7,3.x grammar rules
-- more runtime testing of decompiled code
-- more removal of parenthesis around calls via setting precidence
-
-uncompyle6 3.1.0 2018-03-21 Equinox
-
-- Add code_deparse_with_offset() fragment function.
-- Correct paramenter call fragment deparse_code()
-- Lots of 3.6, 3.x, and 2.7 bug fixes
-  About 5% of 3.6 fail parsing now. But
-  semantics still needs much to be desired.
-
-uncompyle6 3.0.1 2018-02-17
-
-- All Python 2.6.9 standard library files weakly verify
-- Many 3.6 fixes. 84% of the first 200 standard library files weakly compile.
-  One more big push is needed to get the remaining to compile
-- Many decompilation fixes for other Python versions
-- Add more to the test framework
-- And more add tests target previous existing bugs more completely
-- sync recent license changes in metadata
-
-uncompyle6 3.0.0 2018-02-17
-
-- deparse_code() and lookalikes from the various semantic actions are
-  now deprecated. Instead use new API code_deparse() which makes the
-  version optional and bundles debug options into a dictionary.
-- License changed to GPL3.
-- Many Python 3.6 fixes, especially around handling EXTENDED_ARGS
-  Due to the reduction in operand size for JUMP's there are many
-  more EXTENDED_ARGS instructions which can be the targets
-  of jumps, and messes up the peephole-like analysis that is
-  done for control flow since we don't have something better in place.
-- Code has been reorganized to be more instruction nametuple based where it
-  has been more bytecode array based. There was and still is code that had
-  had magic numbers to advance instructions or to pick out operands.
-- Bug fixes in numerous other Python versions
-- Instruction display improved
-- Keep global statements in fixed order (from wangym5106)
-
-A bit more work is still needed for 3.6 especially in the area of
-function calls and definitions.
-
-
-uncompyle6 2.16.0 2018-02-17
-
-- API additions:
-  - add fragments.op_at_code_loc() and
-  - fragments.deparsed_find()_
-- Better 2.7 end_if and COME_FROM determination
-- Fix up 3.6+ CALL_FUNCTION_EX
-- Misc pydisasm fixes
-- Weird comprehension bug seen via new loctraceback
-- Fix Python 3.5+ CALL_FUNCTION_VAR and BUILD_LIST_UNPACK in call; with this
-  we can can handle 3.5+ f(a, b, *c, *d, *e) now
-
-uncompyle6 2.15.1 2018-02-05
-
-- More bug fixes and revert an improper bug fix in 2.15.0
-
-uncompyle6 2.15.0 2018-02-05 pycon2018.co
-
-- Bug fixes
-- Code fragment improvements
-- Code cleanups
-- Expand testing
-
-uncompyle6 2.15.1 2018-01-27
-
-- Add --linemap option to give line correspondences
-  between original source lines and reconstructed line sources.
-  It is far from perfect, but it is a start
-- Add a new class of tests: tests which when decompiled check themselves
-- Split off Python version semantic action customizations into its own file
-- Fix 2.7 bug in ifelse loop statement
-- Handle 3.6+ EXTENDED_ARGs for POP_JUMP_IF... instructions
-- Correct 3.6+ calls with kwargs
-- Describe the difficulty of 3.6 in README
-
-uncompyle6 2.14.3 2018-01-19
-
-- Fix bug in 3.5+ await stmt
-- Better version to magic handling; handle 3.5.2 .. 3.5.4 versions
-- Improve/correct test_pyenvlib.py status messages
-- Fix some 2.7 and 2.6 parser bugs
-- Fix whilelse parsing bugs
-- Correct 2.5- decorator parsing
-- grammar for decorators matches AST a little more
-- better tests in setup.py for running the right version of Python
-- Fix 2.6- parsing of "for .. try/else" ...  with "continue"  inside
-
-uncompyle6 2.14.2 2018-01-09 Samish
-
-Decompilation bug fixes, mostly 3.6 and pre 2.7
-
-- 3.6 FUNCTION_EX (somewhat)
-- 3.6 FUNCTION_EX_KW fixes
-- 3.6 MAKE_FUNCTION fixes
-- correct 3.5 CALL_FUNCTION_VAR
-- stronger 3.x "while 1" testing
-- Fix bug in if's with "pass" bodies. Fixes #104
-- try/else and try/finally fixes on 2.6-
-- limit pypy customization to pypy
-- Add addr fields in COME_FROMS
-- Allow use of full instructions in parser reduction routines
-- Reduce grammar in Python 3 by specialization more to specific
-  Python versions
-- Match Python AST names more closely when possible
-
-uncompyle6 2.14.1 2017-12-10 Dr. Gecko
-
-- Many decompilation bugfixes
-- Grammar rule reduction and version isolation
-- Match higher-level nonterminal names more closely
-  with Python AST
-- Start automated Python stdlib testing - full round trip
-
-uncompyle6 2.14.0 2017-11-26 johnnybamazing
-
-- Start to isolate grammar rules between versions
-  and remove used grammar rules
-- Fix a number of bytecode decompile problems
-  (many more remain)
-- Add stdlib/runtests.sh for even more rigorous testing
-
-uncompyle6 2.13.3 2017-11-13
-
-Overall: better 3.6 decompiling and some much needed code refactoring and cleanup
-
-
-- Start noting names in for template-action names; these are
-  used to check/assert we have the right node type
-- Simplify <import_from> rule
-- Pypy 5.80-beta testing tolerance
-- Start to clean up instruction mangling phase by using 3.6-style instructions
-  rather trying to parse the bytecode array. This largely been done in for versions 3.x;
-  3.0 custom mangling code has been reduced;
-  some 2.x conversion has been done, but more is desired. This make it possible to...
-- Handle EXTENDED_ARGS better. While relevant to all Python versions it is most noticeable in
-  version 3.6+ where in switching to wordcodes the size of operands has been reduced from 2**16
-  to 2**8. JUMP instruction then often need EXTENDED_ARGS.
-- Refactor find_jump_targets() with via working of of instructions rather the bytecode array.
-- use --weak-verify more and additional fuzzing on verify()
-- fragment parser now ignores errors in nested function definitions; an parameter was
-  added to assist here. Ignoring errors may be okay because the fragment parser often just needs,
-  well, *fragments*.
-- Distinguish RETURN_VALUE from RETURN_END_IF in exception bodies better in 3.6
-- bug in 3.x language changes: import queue via import Queue
-- reinstate some bytecode tests since decompiling has gotten better
-- Revise how to report a bug
-
-uncompyle6 2.13.2 2017-10-12
-
-- Re-release using a more automated approach
-
-uncompyle6 2.13.1 2017-10-11
-
-- Re-release because Python 2.4 source uploaded rather than 2.6-3.6
-
-uncompyle6 2.13.0 2017-10-10
-
-- Fixes in deparsing lambda expressions
-- Improve table-semantics descriptions
-- Document hacky customize arg count better (until we can remove it)
-- Update to use xdis 3.7.0 or greater
-
-uncompyle6 2.12.0 2017-09-26
-
-- Use xdis 3.6.0 or greater now
-- Small semantic table cleanups
-- Python 3.4's terms a little names better
-- Slightly more Python 3.7, but still failing a lot
-- Cross Python 2/3 compatibility with annotation arguments
-
-uncompyle6 2.11.5 2017-08-31
-
-- Skeletal support for Python 3.7
-
-uncompyle6 2.11.4 2017-08-15
-
-* scanner and parser now allow 3-part version string lookups,
-  e.g. 2.7.1 We allow a float here, but if passed a string like '2.7'. or
-* unpin 3.5.1. xdis 3.5.4 has been release and fixes the problems we had. Use that.
-* some routines here moved to xdis. Use the xdis version
-* README.rst: Link typo Name is trepan2 now not trepan
-* xdis-forced change adjust for COMPARE_OP "is-not" in
-  semanatic routines. We need "is not".
-* Some PyPy tolerance in validate testing.
-* Some pyston tolerance
-
-uncompyle6 2.11.3 2017-08-09
-
-Very minor changes
-
-- RsT doc fixes and updates
-- use newer xdis, but not too new; 3.5.2 breaks uncompyle6
-- use xdis opcode sets
-- xdis "exception match" is now "exception-match"
-
-uncompyle6 2.11.2 2017-07-09
-
-- Start supporting Pypy 3.5 (5.7.1-beta)
-- use xdis 3.5.0's opcode sets and require xdis 3.5.0
-- Correct some Python 2.4-2.6 loop detection
-- guard against badly formatted bytecode
-
-uncompyle6 2.11.1 2017-06-25
-
-- Python 3.x annotation and function signature fixes
-- Bump xdis version
-- Small pysource bug fixes
-
-uncompyle6 2.11.0 2017-06-18 Fleetwood
-- Major improvements in fragment tracking
-  * Add nonterminal node in extractInfo
-  * tag more offsets in expressions
-  * tag array subscripts
-  * set YIELD value offset in a <yield> expr
-  * fix a long-standing bug in not adjusting final AST when melding other deparse ASTs
-- Fixes yet again for make_function node handling; document what's up here
-- Fix bug in snowflake Python 3.5 *args kwargs
-
-uncompyle6 2.10.1 2017-06-3 Marylin Frankel
-
-- fix some fragments parsing bugs
-  - was returning the wrong type sometimes in deparse_code_around_offset()
-  - capture function name in offsets
-  - track changes to ifelstrmtr node from pysource into fragments
-
-uncompyle6 2.10.0 2017-05-30 Elaine Gordon
-
-- Add fuzzy offset deparse look up
-- 3.6 bug fixes
-   - fix EXTENDED_ARGS handling (and in 2.6 and others)
-   - semantic routine make_function fragments.py
-   - MAKE_FUNCTION handling
-   - CALL_FUNCTION_EX handling
-   - async property on defs
-   - support for CALL_FUNCTION_KW (moagstar)
-- 3.5+ UNMAP_PACK and BUILD_UNMAP_PACK handling
-- 3.5 FUNCTION_VAR bug
-- 3.x pass statement insdie while True
-- Improve 3.2 decompilation
-- Fixed -o argument processing (grkov90)
-- Reduce scope of LOAD_ASSERT as expr to 3.4+
-- "await" statement fixes
-- 2.3, 2.4 "if 1 .." fixes
-- 3.x annotation fixes
-
-uncompyle6 2.9.11 2017-04-06
-
-- Better support for Python 3.5+ BUILD_MAP_UNPACK
-- Start 3.6 CALL_FUNCTION_EX support
-- Many decompilation bug fixes. (Many more remain). See ChangeLog
-
-uncompyle6 2.9.10 2017-02-25
-
-- Python grammar rule fixes
-- Add ability to get grammar coverage on runs
-- Handle Python 3.6 opcode BUILD_CONST_KEYMAP
-
-uncompyle6 2.9.9 2016-12-16
-
-- Remaining Python 3.5 ops handled
-  (this also means more Python 3.6 ops are handled)
-- Python 3.5 and 3.6 async and await handled
-- Python 3.0 decompilation improved
-- Python 3 annotations fixed
-- Better control-flow detection
-- Code cleanups and misc bug fixes
-
-uncompyle6 2.9.8 2016-12-16
-
-- Better control-flow detection
-- pseudo instruction THEN in 2.x
-  to disambiguate if from and
-- fix bug in --verify option
-- DRY (a little) control-flow detection
-- fix syntax in tuples with one element
-- if AST rule inheritance in Python 2.5
-- NAME_MODULE removal for Python <= 2.4
-- verify call fixes for Python <= 2.4
-- more Python lint
-
-uncompyle6 2.9.7 2016-12-16
-
-- Start to handle 3.5/3.6 build_map_unpack_with_call
-- Some Python 3.6 bytecode to wordcode conversion fixes
-- option -g: show start-end range when possible
-- track print_docstring move to help (used in python 3.1)
-- verify: allow RETURN_VALUE to match RETURN_END_IF
-- some 3.2 compatibility
-- Better Python 3 control flow detection by adding Pseudo ELSE opcodes
-
-uncompyle6 2.9.6 2016-12-04
-
-- Shorten Python3 grammars with + and *
-  this requires spark parser 1.5.1
-- Add some AST reduction checks to improve
-  decompile accuracy. This too requires
-  spark parser 1.5.1
-
-uncompyle6 2.9.6 2016-11-20
-
-- Correct MANIFEST.in
-- More AST grammar checking
-- --linemapping option or linenumbers.line_number_mapping()
-  Shows correspondence of lines between source
-  and decompiled source
-- Some control flow adjustments in code for 2.x.
-  This is probably an improvement in 2.6 and before.
-  For 2.7 things are just shuffled around a little. Sigh.
-  Overall I think we are getting more precise in
-  or analysis even if it is not always reflected
-  in the results.
-- better control flow debugging output
-- Python 2 and 3 detect structure code is more similar
-- Handle Docstrings with embedded triple quotes (""")
-
-uncompyle6 2.9.5 2016-11-13
-
-- Fix Python 3 bugs:
-  * improper while 1 else
-  * docstring indent
-  * 3.3 default values in lambda expressions
-  * start 3.0 decompilation (needs newer xdis)
-- Start grammar misparse checking
-
-
-uncompyle6 2.9.4 2016-11-02
-
-- Handle Python 3.x function annotations
-- track def keyword-parameter line-splitting in source code better
-- bump min xdis version to mask previous xdis bug
-
-uncompyle6 2.9.3 2016-10-26
-
-Release forced by incompatibility change in xdis 3.2.0.
-
-- Python 3.1 bugs:
-  * handle "with ... as"
-  * handle "with"
-  * Start handling def (...) -> yy (has bugs still)
-
-- DRY Python 3.x via inheritance
-- Python 3.6 work (from Daniel Bradburn)
-  * Handle 3.6 buildstring
-  * Handle 3.6 handle single and multiple fstring better
-
-
-uncompyle6 2.9.2 2016-10-15
-
-- use source-code line breaks to assist in where to break
-  in tuples and maps
-- Fix Python 1.5 decompyle bugs
-- Fix some Python 2.6 and below bugs
-- DRY fragments.py code a little
-
-uncompyle6 2.9.1 2016-10-09
-
-- Improved Python 1.5 decompiling
-- Handle old-style pre Python 2.2 classes
-
-uncompyle6 2.9.0 2016-10-09
-
-- Use xdis 3.0.0 protocol load_module.
-  this Forces change in requirements.txt and _pkg_info_.py
-- Start Python 1.5 decompiling; another round of work is needed to
-  remove bugs
-- Simplify python 2.1 grammar
-- Fix bug with -t ...  Wasn't showing source text when -t option was given
-- Fix 2.1-2.6 bug in list comprehension
-
-uncompyle6 2.8.4 2016-10-08
-
-- Python 3 disassembly bug fixes
-- Python 3.6 fstring bug fixes (from moagstar)
-- Python 2.1 disassembly
-- COME_FROM suffixes added in Python3
-- use .py extension in verification disassembly
-
-uncompyle6 2.8.3 2016-09-11 live from NYC!
-
-NOTE: this is possibly the last release before a major reworking of
-control-flow structure detection is done.
-
-- Lots of bug fixes in decompilation:
-  * 3.0 .. 3.4 whileTrue bug
-  * 3.x function declaration deparsing:
-    . 3.0 .. 3.2 *args processing
-    . 3.0 .. 3.2 call name and kwargs bug
-    . 3.0 .. getting parameter of *
-    . 3.0 .. handling variable number of args
-    . 3.0 .. "if" structure bugs
-  * 3.5+ if/else bugs
-  * 2.2-2.6 bugs
-    . try/except control flow
-    . a == b == c -like detection
-    . generator detection
-    . "while .. and" statement bugs
-    . handle "except <cond>, <var>"
-    . use older raise format in 2.x
-- scanner "disassemble" is now "ingest". True disassembly is done by xdis
-- Start accepting Python 3.1 bytecode
-- Add --weak-verify option on test_pyenvlib and test_pythonlib. This
-  catches more bugs more easily
-- bump xdis requirement so we can deparse dropbox 2.5 code
-- Added H. Goebel's changes before 2.4 in DECOMPYLE-2.4-CHANGELOG.txt
-
-uncompyle6 2.8.2 2016-08-29
-
-- Handle Python 3.6 format string conversions !r, !s, !a
-- Start to handle 3.1 bytecode
-- Fix some PyPy translation bugs
-- We now only handle 3.6.0a3+ since that is incompatible with 3.6 before that
-
-uncompyle6 2.8.1 2016-08-20
-
-- Add Python 2.2 decompilation
-
-- Fix bugs
- * PyPy LOOKUP_METHOD bug
- * Python 3.6 FORMAT_VALUE handles expressions now
-
-uncompyle6 2.8.0 2016-08-03
-
-- Start Python 3.6 support (moagstar)
-  more work on PEP 498 needed
-- tidy bytecode/word output
-- numerous decompiling bugs fixed
-- grammar testing started
-- show magic number in deparsed output
-- better grammar and semantic action segregation based
-  on python bytecode version
-
-uncompyle6 2.7.1 2016-07-26
-
-- PyPy bytecodes for 2.7 and 3.2 added
-- Instruction formatting improved slightly
-- 2.7 bytecode "continue" bug fixed
-
-uncompyle6 2.7.0 2016-07-15
-
-- Many Syntax and verification bugs removed
-  tested on standard libraries from 2.3.7 to 3.5.1
-  and they all decompile and verify fine.
-  I'm sure there are more bugs though.
-
-uncompyle6 2.6.2 2016-07-11 Manhattenhenge
-
-- Extend bytecodes back to 2.3
-- Fix bugs:
-  * 3.x and 2.7 set comprehensions,
-  * while1 loops
-  * continue statements
-- DRY and segregate grammar more
-
-uncompyle6 2.6.1 2016-07-08
-
-- Go over Python 2.5 bytecode deparsing
-  all library programs now deparse
-- Fix a couple bugs in 2.6 deparsing
-
-uncompyle6 2.6.0 2016-07-07
-
-- Improve Python 2.6 bytecode deparsing:
-  stdlib now will deparse something
-- Better <2.6 vs. 2.7 grammar separation
-- Fix some 2.7 deparsing bugs
-- Fix bug in installing uncompyle6 script
-- Doc improvements
-
-uncompyle6 2.5.0 2016-06-22  Summer Solstice
-
-- Much better Python 3.2-3.5 coverage.
-  3.4.6 is probably the best;3.2 and 3.5 are weaker
-- Better AST printing with -t
-- Better error reporting
-- Better fragment offset tracking
-- Some (much-needed) code refactoring
-
-uncompyle6 2.4.0 2016-05-18 (in memory of Lewis Bernstein)
-
-- Many Python 3 bugs fixed:
-  * Python 3.2 to 3.5 libraries largely
-    uncompyle and most verify
-- pydisassembler:
-  * disassembles all code objects in a file
-  * can select showing bytecode before
-    or after uncompyle mangling, option -U
-- DRY scanner code (but more is desired)
-- Some code cleanup (but more is desired)
-- Misc Bugs fixed:
-  * handle complex number unmarshaling
-  * Running on Python 2 to works on Python 3.5 bytecodes now
-
-uncompyle6 2.3.5 and 2.3.6 2016-05-14
-
-- Python 2 class decorator fix (thanks to Tey)
-- Fix fragment parsing bugs
-- Fix some Python 3 parsing bugs:
-  * Handling single in * parameter
-  * "while True"
-  * escape from for inside if
-  * yield expressions
-- Correct history based on info from Dan Pascu
-- Fix up pip packaging, ugh.
-
-uncompyle6 2.3.4 2016-05-5
-
-- More Python 3.5 parsing bugs addressed
-- decompiling Python 3.5 from other Python versions works
-- test from Python 3.2
-- remove "__module__ = __name__" in 3.0 <= Python 3.2
-
-uncompyle6 2.3.3 2016-05-3
-
-- Fix bug in running uncompyle6 script on Python 3
-- Speed up performance on deparsing long lists by grouping in chunks of 32 and 256 items
-- DRY Python expressions between Python 2 and 3
-
-uncompyle6 2.3.2 2016-05-1
-
-- Add --version option standalone scripts
-- Correct License information in package
-- expose fns uncompyle_file, load_file, and load_module
-- Start to DRY Python2 and Python3 grammars Separate out 3.2, and 3.5+
-  specific grammar code
-- Fix bug in 3.5+ constant map parsing
-
-uncompyle6 2.3.0, 2.3.1 2016-04-30
-
-- Require spark_parser  >= 1.1.0
-
-uncompyle6 2.2.0 2016-04-30
-
-- Spark is no longer here but pulled separate package spark_parse
-- Python 3 parsing fixes
-- More tests
-
-uncompyle6 2.2.0 2016-04-02
-
-- Support single-mode (in addition to exec-mode) compilation
-- Start to DRY Python 2 and Python 3 grammars
-- Fix bug in if else ternary  construct
-- Fix bug in uncomplye6 -d and -r options (via lelicopter)
-
-
-uncompyle6 2.1.3 2016-01-02
-
-- Limited support for decompiling Python 3.5
-- Improve Python 3 class deparsing
-- Handle MAKE_CLOSURE opcode
-- Start to DRY opcode code.
-- increase test coverage
-- fix misc small bugs and some improvements
-
-uncompyle6 2.1.2 2015-12-31
-
-- Fix cross-version Marshal loading
-- Handle Python 3.3 . dotted class names
-- Limited 3.5 support: allows deparsing other versions
-- Refactor code more, misc bug fixes
-
-uncompyle6 2.1.1 2015-12-27
-
-- packaging issues
-
-uncompyle6 2.1.0 2015-12-27
-
-- Python 3.x deparsing much more solid
-- Better cross-version deparsing
-
-Some bugs squashed while other run rampant. Some code cleanup while
-much more is yet needed. More tests added, but many more are needed.
-
-
-uncompyle6 2.0.0 2015-12-11
-
-Changes from uncompyle2
-
-- Can give code fragments given an instruction offset. See
-  https://github.com/rocky/python-uncompyle6/wiki/Deparsing-technology-and-its-use-in-exact-location-reporting
-- Runs under Python3. Decompiles Python 2.5-2.7 and some Python 3.2-3.4
-- Allows for multiple Python grammars, specifically Python2 vs Python 3
-- Add a cross-version Python disassembler command-line utility
-- Add some py.test and start reorganizing tests
-
-SPARK:
-  add option to show grammar rules applied
-  allow Python-style # comments in grammar
-  Runs on Python 3 and Python 2

NEW_FEATURES.rst

@@ -0,0 +1,221 @@
+Introduction
+============
+
+The original versions of this code up until the time I started were
+pretty awesome.  You can get a sense of this by running it.  For the
+most part it was remarkably fast, and a single module with few dependencies.
+
+Here I will largely give what are the major improvements over old code.
+
+This also serves to outline a little bit about what is in this code.
+
+See also `How does this code work? <https://github.com/rocky/python-uncompyle6/wiki/How-does-this-code-work%3F>`_.
+
+Old Cool Features
+==================
+
+Before getting to the new stuff, I'll describe cool things that was there before.
+
+I particularly liked the ability to show the assembly, grammar
+reduction rules as they occurred, and the resulting parse tree. It is
+neat that you could follow the process and steps that deparser takes,
+and in this not only see the result how the bytecode corresponds to
+the resulting source. Compare this with other Python decompilers.
+
+And of course also neat was that this used a grammar and table-driven
+approach to decompile.
+
+
+Expanding decompilation to multiple Python Versions
+==================================================
+
+Aside from ``pycdc``, most of the Python decompilers handle a small
+number of Python versions, if they supported more than one. And even
+when more than one version is supported if you have to be running the
+Python version that the bytecode was compiled for.
+
+There main reason that you have to be running the Python bytecode
+interpreter as the one you want to decompile largely stems from the
+fact that Python's ``dis`` module is often what is used and that has this limitation.
+
+``pycdc`` doesn't suffer this problem because it is written in C++,
+not Python.  Hartmut Goebel's code had provisions for multiple Python
+versions running from an interpreter different from the one that was
+running the decompiler. That however used compiled code in the process
+was tied a bit to the Python C headers for a particular version.
+
+You need to not only to account for different "marshal" and "unmarshal"
+routines for the different Python versions, but also, as the Python versions
+extend, you need a different code type as well.
+
+Enter ``xdis``
+--------------
+
+To handle all of these problems, I split off the marshal loading
+portion and disassembly routines into a separate module,
+`xdis <https://pypi.org/project/xdis/>`_. This also allows older Pythons to have access to features
+found in newer Pythons, such as parsing the bytecode, a uniform stream
+of bytes, into a list of structured bytecode instructions.
+
+Python 2.7's ``dis`` module doesn't has provide a instruction abstraction.
+Therefore in ``uncompyle2`` and other earlier decompilers you see code with magic numbers like 4 in::
+
+    if end > jump_back+4 and code[end] in (JF, JA):
+        if code[jump_back+4] in (JA, JF):
+            if self.get_target(jump_back+4) == self.get_target(end):
+                self.fixed_jumps[pos] = jump_back+4
+                end = jump_back+4
+    elif target < pos:
+        self.fixed_jumps[pos] = jump_back+4
+        end = jump_back+4
+
+and in other code -1 and 3 in::
+
+        if self.get_target(jmp) != start_else:
+            end_else = self.get_target(jmp)
+        if self.code[jmp] == JF:
+            self.fixed_jumps[jmp] = -1
+        self.structs.append({'type':  'except',
+                       'start': i,
+                       'end':   jmp})
+        i = jmp + 3
+
+All of that offset arithmetic is trying to find the next instruction
+offset or the previous offset. Using a list of instructions you simply
+take the ``offset`` field of the previous or next instruction.
+
+The above code appears in the ``uncompyle2`` "Scanner" class in
+service of trying to figure out control flow. Note also that there
+isn't a single comment in there about what specifically it is trying
+to do, the logic or that would lead one to be confident that this is
+correct, let alone assumptions that are needed for this to be true.
+
+While this might largely work for Python 2.7, and ``uncompyle2`` does
+get control flow wrong sometimes, it is impossible to adapt code for
+other versions of Python.
+
+In addition adding an instruction structure, ``xdis`` adds various
+flags and features that assist in working with instructions. In the
+example above this replaces code like ``... in (JF, JA)`` which is
+some sort of unconditional jump instruction.
+
+Although not needed in the decompiler, ``xdis`` also has nicer
+instruction print format. It can show you the bytes as well as the
+interpreted instructions. It will interpret flag bits and packed
+structures in operands so you don't have to. It can even do a limited
+form of inspection at previous instructions to give a more complete
+description of an operand. For example on ``LOAD_ATTR`` which loads
+the attribute of a variable, often the variable name can be found as
+the previous instruction. When that is the case the disassembler can
+include that in the disassembly display for the ``LOAD_ATTR`` operand.
+
+
+Python Grammar Isolation
+------------------------
+
+If you want to support multiple versions of Python in a manageable way
+you really need to provide different grammars for the different
+versions, in a grammar-based system. None of the published versions of
+this decompiler did this.
+
+If you look at the changes in this code, right now there are no
+grammar changes needed between 1.0 to 1.3. (Some of this may be wrong
+though since we haven't extensively tested these earliest Python versions
+
+For Python 1.4 which is based off of the grammar for 1.5 though there
+are number of changes, about 6 grammar rules. Later versions of though
+we start to see larger upheaval and at certain places, especially
+those where new opcodes are introduced, especially those that change
+the way calls or exceptions get handled, we have major upheaval in the
+grammar. It is not just that some rules get added, but we also need to
+*remove* some grammar rules as well.
+
+I have been largely managing this as incremental differences between versions.
+However in the future I am leaning more towards totally separate grammars.
+A well constructed grammar doesn't need to be that large.
+
+When starting out a new version, we can just copy the grammar from the
+prior version.  Within a Python version though, I am breaking these
+into composable pieces. In particular the grammar for handling what
+can appear as the body of a lambda, is a subset of the full Python
+language. The language allowed in an ``eval`` is also a subset of the
+full Python language, as are what can appear in the various
+compilation modes like "single" versus "exec".
+
+Another nice natural self-contain grammar section is what can appear
+in list comprehensions and generators. The bodies of these are
+generally represented in a self-contained code block.
+
+Often in decompilation you may be interested not just in decompiling
+the entire code but you may be interested in only focusing on a
+specific part of the code. And if there is a problem in decompiling
+the entire piece of code, having these smaller breaking points can be
+of assistance.
+
+Other Modularity
+----------------
+
+Above we have mentioned the need for separate grammars or to isolate
+these per versions. But there are other major pieces that make up this
+decompiler. In particular there is a scanner and the source code
+generation part.
+
+Even though differences in version that occur in disassembly are
+handled by ``xdis``, we still have to do conversion of that to a token
+stream for parsing. So the scanners are again broken out per version
+with various OO mechanisms for reusing code. The same is true for
+source code generation.
+
+
+Expanding decompiler availability to multiple Python Versions
+--------------------------------------------------------------
+
+Above we mention decompiling multiple versions of bytecode from a
+single Python interpreter. We we talk about having the decompiler
+runnable from multiple versions of Python, independent of the set of
+bytecode that the decompiler supports.
+
+
+There are slight advantages in having a decompiler that runs the same
+version as the code you are decompiling. The most obvious one is that
+it makes it easy to test to see whether the decompilation correct
+because you can run the decompiled code. Python comes with a suite of
+Python programs that check themselves and that aspects of Python are
+implemented correctly. These also make excellent programs to check
+whether a program has decompiled correctly.
+
+Aside from this, debugging can be easier as well. To assist
+understanding bytcode and single stepping it see `x-python
+<https://pypi.org/project/x-python/>`_ and the debugger for it
+`trepan-xpy <https://pypi.org/project/trepanxpy/>`_.
+
+Handling Language Drift
+-----------------------
+
+Given the desirability of having this code running on logs of Python
+versions, how can we get this done?
+
+The solution used here is to have several git branches of the
+code. Right now there are 3 branches. Each branch handles works across
+3 or so different releases of Python. In particular one branch handles
+Python 2.4 to 2.7 Another handles Python 3.3 to 3.5, and the master
+branch handles 3.6 to 3.10. (Again note that the 3.9 and 3.10
+decompilers do not decompile Python 3.9 or 3.10, but they do handle
+bytecode for all earlier versions.)
+
+
+Cool features of the Parser
+===========================
+
+* reduction rule checking
+* numbering tokens
+* showing a stack of completions
+
+Cool features Semantic Analysis
+===============================
+
+* ``--tree++`` (``-T``) option
+* showing precedence
+* See `Adding a tree transformation phase to uncompyle6 <https://github.com/rocky/python-uncompyle6/wiki/Adding-a-tree-transformation-phase-to-uncompyle6>`_
+* following AST
+* Fragment deparsing

README.rst

@@ -1,4 +1,8 @@
-|buildstatus| |Latest Version| |Supported Python Versions|
+|buildstatus|  |Pypi Installs| |Latest Version| |Supported Python Versions|
+
+|packagestatus|
+
+.. contents::
 
 uncompyle6
 ==========
@@ -11,9 +15,9 @@ Introduction
 ------------
 
 *uncompyle6* translates Python bytecode back into equivalent Python
-source code. It accepts bytecodes from Python version 1.3 to version
-3.7, spanning over 22 years of Python releases. We include Dropbox's
-Python 2.5 bytecode and some PyPy bytecode.
+source code. It accepts bytecodes from Python version 1.0 to version
+3.8, spanning over 24 years of Python releases. We include Dropbox's
+Python 2.5 bytecode and some PyPy bytecodes.
 
 Why this?
 ---------
@@ -44,14 +48,15 @@ not exist and there is just bytecode. Again, my debuggers make use of
 this.
 
 There were (and still are) a number of decompyle, uncompyle,
-uncompyle2, uncompyle3 forks around. Almost all of them come basically
-from the same code base, and (almost?) all of them are no longer
-actively maintained. One was really good at decompiling Python 1.5-2.3
-or so, another really good at Python 2.7, but that only. Another
-handles Python 3.2 only; another patched that and handled only 3.3.
-You get the idea. This code pulls all of these forks together and
-*moves forward*. There is some serious refactoring and cleanup in this
-code base over those old forks.
+uncompyle2, uncompyle3 forks around. Many of them come basically from
+the same code base, and (almost?) all of them are no longer actively
+maintained. One was really good at decompiling Python 1.5-2.3, another
+really good at Python 2.7, but that only. Another handles Python 3.2
+only; another patched that and handled only 3.3.  You get the
+idea. This code pulls all of these forks together and *moves
+forward*. There is some serious refactoring and cleanup in this code
+base over those old forks. Even more experimental refactoring is going
+on in decompyle3_.
 
 This demonstrably does the best in decompiling Python across all
 Python versions. And even when there is another project that only
@@ -65,36 +70,55 @@ are syntactically correct by running the Python interpreter for that
 bytecode version.  Finally, in cases where the program has a test for
 itself, we can run the check on the decompiled code.
 
-We are serious about testing, and use automated processes to find
-bugs. In the issue trackers for other decompilers, you will find a
-number of bugs we've found along the way. Very few to none of them are
-fixed in the other decompilers.
+We use an automated processes to find bugs. In the issue trackers for
+other decompilers, you will find a number of bugs we've found along
+the way. Very few to none of them are fixed in the other decompilers.
 
 Requirements
 ------------
 
-The code here can be run on Python versions 2.6 or later, PyPy 3-2.4,
-or PyPy-5.0.1.  Python versions 2.4-2.7 are supported in the
-python-2.4 branch.  The bytecode files it can read have been tested on
-Python bytecodes from versions 1.4, 2.1-2.7, and 3.0-3.6 and the
-above-mentioned PyPy versions.
+The code in the git repository can be run from Python 2.4 to the
+latest Python version, with the exception of Python 3.0 through
+3.2. Volunteers are welcome to address these deficiencies if there a
+desire to do so.
+
+The way it does this though is by segregating consecutive Python versions into
+git branches:
+
+master
+   Python 3.6 and up (uses type annotations)
+python-3.3-to-3.5
+   Python 3.3 through 3.5 (Generic Python 3)
+python-2.4
+   Python 2.4 through 2.7 (Generic Python 2)
+
+PyPy 3-2.4 and later works as well.
+
+The bytecode files it can read have been tested on Python
+bytecodes from versions 1.4, 2.1-2.7, and 3.0-3.8 and later PyPy
+versions.
 
 Installation
 ------------
 
-This uses setup.py, so it follows the standard Python routine:
+You can install from PyPI using the name ``uncompyle6``::
 
-::
+    pip install uncompyle6
 
-    pip install -e .  # set up to run from source tree
-                      # Or if you want to install instead
-    python setup.py install # may need sudo
 
-A GNU makefile is also provided so :code:`make install` (possibly as root or
+To install from source code, this project uses setup.py, so it follows the standard Python routine::
+
+    $ pip install -e .  # set up to run from source tree
+
+or::
+
+    $ python setup.py install # may need sudo
+
+A GNU Makefile is also provided so :code:`make install` (possibly as root or
 sudo) will do the steps above.
 
-Testing
--------
+Running Tests
+-------------
 
 ::
 
@@ -122,16 +146,39 @@ For usage help:
 
    $ uncompyle6 -h
 
-If you want strong verification of the correctness of the
-decompilation process, add the `--verify` option. But there are
-situations where this will indicate a failure, although the generated
-program is semantically equivalent. Using option `--weak-verify` will
-tell you if there is something definitely wrong. Generally, large
-swaths of code are decompiled correctly, if not the entire program.
+Verification
+------------
 
-You can also cross compare the results with pycdc_ . Since they work
-differently, bugs here often aren't in that, and vice versa.
+In older versions of Python it was possible to verify bytecode by
+decompiling bytecode, and then compiling using the Python interpreter
+for that bytecode version. Having done this, the bytecode produced
+could be compared with the original bytecode. However as Python's code
+generation got better, this no longer was feasible.
 
+If you want Python syntax verification of the correctness of the
+decompilation process, add the :code:`--syntax-verify` option. However since
+Python syntax changes, you should use this option if the bytecode is
+the right bytecode for the Python interpreter that will be checking
+the syntax.
+
+You can also cross compare the results with either another version of
+`uncompyle6` since there are are sometimes regressions in decompiling
+specific bytecode as the overall quality improves.
+
+For Python 3.7 and 3.8, the code in decompyle3_ is generally
+better.
+
+Or try specific another python decompiler like uncompyle2_, unpyc37_,
+or pycdc_.  Since the later two work differently, bugs here often
+aren't in that, and vice versa.
+
+There is an interesting class of these programs that is readily
+available give stronger verification: those programs that when run
+test themselves. Our test suite includes these.
+
+And Python comes with another a set of programs like this: its test
+suite for the standard library. We have some code in :code:`test/stdlib` to
+facilitate this kind of checking too.
 
 Known Bugs/Restrictions
 -----------------------
@@ -146,98 +193,125 @@ All of the Python decompilers that I have looked at have problems
 decompiling Python's control flow. In some cases we can detect an
 erroneous decompilation and report that.
 
-In older versions of Python it was possible to verify bytecode by
-decompiling bytecode, and then compiling using the Python interpreter
-for that bytecode version. Having done this the bytecode produced
-could be compared with the original bytecode. However as Python's code
-generation got better, this is no longer feasible.
+Python support is pretty good for Python 2
 
-There verification that we use that doesn't check bytecode for
-equivalence but does check to see if the resulting decompiled source
-is a valid Python program by running the Python interpreter. Because
-the Python language has changed so much, for best results you should
-use the same Python version in checking as was used in creating the
-bytecode.
-
-There are however an interesting class of these programs that is
-readily available give stronger verification: those programs that
-when run check some computation, or even better themselves.
-
-And already Python has a set of programs like this: the test suite
-for the standard library that comes with Python. We have some
-code in `test/stdlib` to facilitate this kind of checking.
-
-Python support is strongest in Python 2 for 2.7 and drops off as you
-get further away from that. Support is also probably pretty good for
-python 2.3-2.4 since a lot of the goodness of early the version of the
-decompiler from that era has been preserved (and Python compilation in
-that era was minimal)
-
-There is some work to do on the lower end Python versions which is
-more difficult for us to handle since we don't have a Python
-interpreter for versions 1.6, and 2.0.
+On the lower end of Python versions, decompilation seems pretty good although
+we don't have any automated testing in place for Python's distributed tests.
+Also, we don't have a Python interpreter for versions 1.6, and 2.0.
 
 In the Python 3 series, Python support is is strongest around 3.4 or
 3.3 and drops off as you move further away from those versions. Python
 3.0 is weird in that it in some ways resembles 2.6 more than it does
 3.1 or 2.7. Python 3.6 changes things drastically by using word codes
 rather than byte codes. As a result, the jump offset field in a jump
-instruction argument has been reduced. This makes the `EXTENDED_ARG`
+instruction argument has been reduced. This makes the :code:`EXTENDED_ARG`
 instructions are now more prevalent in jump instruction; previously
 they had been rare.  Perhaps to compensate for the additional
-`EXTENDED_ARG` instructions, additional jump optimization has been
+:code:`EXTENDED_ARG` instructions, additional jump optimization has been
 added. So in sum handling control flow by ad hoc means as is currently
 done is worse.
 
-Between Python 3.5, 3.6 and 3.7 there have been major changes to the
-`MAKE_FUNCTION` and `CALL_FUNCTION` instructions.
+Between Python 3.5, 3.6, 3.7 there have been major changes to the
+:code:`MAKE_FUNCTION` and :code:`CALL_FUNCTION` instructions.
+
+Python 3.8 removes :code:`SETUP_LOOP`, :code:`SETUP_EXCEPT`,
+:code:`BREAK_LOOP`, and :code:`CONTINUE_LOOP`, instructions which may
+make control-flow detection harder, lacking the more sophisticated
+control-flow analysis that is planned. We'll see.
 
 Currently not all Python magic numbers are supported. Specifically in
 some versions of Python, notably Python 3.6, the magic number has
-changes several times within a version. We support only the released
-magic. There are also customized Python interpreters, notably Dropbox,
-which use their own magic and encrypt bytcode. With the exception of
+changes several times within a version.
+
+**We support only released versions, not candidate versions.** Note
+however that the magic of a released version is usually the same as
+the *last* candidate version prior to release.
+
+There are also customized Python interpreters, notably Dropbox,
+which use their own magic and encrypt bytecode. With the exception of
 the Dropbox's old Python 2.5 interpreter this kind of thing is not
 handled.
 
-We also don't handle PJOrion_ obfuscated code. For that try: PJOrion
-Deobfuscator_ to unscramble the bytecode to get valid bytecode before
-trying this tool. This program can't decompile Microsoft Windows EXE
-files created by Py2EXE_, although we can probably decompile the code
-after you extract the bytecode properly. For situations like this, you
-might want to consider a decompilation service like `Crazy Compilers
-<http://www.crazy-compilers.com/decompyle/>`_.  Handling
-pathologically long lists of expressions or statements is slow.
+We also don't handle PJOrion_ or otherwise obfuscated code. For
+PJOrion try: PJOrion Deobfuscator_ to unscramble the bytecode to get
+valid bytecode before trying this tool; pydecipher_ might help with that.
 
+This program can't decompile Microsoft Windows EXE files created by
+Py2EXE_, although we can probably decompile the code after you extract
+the bytecode properly. `Pydeinstaller <https://github.com/charles-dyfis-net/pydeinstaller>`_ may help with unpacking Pyinstaller bundlers.
 
-There is lots to do, so please dig in and help.
+Handling pathologically long lists of expressions or statements is
+slow. We don't handle Cython_ or MicroPython which don't use bytecode.
+
+There are numerous bugs in decompilation. And that's true for every
+other CPython decompiler I have encountered, even the ones that
+claimed to be "perfect" on some particular version like 2.4.
+
+As Python progresses decompilation also gets harder because the
+compilation is more sophisticated and the language itself is more
+sophisticated. I suspect that attempts there will be fewer ad-hoc
+attempts like unpyc37_ (which is based on a 3.3 decompiler) simply
+because it is harder to do so. The good news, at least from my
+standpoint, is that I think I understand what's needed to address the
+problems in a more robust way. But right now until such time as
+project is better funded, I do not intend to make any serious effort
+to support Python versions 3.8 or 3.9, including bugs that might come
+in. I imagine at some point I may be interested in it.
+
+You can easily find bugs by running the tests against the standard
+test suite that Python uses to check itself. At any given time, there are
+dozens of known problems that are pretty well isolated and that could
+be solved if one were to put in the time to do so. The problem is that
+there aren't that many people who have been working on bug fixing.
+
+Some of the bugs in 3.7 and 3.8 are simply a matter of back-porting
+the fixes in decompyle3. Volunteers are welcome to do so.
+
+You may run across a bug, that you want to report. Please do so after
+reading `How to report a bug
+<https://github.com/rocky/python-uncompyle6/blob/master/HOW-TO-REPORT-A-BUG.md>`_ and
+follow the `instructions when opening an issue <https://github.com/rocky/python-uncompyle6/issues/new?assignees=&labels=&template=bug-report.md>`_.
+
+Be aware that it might not get my attention for a while. If you
+sponsor or support the project in some way, I'll prioritize your
+issues above the queue of other things I might be doing instead.
 
 See Also
 --------
 
-* https://github.com/zrax/pycdc : purports to support all versions of Python. It is written in C++ and is most accurate for Python versions around 2.7 and 3.3 when the code was more actively developed. Accuracy for more recent versions of Python 3 and early versions of Python are especially lacking. See its `issue tracker <https://github.com/zrax/pycdc/issues>`_ for details. Currently lightly maintained.
+* https://github.com/rocky/python-decompile3 : Much smaller and more modern code, focusing on 3.7 and 3.8. Changes in that will get migrated back here.
 * https://code.google.com/archive/p/unpyc3/ : supports Python 3.2 only. The above projects use a different decompiling technique than what is used here. Currently unmaintained.
 * https://github.com/figment/unpyc3/ : fork of above, but supports Python 3.3 only. Includes some fixes like supporting function annotations. Currently unmaintained.
-* https://github.com/wibiti/uncompyle2 : supports Python 2.7 only, but does that fairly well. There situtations where `uncompyle6` results are incorrect while `uncompyle2` results are not, but more often uncompyle6 is correct when uncompyle2 is not. Because `uncompyle6` adheres to accuracy over idiomatic Python, `uncompyle2` can produce more natural-looking code when it is correct. Currently `uncompyle2` is lightly maintained. See its issue `tracker <https://github.com/wibiti/uncompyle2/issues>`_ for more details
+* https://github.com/wibiti/uncompyle2 : supports Python 2.7 only, but does that fairly well. There are situations where :code:`uncompyle6` results are incorrect while :code:`uncompyle2` results are not, but more often uncompyle6 is correct when uncompyle2 is not. Because :code:`uncompyle6` adheres to accuracy over idiomatic Python, :code:`uncompyle2` can produce more natural-looking code when it is correct. Currently :code:`uncompyle2` is lightly maintained. See its issue `tracker <https://github.com/wibiti/uncompyle2/issues>`_ for more details.
 * `How to report a bug <https://github.com/rocky/python-uncompyle6/blob/master/HOW-TO-REPORT-A-BUG.md>`_
 * The HISTORY_ file.
 * https://github.com/rocky/python-xdis : Cross Python version disassembler
 * https://github.com/rocky/python-xasm : Cross Python version assembler
 * https://github.com/rocky/python-uncompyle6/wiki : Wiki Documents which describe the code and aspects of it in more detail
+* https://github.com/zrax/pycdc : The README for this C++ code says it aims to support all versions of Python. You can aim your slign shot for the moon too, but I doubt you are going to hit it. This code is best for Python versions around 2.7 and 3.3 when the code was initially developed. Accuracy for current versions of Python3 and early versions of Python is lacking. Without major effort, it is unlikely it can be made to support current Python 3. See its `issue tracker <https://github.com/zrax/pycdc/issues>`_ for details. Currently lightly maintained.
 
 
-.. _trepan: https://pypi.python.org/pypi/trepan2
-.. _compiler: https://pypi.python.org/pypi/spark_parser
+.. _Cython: https://en.wikipedia.org/wiki/Cython
+.. _trepan: https://pypi.python.org/pypi/trepan3k
+.. _compiler: https://github.com/rocky/python-uncompyle6/wiki/How-does-this-code-work%3F
 .. _HISTORY: https://github.com/rocky/python-uncompyle6/blob/master/HISTORY.md
+.. _report_bug: https://github.com/rocky/python-uncompyle6/blob/master/HOW-TO-REPORT-A-BUG.md
 .. _debuggers: https://pypi.python.org/pypi/trepan3k
 .. _remake: https://bashdb.sf.net/remake
 .. _pycdc: https://github.com/zrax/pycdc
+.. _decompyle3: https://github.com/rocky/python-decompile3
+.. _uncompyle2: https://github.com/wibiti/uncompyle2
+.. _unpyc37: https://github.com/andrew-tavera/unpyc37
 .. _this: https://github.com/rocky/python-uncompyle6/wiki/Deparsing-technology-and-its-use-in-exact-location-reporting
 .. |buildstatus| image:: https://travis-ci.org/rocky/python-uncompyle6.svg
 		 :target: https://travis-ci.org/rocky/python-uncompyle6
+.. |packagestatus| image:: https://repology.org/badge/vertical-allrepos/python:uncompyle6.svg
+		 :target: https://repology.org/project/python:uncompyle6/versions
 .. _PJOrion: http://www.koreanrandom.com/forum/topic/15280-pjorion-%D1%80%D0%B5%D0%B4%D0%B0%D0%BA%D1%82%D0%B8%D1%80%D0%BE%D0%B2%D0%B0%D0%BD%D0%B8%D0%B5-%D0%BA%D0%BE%D0%BC%D0%BF%D0%B8%D0%BB%D1%8F%D1%86%D0%B8%D1%8F-%D0%B4%D0%B5%D0%BA%D0%BE%D0%BC%D0%BF%D0%B8%D0%BB%D1%8F%D1%86%D0%B8%D1%8F-%D0%BE%D0%B1%D1%84
+.. _pydecipher: https://github.com/mitre/pydecipher
 .. _Deobfuscator: https://github.com/extremecoders-re/PjOrion-Deobfuscator
 .. _Py2EXE: https://en.wikipedia.org/wiki/Py2exe
 .. |Supported Python Versions| image:: https://img.shields.io/pypi/pyversions/uncompyle6.svg
 .. |Latest Version| image:: https://badge.fury.io/py/uncompyle6.svg
 		 :target: https://badge.fury.io/py/uncompyle6
+.. |Pypi Installs| image:: https://pepy.tech/badge/uncompyle6/month

__pkginfo__.py

@@ -1,4 +1,4 @@
-# Copyright (C) 2018 Rocky Bernstein <rocky@gnu.org>
+# Copyright (C) 2018, 2020-2021 Rocky Bernstein <rocky@gnu.org>
 #
 #  This program is free software: you can redistribute it and/or modify
 #  it under the terms of the GNU General Public License as published by
@@ -21,65 +21,88 @@
 # less elegant than having it here with reduced code, albeit there
 # still is some room for improvement.
 
+# Python-version | package | last-version |
+# -----------------------------------------
+# 2.5            | pip     |  1.1         |
+# 2.6            | pip     |  1.5.6       |
+# 2.7            | pip     | 19.2.3       |
+# 2.7            | pip     |  1.2.1       |
+# 3.1            | pip     |  1.5.6       |
+# 3.2            | pip     |  7.1.2       |
+# 3.3            | pip     | 10.0.1       |
+# 3.4            | pip     | 19.1.1       |
+
 # Things that change more often go here.
-copyright   = """
-Copyright (C) 2015-2018 Rocky Bernstein <rb@dustyfeet.com>.
+copyright = """
+Copyright (C) 2015-2021 Rocky Bernstein <rb@dustyfeet.com>.
 """
 
-classifiers =  ['Development Status :: 5 - Production/Stable',
-                'Intended Audience :: Developers',
-                'License :: OSI Approved :: GNU General Public License v3 (GPLv3)',
-                'Operating System :: OS Independent',
-                'Programming Language :: Python',
-                'Programming Language :: Python :: 2.4',
-                'Programming Language :: Python :: 2.5',
-                'Programming Language :: Python :: 2.6',
-                'Programming Language :: Python :: 2.7',
-                'Programming Language :: Python :: 3.0',
-                'Programming Language :: Python :: 3.1',
-                'Programming Language :: Python :: 3.2',
-                'Programming Language :: Python :: 3.3',
-                'Programming Language :: Python :: 3.4',
-                'Programming Language :: Python :: 3.5',
-                'Programming Language :: Python :: 3.6',
-                'Programming Language :: Python :: 3.7',
-                'Topic :: Software Development :: Debuggers',
-                'Topic :: Software Development :: Libraries :: Python Modules',
-                ]
+classifiers = [
+    "Development Status :: 5 - Production/Stable",
+    "Intended Audience :: Developers",
+    "License :: OSI Approved :: GNU General Public License v3 (GPLv3)",
+    "Operating System :: OS Independent",
+    "Programming Language :: Python",
+    "Programming Language :: Python :: 2",
+    "Programming Language :: Python :: 2.4",
+    "Programming Language :: Python :: 2.5",
+    "Programming Language :: Python :: 2.6",
+    "Programming Language :: Python :: 2.7",
+    "Programming Language :: Python :: 3",
+    "Programming Language :: Python :: 3.0",
+    "Programming Language :: Python :: 3.1",
+    "Programming Language :: Python :: 3.2",
+    "Programming Language :: Python :: 3.3",
+    "Programming Language :: Python :: 3.4",
+    "Programming Language :: Python :: 3.5",
+    "Programming Language :: Python :: 3.6",
+    "Programming Language :: Python :: 3.7",
+    "Programming Language :: Python :: 3.8",
+    "Programming Language :: Python :: 3.9",
+    "Programming Language :: Python :: 3.10",
+    "Programming Language :: Python :: Implementation :: PyPy",
+    "Topic :: Software Development :: Debuggers",
+    "Topic :: Software Development :: Libraries :: Python Modules",
+]
 
 # The rest in alphabetic order
-author             = "Rocky Bernstein, Hartmut Goebel, John Aycock, and others"
-author_email       = "rb@dustyfeet.com"
-entry_points       = {
-    'console_scripts': [
-        'uncompyle6=uncompyle6.bin.uncompile:main_bin',
-        'pydisassemble=uncompyle6.bin.pydisassemble:main',
-    ]}
-ftp_url            = None
-install_requires   = ['spark-parser >= 1.8.5, < 1.9.0',
-                      'xdis >= 3.8.4, < 3.9.0']
+author = "Rocky Bernstein, Hartmut Goebel, John Aycock, and others"
+author_email = "rb@dustyfeet.com"
+entry_points = {
+    "console_scripts": [
+        "uncompyle6=uncompyle6.bin.uncompile:main_bin",
+        "pydisassemble=uncompyle6.bin.pydisassemble:main",
+    ]
+}
+ftp_url = None
+install_requires = ["spark-parser >= 1.8.9, < 1.9.0", "xdis >= 6.0.2, < 6.1.0"]
 
-license            = 'GPL3'
-mailing_list       = 'python-debugger@googlegroups.com'
-modname            = 'uncompyle6'
-py_modules         = None
-short_desc         = 'Python cross-version byte-code decompiler'
-web                = 'https://github.com/rocky/python-uncompyle6/'
+license = "GPL3"
+mailing_list = "python-debugger@googlegroups.com"
+modname = "uncompyle6"
+py_modules = None
+short_desc = "Python cross-version byte-code decompiler"
+web = "https://github.com/rocky/python-uncompyle6/"
 
 # tracebacks in zip files are funky and not debuggable
 zip_safe = True
 
 
 import os.path
+
+
 def get_srcdir():
     filename = os.path.normcase(os.path.dirname(os.path.abspath(__file__)))
     return os.path.realpath(filename)
 
+
 srcdir = get_srcdir()
 
+
 def read(*rnames):
     return open(os.path.join(srcdir, *rnames)).read()
 
-# Get info from files; set: long_description and VERSION
-long_description   = ( read("README.rst") + '\n' )
-exec(read('uncompyle6/version.py'))
+
+# Get info from files; set: long_description and __version__
+long_description = read("README.rst") + "\n"
+exec(read("uncompyle6/version.py"))

admin-tools/README.md

@@ -8,4 +8,15 @@ They are customized to my environment:
 - I have git repos for xdis, and spark parser at the same level as uncompyle6
 
 There may be other rocky-specific things that need customization.
-how-to-make-a-release.txt has overall how I make a release
+how-to-make-a-release.md has overall how I make a release
+
+Since this project uses python over a wide variety of release, some versions
+of projects that should be used for specific Python versions
+
+for 3.2.6:
+   pytest==2.9.2
+
+for 3.1.5
+   pytset==2.1.0
+   py=1.8.0 and comment out line 10 of _builtin.py # callable = callable
+   six==1.10.0

admin-tools/check-2.4-2.7-versions.sh

@@ -6,7 +6,7 @@ owd=$(pwd)
 trap finish EXIT
 
 cd $(dirname ${BASH_SOURCE[0]})
-if ! source ./pyenv-older-versions ; then
+if ! source ./pyenv-2.4-2.7-versions ; then
     exit $?
 fi
 if ! source ./setup-python-2.4.sh ; then

admin-tools/check-3.3-3.5-versions.sh

@@ -0,0 +1,27 @@
+#!/bin/bash
+function finish {
+  cd $owd
+}
+owd=$(pwd)
+trap finish EXIT
+
+cd $(dirname ${BASH_SOURCE[0]})
+if ! source ./pyenv-3.3-3.5-versions ; then
+    exit $?
+fi
+if ! source ./setup-python-3.3.sh ; then
+    exit $?
+fi
+
+cd ..
+for version in $PYVERSIONS; do
+    echo --- $version ---
+    if ! pyenv local $version ; then
+	exit $?
+    fi
+    make clean && python setup.py develop
+    if ! make check ; then
+	exit $?
+    fi
+    echo === $version ===
+done

admin-tools/check-newest-versions.sh

@@ -8,7 +8,7 @@ owd=$(pwd)
 trap finish EXIT
 
 cd $(dirname ${BASH_SOURCE[0]})
-if ! source ./pyenv-newer-versions ; then
+if ! source ./pyenv-newest-versions ; then
     exit $?
 fi
 if ! source ./setup-master.sh ; then
@@ -21,8 +21,9 @@ for version in $PYVERSIONS; do
 	exit $?
     fi
     make clean && pip install -e .
-    if ! make check; then
+    if ! make check-short; then
 	exit $?
     fi
     echo === $version ===
 done
+make check

admin-tools/how-to-make-a-release.md

@@ -2,17 +2,17 @@
 **Table of Contents**
 
 - [Get latest sources:](#get-latest-sources)
-- [Change version in uncompyle6/version.py](#change-version-in-uncompyle6versionpy)
+- [Change version in uncompyle6/version.py:](#change-version-in-uncompyle6versionpy)
 - [Update ChangeLog:](#update-changelog)
-- [Update NEWS from ChangeLog:](#update-news-from-changelog)
+- [Update NEWS.md from ChangeLog:](#update-newsmd-from-changelog)
 - [Make sure pyenv is running and check newer versions](#make-sure-pyenv-is-running-and-check-newer-versions)
 - [Switch to python-2.4, sync that up and build that first since it creates a tarball which we don't want.](#switch-to-python-24-sync-that-up-and-build-that-first-since-it-creates-a-tarball-which-we-dont-want)
-- [Update NEWS from master branch](#update-news-from-master-branch)
-- [Check against all versions](#check-against-all-versions)
+- [Check against older versions](#check-against-older-versions)
 - [Make packages and tag](#make-packages-and-tag)
-- [Upload single package and look at Rst Formating](#upload-single-package-and-look-at-rst-formating)
-- [Upload rest of versions](#upload-rest-of-versions)
-- [Push tags:](#push-tags)
+- [Check package on github](#check-package-on-github)
+- [Release on Github](#release-on-github)
+- [Get onto PyPI](#get-onto-pypi)
+- [Update tags:](#update-tags)
 
 <!-- markdown-toc end -->
 # Get latest sources:
@@ -30,16 +30,16 @@
 
     $ make ChangeLog
 
-#  Update NEWS from ChangeLog:
+#  Update NEWS.md from ChangeLog:
 
-    $ emacs NEWS
+    $ emacs NEWS.md
     $ make check
     $ git commit --amend .
     $ git push   # get CI testing going early
 
 # Make sure pyenv is running and check newer versions
 
-    $ pyenv local && source admin-tools/check-newer-versions.sh
+    $ admin-tools/check-newer-versions.sh
 
 # Switch to python-2.4, sync that up and build that first since it creates a tarball which we don't want.
 
@@ -50,34 +50,51 @@
 
 # Check against older versions
 
-    $ source admin-tools/check-older-versions.sh
+    $ admin-tools/check-older-versions.sh
 
 # Make packages and tag
 
     $ . ./admin-tools/make-dist-older.sh
+	$ pyenv local 3.8.5
+	$ twine check dist/uncompyle6-$VERSION*
     $ git tag release-python-2.4-$VERSION
+    $ ./admin-tools/make-dist-newer.sh
+	$ twine check dist/uncompyle6-$VERSION*
 
-    $ . ./admin-tools/make-dist-newer.sh
-    $ git tag release-$VERSION
+# Check package on github
 
-# Upload single package and look at Rst Formating
+	$ [[ ! -d /tmp/gittest ]] && mkdir /tmp/gittest; pushd /tmp/gittest
+	$ pyenv local 3.8.3
+	$ pip install -e git://github.com/rocky/python-uncompyle6.git#egg=uncompyle6
+	$ uncompyle6 --help
+	$ pip uninstall uncompyle6
+	$ popd
 
-    $ twine upload dist/uncompyle6-${VERSION}-py3.3.egg
+# Release on Github
 
-# Upload rest of versions
+Goto https://github.com/rocky/python-uncompyle6/releases
+
+Now check the *tagged* release. (Checking the untagged release was previously done).
+
+Todo: turn this into a script in `admin-tools`
+
+	$ pushd /tmp/gittest
+	$ pip install -e git://github.com/rocky/python-uncompyle6.git@$VERSION#egg=uncompyle6
+	$ uncompyle6 --help
+	$ pip uninstall uncompyle6
+	$ popd
+
+
+# Get onto PyPI
 
     $ twine upload dist/uncompyle6-${VERSION}*
 
-# Push tags:
+
+# Update tags:
 
     $ git push --tags
+    $ git pull --tags
 
-# Check on a VM
+# Move dist files to uploaded
 
-    $ cd /virtual/vagrant/virtual/vagrant/ubuntu-zesty
-	$ vagrant up
-	$ vagrant ssh
-	$ pyenv local 3.5.2
-	$ pip install --upgrade uncompyle6
-	$ exit
-	$ vagrant halt
+	$ mv -v dist/uncompyle6-${VERSION}* dist/uploaded

admin-tools/make-dist-2.4-2.7.sh

@@ -9,7 +9,7 @@ owd=$(pwd)
 trap finish EXIT
 
 cd $(dirname ${BASH_SOURCE[0]})
-if ! source ./pyenv-older-versions ; then
+if ! source ./pyenv-2.4-2.7-versions ; then
     exit $?
 fi
 if ! source ./setup-python-2.4.sh ; then
@@ -18,7 +18,7 @@ fi
 
 cd ..
 source $PACKAGE/version.py
-echo $VERSION
+echo $__version__
 
 for pyversion in $PYVERSIONS; do
     if ! pyenv local $pyversion ; then
@@ -29,11 +29,15 @@ for pyversion in $PYVERSIONS; do
     python setup.py bdist_egg
 done
 
+pyenv local 2.7.18
+python setup.py bdist_wheel
+mv -v dist/${PACKAGE}-$__version__-py2{.py3,}-none-any.whl
+
 # Pypi can only have one source tarball.
 # Tarballs can get created from the above setup, so make sure to remove them since we want
 # the tarball from master.
 
-tarball=dist/${PACKAGE}-$VERSION-tar.gz
+tarball=dist/${PACKAGE}-${__version_}_-tar.gz
 if [[ -f $tarball ]]; then
-    rm -v dist/${PACKAGE}-$VERSION-tar.gz
+    rm -v dist/${PACKAGE}-${__version__}-tar.gz
 fi

admin-tools/make-dist-3.3-3.5.sh

@@ -0,0 +1,38 @@
+#!/bin/bash
+PACKAGE=uncompyle6
+
+# FIXME put some of the below in a common routine
+function finish {
+  cd $owd
+}
+
+cd $(dirname ${BASH_SOURCE[0]})
+owd=$(pwd)
+trap finish EXIT
+
+if ! source ./pyenv-3.3-3.5-versions ; then
+    exit $?
+fi
+if ! source ./setup-python-3.3.sh ; then
+    exit $?
+fi
+
+cd ..
+source $PACKAGE/version.py
+echo $__version__
+
+for pyversion in $PYVERSIONS; do
+    if ! pyenv local $pyversion ; then
+	exit $?
+    fi
+    # pip bdist_egg create too-general wheels. So
+    # we narrow that by moving the generated wheel.
+
+    # Pick out first two number of version, e.g. 3.5.1 -> 35
+    first_two=$(echo $pyversion | cut -d'.' -f 1-2 | sed -e 's/\.//')
+    rm -fr build
+    python setup.py bdist_egg bdist_wheel
+    mv -v dist/${PACKAGE}-$__version__-{py2.py3,py$first_two}-none-any.whl
+done
+
+python ./setup.py sdist

admin-tools/make-dist-newest.sh

@@ -10,7 +10,7 @@ cd $(dirname ${BASH_SOURCE[0]})
 owd=$(pwd)
 trap finish EXIT
 
-if ! source ./pyenv-newer-versions ; then
+if ! source ./pyenv-newest-versions ; then
     exit $?
 fi
 if ! source ./setup-master.sh ; then
@@ -19,7 +19,7 @@ fi
 
 cd ..
 source $PACKAGE/version.py
-echo $VERSION
+echo $__version__
 
 for pyversion in $PYVERSIONS; do
     if ! pyenv local $pyversion ; then
@@ -32,7 +32,7 @@ for pyversion in $PYVERSIONS; do
     first_two=$(echo $pyversion | cut -d'.' -f 1-2 | sed -e 's/\.//')
     rm -fr build
     python setup.py bdist_egg bdist_wheel
-    mv -v dist/${PACKAGE}-$VERSION-{py2.py3,py$first_two}-none-any.whl
+    mv -v dist/${PACKAGE}-$__version__-{py2.py3,py$first_two}-none-any.whl
 done
 
 python ./setup.py sdist

admin-tools/pyenv-2.4-2.7-versions

@@ -0,0 +1,9 @@
+# -*- shell-script -*-
+# Sets PYVERSIONS to be pyenv versions that
+# we can use in the python-2.4-to-2.7 branch.
+
+if [[ $0 == ${BASH_SOURCE[0]} ]] ; then
+    echo "This script should be *sourced* rather than run directly through bash"
+    exit 1
+fi
+export PYVERSIONS='2.4.6 2.5.6 2.6.9 2.7.18'

admin-tools/pyenv-3.0-3.2-versions

@@ -6,4 +6,4 @@ if [[ $0 == ${BASH_SOURCE[0]} ]] ; then
     echo "This script should be *sourced* rather than run directly through bash"
     exit 1
 fi
-export PYVERSIONS='2.4.6 2.5.6'
+export PYVERSIONS='3.0.1 3.1.5 3.2.6'

admin-tools/pyenv-3.3-3.5-versions

@@ -0,0 +1,8 @@
+# -*- shell-script -*-
+# Sets PYVERSIONS to be pyenv versions that
+# we can use in the python-3.3-to-3.5 branch.
+if [[ $0 == ${BASH_SOURCE[0]} ]] ; then
+    echo "This script should be *sourced* rather than run directly through bash"
+    exit 1
+fi
+export PYVERSIONS='3.5.10 3.3.7 3.4.10'

admin-tools/pyenv-newest-versions

@@ -0,0 +1,8 @@
+# -*- shell-script -*-
+# Sets PYVERSIONS to be pyenv versions that
+# we can use in the master branch.
+if [[ $0 == ${BASH_SOURCE[0]} ]] ; then
+    echo "This script should be *sourced* rather than run directly through bash"
+    exit 1
+fi
+export PYVERSIONS='3.6.15 pypy3.6-7.3.1 3.7.16 pypy3.7-7.3.9 pypy3.8-7.3.10 pyston-2.3.5 3.8.16'

admin-tools/pyenv-versions

@@ -5,4 +5,4 @@ if [[ $0 == ${BASH_SOURCE[0]} ]] ; then
     echo "This script should be *sourced* rather than run directly through bash"
     exit 1
 fi
-export PYVERSIONS='3.5.5 3.6.5 2.6.9 3.3.7 2.7.14 3.2.6 3.1.5 3.4.8'
+export PYVERSIONS='3.7.13 pyston-2.3.3 3.8.13'

admin-tools/setup-master.sh

@@ -1,5 +1,14 @@
 #!/bin/bash
-PYTHON_VERSION=3.6.5
+PYTHON_VERSION=3.7.16
+
+function checkout_version {
+    local repo=$1
+    version=${2:-master}
+    echo Checking out $version on $repo ...
+    (cd ../$repo && git checkout $version && pyenv local $PYTHON_VERSION) && \
+	git pull
+    return $?
+}
 
 # FIXME put some of the below in a common routine
 function finish {
@@ -16,7 +25,7 @@ fi
 mydir=$(dirname $bs)
 fulldir=$(readlink -f $mydir)
 cd $fulldir/..
-(cd ../python-spark && git checkout master && pyenv local $PYTHON_VERSION) && git pull && \
-    (cd ../python-xdis && git checkout master && pyenv local $PYTHON_VERSION) && git pull && \
-    git checkout master && pyenv local $PYTHON_VERSION && git pull
+(cd $fulldir/.. && checkout_version python-spark && checkout_version python-xdis &&
+      checkout_version python-uncompyle6)
 cd $owd
+rm -v */.python-version >/dev/null 2>&1 || true

admin-tools/setup-python-2.4.sh

@@ -1,6 +1,15 @@
 #!/bin/bash
 PYTHON_VERSION=2.4.6
 
+function checkout_version {
+    local repo=$1
+    version=${2:-python-2.4}
+    echo Checking out $version.4 on $repo ...
+    (cd ../$repo && git checkout $version && pyenv local $PYTHON_VERSION) && \
+	git pull
+    return $?
+}
+
 owd=$(pwd)
 bs=${BASH_SOURCE[0]}
 if [[ $0 == $bs ]] ; then
@@ -9,8 +18,7 @@ if [[ $0 == $bs ]] ; then
 fi
 mydir=$(dirname $bs)
 fulldir=$(readlink -f $mydir)
-cd $fulldir/..
-(cd ../python-spark && git checkout python-2.4 && pyenv local $PYTHON_VERSION) && git pull && \
-    (cd ../python-xdis && git checkout python-2.4 && pyenv local $PYTHON_VERSION) && git pull && \
-    git checkout python-2.4 &&  pyenv local $PYTHON_VERSION && git pull
+(cd $fulldir/.. && checkout_version python-spark && checkout_version python-xdis python-2.4-to-2.7 &&
+      checkout_version python-uncompyle6)
 cd $owd
+rm -v */.python-version || true

admin-tools/setup-python-3.0.sh

@@ -0,0 +1,35 @@
+#!/bin/bash
+PYTHON_VERSION=3.0.1
+pyenv local $PYTHON_VERSION
+
+# FIXME put some of the below in a common routine
+function checkout_version {
+    local repo=$1
+    version=${2:-python-3.0-to-3.2}
+    echo Checking out $version on $repo ...
+    (cd ../$repo && git checkout $version && pyenv local $PYTHON_VERSION) && \
+	git pull
+    return $?
+}
+
+function finish {
+  cd $owd
+}
+
+export PATH=$HOME/.pyenv/bin/pyenv:$PATH
+owd=$(pwd)
+bs=${BASH_SOURCE[0]}
+if [[ $0 == $bs ]] ; then
+    echo "This script should be *sourced* rather than run directly through bash"
+    exit 1
+fi
+
+mydir=$(dirname $bs)
+fulldir=$(readlink -f $mydir)
+cd $fulldir/..
+(cd $fulldir/.. && checkout_version python-spark master && checkout_version python-xdis &&
+      checkout_version python-uncompyle6)
+cd $owd
+rm -v */.python-version || true
+
+git checkout python-3.0-to-3.2  && git pull && pyenv local $PYTHON_VERSION

admin-tools/setup-python-3.3.sh

@@ -0,0 +1,35 @@
+#!/bin/bash
+PYTHON_VERSION=3.3.7
+pyenv local $PYTHON_VERSION
+
+# FIXME put some of the below in a common routine
+function checkout_version {
+    local repo=$1
+    version=${2:-python-3.3-to-3.5}
+    echo Checking out $version on $repo ...
+    (cd ../$repo && git checkout $version && pyenv local $PYTHON_VERSION) && \
+	git pull
+    return $?
+}
+
+function finish {
+  cd $owd
+}
+
+export PATH=$HOME/.pyenv/bin/pyenv:$PATH
+owd=$(pwd)
+bs=${BASH_SOURCE[0]}
+if [[ $0 == $bs ]] ; then
+    echo "This script should be *sourced* rather than run directly through bash"
+    exit 1
+fi
+
+mydir=$(dirname $bs)
+fulldir=$(readlink -f $mydir)
+cd $fulldir/..
+(cd $fulldir/.. && checkout_version python-spark master && checkout_version python-xdis &&
+      checkout_version python-uncompyle6)
+cd $owd
+rm -v */.python-version || true
+
+git checkout python-3.3-to-3.5  && git pull && pyenv local $PYTHON_VERSION

appveyor.yml

@@ -1,78 +0,0 @@
-environment:
-  global:
-    # SDK v7.0 MSVC Express 2008's SetEnv.cmd script will fail if the
-    # /E:ON and /V:ON options are not enabled in the batch script intepreter
-    # See: http://stackoverflow.com/a/13751649/163740
-    CMD_IN_ENV: "cmd /E:ON /V:ON /C .\\appveyor\\run_with_env.cmd"
-
-  matrix:
-
-    # Pre-installed Python versions, which Appveyor may upgrade to
-    # a later point release.
-    # See: http://www.appveyor.com/docs/installed-software#python
-
-    # - PYTHON: "C:\\Python27"
-    #   PYTHON_VERSION: "2.7.x"
-    #   PYTHON_ARCH: "32"
-
-    - PYTHON: "C:\\Python27-x64"
-      PYTHON_VERSION: "2.7.x"
-      PYTHON_ARCH: "64"
-
-    # - PYTHON: "C:\\Python26"
-    #   PYTHON_VERSION: "2.6.x"
-    #   PYTHON_ARCH: "32"
-
-    # - PYTHON: "C:\\Python26-x64"
-    #   PYTHON_VERSION: "2.6.x"
-    #   PYTHON_ARCH: "64"
-
-install:
-  # We need wheel installed to build wheels
-  - "%PYTHON%\\python.exe -m pip install wheel"
-
-  # Install Python (from the official .msi of http://python.org) and pip when
-  # not already installed.
-  - ps: if (-not(Test-Path($env:PYTHON))) { & appveyor\install.ps1 }
-
-  # Prepend newly installed Python to the PATH of this build (this cannot be
-  # done from inside the powershell script as it would require to restart
-  # the parent CMD process).
-  - "SET PATH=%PYTHON%;%PYTHON%\\Scripts;%PATH%"
-  - "SET HOME=."
-
-  # Check that we have the expected version and architecture for Python
-  - "python --version"
-  - "python -c \"import struct; print(struct.calcsize('P') * 8)\""
-
-  # Upgrade to the latest version of pip to avoid it displaying warnings
-  # about it being out of date.
-  - "%PYTHON%\\python.exe -m pip install --disable-pip-version-check --user --upgrade pip"
-
-  # Install the build dependencies of the project. If some dependencies contain
-  # compiled extensions and are not provided as pre-built wheel packages,
-  # pip will build them from source using the MSVC compiler matching the
-  # target Python version and architecture
-  - "%CMD_IN_ENV% pip install -r requirements.txt"
-
-build_script:
-  # Build the compiled extension
-  - "%CMD_IN_ENV% python setup.py build"
-
-test_script:
-  # Run the project tests
-  - "%CMD_IN_ENV% python test/test_pyenvlib.py --native --weak-verify"
-
-after_test:
-  # If tests are successful, create binary packages for the project.
-  - "%CMD_IN_ENV% python setup.py bdist_wininst"
-  - "%CMD_IN_ENV% python setup.py bdist_msi"
-  - ps: "ls dist"
-
-artifacts:
-  # Archive the generated packages in the ci.appveyor.com build report.
-  - path: dist\*
-
-#on_success:
-#  - TODO: upload the content of dist/*.whl to a public wheelhouse
-#

circle.yml

@@ -1,15 +0,0 @@
-machine:
-    python:
-        version: 2.7.10
-    environment:
-        COMPILE: --compile
-
-dependencies:
-    override:
-        - pip install --upgrade setuptools
-        - pip install -e .
-        - pip install pytest==3.2.5 hypothesis
-test:
-    override:
-        - python ./setup.py develop && make check-2.7
-        - cd ./test/stdlib && pyenv local 2.7.10 && bash ./runtests.sh 'test_[p-z]*.py'

pytest/Makefile

@@ -1,7 +1,11 @@
 PHONY=check test pytest
+SHELL=/bin/bash
 
 PYTHON ?= python
 
 #: Run all tests
 test check pytest:
-	py.test
+	@PYTHON_VERSION=`$(PYTHON) -V 2>&1 | cut -d ' ' -f 2 | cut -d'.' -f1,2`; \
+	 if [[ $$PYTHON_VERSION > 3.2 ]] || [[ $$PYTHON_VERSION == 2.7 ]] || [[ $$PYTHON_VERSION == 2.6 ]]; then \
+		$(PYTHON) -m pytest .; \
+	fi

pytest/test_build_const_key_map.py

@@ -1,10 +1,10 @@
 import pytest
 # uncompyle6
-from uncompyle6 import PYTHON_VERSION
+from xdis.version_info import PYTHON_VERSION_TRIPLE, IS_PYPY
 from validate import validate_uncompyle
 
 
-@pytest.mark.skipif(PYTHON_VERSION < 3.6, reason='need at least python 3.6')
+@pytest.mark.skipif(PYTHON_VERSION_TRIPLE < (3, 6) or IS_PYPY, reason="need at least Python 3.6 and not PyPY")
 @pytest.mark.parametrize('text', (
     "{0.: 'a', -1: 'b'}",   # BUILD_MAP
     "{'a':'b'}",            # BUILD_MAP

pytest/test_deparse.py

@@ -1,6 +1,5 @@
-import pytest
-from uncompyle6.semantics.fragments import code_deparse as deparse, deparsed_find
-from uncompyle6 import PYTHON_VERSION, PYTHON3
+from uncompyle6.semantics.fragments import code_deparse as deparse
+from xdis.version_info import PYTHON_VERSION_TRIPLE
 
 def map_stmts(x, y):
     x = []
@@ -30,20 +29,18 @@ def list_comp():
     [y for y in range(3)]
 
 def get_parsed_for_fn(fn):
-    code = fn.__code__ if PYTHON3 else fn.func_code
-    return deparse(code, version=PYTHON_VERSION)
+    code = fn.__code__
+    return deparse(code, version=PYTHON_VERSION_TRIPLE)
 
 def check_expect(expect, parsed, fn_name):
     debug = False
     i = 2
     max_expect = len(expect)
-    code = get_parsed_for_fn(fn_name)
     for name, offset in sorted(parsed.offsets.keys()):
         assert i+1 <= max_expect, (
             "%s: ran out if items in testing node" % fn_name)
         nodeInfo = parsed.offsets[name, offset]
         node = nodeInfo.node
-        nodeInfo2 = deparsed_find((name, offset), parsed, code)
         extractInfo = parsed.extract_node_info(node)
 
         assert expect[i] == extractInfo.selectedLine, \
@@ -319,5 +316,3 @@ for i in range(2): ...
 .
 """.split("\n")
     parsed = get_parsed_for_fn(for_range_stmt)
-    if not PYTHON3:
-        check_expect(expect, parsed, 'range_stmt')

pytest/test_disasm.py

@@ -1,7 +1,7 @@
 import os.path
 import pytest
 
-from uncompyle6.disas import disassemble_file
+from uncompyle6.code_fns import disassemble_file
 
 def get_srcdir():
     filename = os.path.normcase(os.path.dirname(__file__))

pytest/test_docstring.py

@@ -1,78 +0,0 @@
-import sys
-from uncompyle6 import PYTHON3
-if PYTHON3:
-    from io import StringIO
-    minint = -sys.maxsize-1
-    maxint = sys.maxsize
-else:
-    from StringIO import StringIO
-    minint = -sys.maxint-1
-    maxint = sys.maxint
-from uncompyle6.semantics.helper import print_docstring
-
-class PrintFake():
-    def __init__(self):
-        self.pending_newlines = 0
-        self.f = StringIO()
-
-    def write(self, *data):
-        if (len(data) == 0) or (len(data) == 1 and data[0] == ''):
-            return
-        out = ''.join((str(j) for j in data))
-        n = 0
-        for i in out:
-            if i == '\n':
-                n += 1
-                if n == len(out):
-                    self.pending_newlines = max(self.pending_newlines, n)
-                    return
-            elif n:
-                self.pending_newlines = max(self.pending_newlines, n)
-                out = out[n:]
-                break
-            else:
-                break
-
-        if self.pending_newlines > 0:
-            self.f.write('\n'*self.pending_newlines)
-            self.pending_newlines = 0
-
-        for i in out[::-1]:
-            if i == '\n':
-                self.pending_newlines += 1
-            else:
-                break
-
-        if self.pending_newlines:
-            out = out[:-self.pending_newlines]
-        self.f.write(out)
-    def println(self, *data):
-        if data and not(len(data) == 1 and data[0] ==''):
-            self.write(*data)
-        self.pending_newlines = max(self.pending_newlines, 1)
-        return
-    pass
-
-def test_docstring():
-
-    for doc, expect in (
-            ("Now is the time",
-             '  """Now is the time"""'),
-           ("""
-Now is the time
-""",
-            '''  """
-  Now is the time
-  """''')
-
-            # (r'''func placeholder - ' and with ("""\nstring\n  """)''',
-            #  """  r'''func placeholder - ' and with (\"\"\"\nstring\n\"\"\")'''"""),
-            # (r"""func placeholder - ' and with ('''\nstring\n''') and \"\"\"\nstring\n\"\"\" """,
-            # """  r\"\"\"func placeholder - ' and with ('''\nstring\n''') and \"\"\"\nstring\n\"\"\" \"\"\"""")
-            ):
-
-        o = PrintFake()
-        # print(doc)
-        # print(expect)
-        print_docstring(o, '  ', doc)
-        assert expect == o.f.getvalue()

pytest/test_fjt.py

@@ -1,5 +1,5 @@
 #!/usr/bin/env python
-from uncompyle6 import PYTHON_VERSION, IS_PYPY
+from xdis.version_info import PYTHON_VERSION_TRIPLE, IS_PYPY, version_tuple_to_str
 from uncompyle6.scanner import get_scanner
 def bug(state, slotstate):
     if state:
@@ -21,8 +21,8 @@ def bug_loop(disassemble, tb=None):
 
 def test_if_in_for():
     code = bug.__code__
-    scan = get_scanner(PYTHON_VERSION)
-    if 2.7 <= PYTHON_VERSION <= 3.0 and not IS_PYPY:
+    scan = get_scanner(PYTHON_VERSION_TRIPLE)
+    if (2, 7) <= PYTHON_VERSION_TRIPLE < (3, 1) and not IS_PYPY:
         scan.build_instructions(code)
         fjt = scan.find_jump_targets(False)
 
@@ -51,7 +51,7 @@ def test_if_in_for():
             # previous bug was not mistaking while-loop for if-then
             {'start': 48, 'end': 67, 'type': 'while-loop'}]
 
-    elif 3.2 < PYTHON_VERSION <= 3.4:
+    elif (3, 2) < PYTHON_VERSION_TRIPLE <= (3, 4):
         scan.build_instructions(code)
         fjt  = scan.find_jump_targets(False)
         assert {69: [66], 63: [18]} == fjt
@@ -62,6 +62,6 @@ def test_if_in_for():
            {'end': 59, 'type': 'for-loop', 'start': 31},
            {'end': 63, 'type': 'for-else', 'start': 62}]
     else:
-        print("FIXME: should fix for %s" % PYTHON_VERSION)
+        print("FIXME: should fix for %s" % version_tuple_to_str())
         assert True
     return

pytest/test_fstring.py

@@ -1,154 +0,0 @@
-# std
-# test
-from uncompyle6 import PYTHON_VERSION, deparse_code
-import pytest
-pytestmark = pytest.mark.skipif(PYTHON_VERSION <= 2.6,
-                                reason='hypothesis needs 2.7 or later')
-if PYTHON_VERSION > 2.6:
-
-    import hypothesis
-    from hypothesis import strategies as st
-
-    # uncompyle6
-
-
-    @st.composite
-    def expressions(draw):
-        # todo : would be nice to generate expressions using hypothesis however
-        # this is pretty involved so for now just use a corpus of expressions
-        # from which to select.
-        return draw(st.sampled_from((
-            'abc',
-            'len(items)',
-            'x + 1',
-            'lineno',
-            'container',
-            'self.attribute',
-            'self.method()',
-            # These expressions are failing, I think these are control
-            # flow problems rather than problems with FORMAT_VALUE,
-            # however I need to confirm this...
-            #'sorted(items, key=lambda x: x.name)',
-            #'func(*args, **kwargs)',
-            #'text or default',
-            #'43 if life_the_universe and everything else None'
-        )))
-
-
-    @st.composite
-    def format_specifiers(draw):
-        """
-        Generate a valid format specifier using the rules:
-
-        format_spec ::=  [[fill]align][sign][#][0][width][,][.precision][type]
-        fill        ::=  <any character>
-        align       ::=  "<" | ">" | "=" | "^"
-        sign        ::=  "+" | "-" | " "
-        width       ::=  integer
-        precision   ::=  integer
-        type        ::=  "b" | "c" | "d" | "e" | "E" | "f" | "F" | "g" | "G" | "n" | "o" | "s" | "x" | "X" | "%"
-
-        See https://docs.python.org/2/library/string.html
-
-        :param draw: Let hypothesis draw from other strategies.
-
-        :return: An example format_specifier.
-        """
-        alphabet_strategy = st.characters(min_codepoint=ord('a'), max_codepoint=ord('z'))
-        fill = draw(st.one_of(alphabet_strategy, st.none()))
-        align = draw(st.sampled_from(list('<>=^')))
-        fill_align = (fill + align or '') if fill else ''
-
-        type_ = draw(st.sampled_from('bcdeEfFgGnosxX%'))
-        can_have_sign = type_ in 'deEfFgGnoxX%'
-        can_have_comma = type_ in 'deEfFgG%'
-        can_have_precision = type_ in 'fFgG'
-        can_have_pound = type_ in 'boxX%'
-        can_have_zero = type_ in 'oxX'
-
-        sign = draw(st.sampled_from(list('+- ') + [''])) if can_have_sign else ''
-        pound = draw(st.sampled_from(('#', '',))) if can_have_pound else ''
-        zero = draw(st.sampled_from(('0', '',))) if can_have_zero else ''
-
-        int_strategy = st.integers(min_value=1, max_value=1000)
-
-        width = draw(st.one_of(int_strategy, st.none()))
-        width = str(width) if width is not None else ''
-
-        comma = draw(st.sampled_from((',', '',))) if can_have_comma else ''
-        if can_have_precision:
-            precision = draw(st.one_of(int_strategy, st.none()))
-            precision = '.' + str(precision) if precision else ''
-        else:
-            precision = ''
-
-        return ''.join((fill_align, sign, pound, zero, width, comma, precision, type_,))
-
-
-    @st.composite
-    def fstrings(draw):
-        """
-        Generate a valid f-string.
-        See https://www.python.org/dev/peps/pep-0498/#specification
-
-        :param draw: Let hypothsis draw from other strategies.
-
-        :return: A valid f-string.
-        """
-        character_strategy = st.characters(
-            blacklist_characters='\r\n\'\\s{}',
-            min_codepoint=1,
-            max_codepoint=1000,
-        )
-        is_raw = draw(st.booleans())
-        integer_strategy = st.integers(min_value=0, max_value=3)
-        expression_count = draw(integer_strategy)
-        content = []
-        for _ in range(expression_count):
-            expression = draw(expressions())
-            conversion = draw(st.sampled_from(('', '!s', '!r', '!a',)))
-            has_specifier = draw(st.booleans())
-            specifier = ':' + draw(format_specifiers()) if has_specifier else ''
-            content.append('{{{}{}}}'.format(expression, conversion, specifier))
-            content.append(draw(st.text(character_strategy)))
-        content = ''.join(content)
-        return "f{}'{}'".format('r' if is_raw else '', content)
-
-
-    @pytest.mark.skipif(PYTHON_VERSION < 3.6, reason='need at least python 3.6')
-    @hypothesis.given(format_specifiers())
-    def test_format_specifiers(format_specifier):
-        """Verify that format_specifiers generates valid specifiers"""
-        try:
-            exec('"{:' + format_specifier + '}".format(0)')
-        except ValueError as e:
-            if 'Unknown format code' not in str(e):
-                raise
-
-
-    def run_test(text):
-        hypothesis.assume(len(text))
-        hypothesis.assume("f'{" in text)
-        expr = text + '\n'
-        code = compile(expr, '<string>', 'single')
-        deparsed = deparse_code(PYTHON_VERSION, code, compile_mode='single')
-        recompiled = compile(deparsed.text, '<string>', 'single')
-        if recompiled != code:
-            assert 'dis(' + deparsed.text.strip('\n') + ')' == 'dis(' + expr.strip('\n') + ')'
-
-
-    @pytest.mark.skipif(PYTHON_VERSION < 3.6, reason='need at least python 3.6')
-    @hypothesis.given(fstrings())
-    def test_uncompyle_fstring(fstring):
-        """Verify uncompyling fstring bytecode"""
-        run_test(fstring)
-
-
-    @pytest.mark.skipif(PYTHON_VERSION < 3.6, reason='need at least python 3.6')
-    @pytest.mark.parametrize('fstring', [
-        "f'{abc}{abc!s}'",
-        "f'{abc}0'",
-    ])
-    def test_uncompyle_direct(fstring):
-        """useful for debugging"""
-        run_test(fstring)

pytest/test_function_call.py

@@ -1,185 +0,0 @@
-import string
-from uncompyle6 import PYTHON_VERSION
-import pytest
-pytestmark = pytest.mark.skip(PYTHON_VERSION < 2.7,
-                              reason="need at least Python 2.7")
-
-if PYTHON_VERSION > 2.6:
-    from hypothesis import given, assume, example, settings, strategies as st
-    from validate import validate_uncompyle
-    from test_fstring import expressions
-
-    alpha = st.sampled_from(string.ascii_lowercase)
-    numbers = st.sampled_from(string.digits)
-    alphanum = st.sampled_from(string.ascii_lowercase + string.digits)
-
-
-    @st.composite
-    def function_calls(draw,
-                       min_keyword_args=0, max_keyword_args=5,
-                       min_positional_args=0, max_positional_args=5,
-                       min_star_args=0, max_star_args=1,
-                       min_double_star_args=0, max_double_star_args=1):
-        """
-        Strategy factory for generating function calls.
-
-        :param draw: Callable which draws examples from other strategies.
-
-        :return: The function call text.
-        """
-        st_positional_args = st.lists(
-            alpha,
-            min_size=min_positional_args,
-            max_size=max_positional_args
-        )
-        st_keyword_args = st.lists(
-            alpha,
-            min_size=min_keyword_args,
-            max_size=max_keyword_args
-        )
-        st_star_args = st.lists(
-            alpha,
-            min_size=min_star_args,
-            max_size=max_star_args
-        )
-        st_double_star_args = st.lists(
-            alpha,
-            min_size=min_double_star_args,
-            max_size=max_double_star_args
-        )
-
-        positional_args = draw(st_positional_args)
-        keyword_args = draw(st_keyword_args)
-        st_values = st.lists(
-            expressions(),
-            min_size=len(keyword_args),
-            max_size=len(keyword_args)
-        )
-        keyword_args = [
-            x + '=' + e
-            for x, e in
-            zip(keyword_args, draw(st_values))
-        ]
-        star_args = ['*' + x for x in draw(st_star_args)]
-        double_star_args = ['**' + x for x in draw(st_double_star_args)]
-
-        arguments = positional_args + keyword_args + star_args + double_star_args
-        draw(st.randoms()).shuffle(arguments)
-        arguments = ','.join(arguments)
-
-        function_call = 'fn({arguments})'.format(arguments=arguments)
-        try:
-            # TODO: Figure out the exact rules for ordering of positional, keyword,
-            # star args, double star args and in which versions the various
-            # types of arguments are supported so we don't need to check that the
-            # expression compiles like this.
-            compile(function_call, '<string>', 'single')
-        except:
-            assume(False)
-        return function_call
-
-
-    def test_function_no_args():
-        validate_uncompyle("fn()")
-
-    @pytest.mark.skipif(PYTHON_VERSION < 2.7,
-                        reason="need at least Python 2.7")
-    def isolated_function_calls(which):
-        """
-        Returns a strategy for generating function calls, but isolated to
-        particular types of arguments, for example only positional arguments.
-
-        This can help reason about debugging errors in specific types of function
-        calls.
-
-        :param which: One of 'keyword', 'positional', 'star', 'double_star'
-
-        :return: Strategy for generating an function call isolated to specific
-                 argument types.
-        """
-        kwargs = dict(
-            max_keyword_args=0,
-            max_positional_args=0,
-            max_star_args=0,
-            max_double_star_args=0,
-        )
-        kwargs['_'.join(('min', which, 'args'))] = 1
-        kwargs['_'.join(('max', which, 'args'))] = 5 if 'star' not in which else 1
-        return function_calls(**kwargs)
-
-
-    with settings(max_examples=25):
-
-        @pytest.mark.skipif(PYTHON_VERSION < 2.7,
-                            reason="need at least Python 2.7")
-        @given(isolated_function_calls('positional'))
-        @example("fn(0)")
-        def test_function_positional_only(expr):
-            validate_uncompyle(expr)
-
-        @pytest.mark.skipif(PYTHON_VERSION < 2.7,
-                            reason="need at least Python 2.7")
-        @given(isolated_function_calls('keyword'))
-        @example("fn(a=0)")
-        def test_function_call_keyword_only(expr):
-            validate_uncompyle(expr)
-
-        @pytest.mark.skipif(PYTHON_VERSION < 2.7,
-                            reason="need at least Python 2.7")
-        @given(isolated_function_calls('star'))
-        @example("fn(*items)")
-        def test_function_call_star_only(expr):
-            validate_uncompyle(expr)
-
-        @pytest.mark.skipif(PYTHON_VERSION < 2.7,
-                            reason="need at least Python 2.7")
-        @given(isolated_function_calls('double_star'))
-        @example("fn(**{})")
-        def test_function_call_double_star_only(expr):
-            validate_uncompyle(expr)
-
-
-    @pytest.mark.xfail()
-    def test_BUILD_CONST_KEY_MAP_BUILD_MAP_UNPACK_WITH_CALL_BUILD_TUPLE_CALL_FUNCTION_EX():
-        validate_uncompyle("fn(w=0,m=0,**v)")
-
-
-    @pytest.mark.xfail()
-    def test_BUILD_MAP_BUILD_MAP_UNPACK_WITH_CALL_BUILD_TUPLE_CALL_FUNCTION_EX():
-        validate_uncompyle("fn(a=0,**g)")
-
-
-    @pytest.mark.xfail()
-    def test_CALL_FUNCTION_EX():
-        validate_uncompyle("fn(*g,**j)")
-
-
-    @pytest.mark.xfail()
-    def test_BUILD_MAP_CALL_FUNCTION_EX():
-        validate_uncompyle("fn(*z,u=0)")
-
-
-    @pytest.mark.xfail()
-    def test_BUILD_TUPLE_CALL_FUNCTION_EX():
-        validate_uncompyle("fn(**a)")
-
-
-    @pytest.mark.xfail()
-    def test_BUILD_MAP_BUILD_TUPLE_BUILD_TUPLE_UNPACK_WITH_CALL_CALL_FUNCTION_EX():
-        validate_uncompyle("fn(b,b,b=0,*a)")
-
-
-    @pytest.mark.xfail()
-    def test_BUILD_TUPLE_BUILD_TUPLE_UNPACK_WITH_CALL_CALL_FUNCTION_EX():
-        validate_uncompyle("fn(*c,v)")
-
-
-    @pytest.mark.xfail()
-    def test_BUILD_CONST_KEY_MAP_CALL_FUNCTION_EX():
-        validate_uncompyle("fn(i=0,y=0,*p)")
-
-
-    @pytest.mark.skip(reason='skipping property based test until all individual tests are passing')
-    @given(function_calls())
-    def test_function_call(function_call):
-        validate_uncompyle(function_call)

pytest/test_grammar.py

@@ -1,105 +1,147 @@
 import re
-from uncompyle6 import PYTHON_VERSION, PYTHON3, IS_PYPY # , PYTHON_VERSION
 from uncompyle6.parser import get_python_parser, python_parser
 from uncompyle6.scanner import get_scanner
+from xdis.version_info import PYTHON_VERSION_TRIPLE, IS_PYPY
+
 
 def test_grammar():
-
     def check_tokens(tokens, opcode_set):
         remain_tokens = set(tokens) - opcode_set
-        remain_tokens = set([re.sub('_\d+$','', t) for t in remain_tokens])
-        remain_tokens = set([re.sub('_CONT$','', t) for t in remain_tokens])
+        remain_tokens = set([re.sub(r"_\d+$", "", t) for t in remain_tokens])
+        remain_tokens = set([re.sub("_CONT$", "", t) for t in remain_tokens])
+        remain_tokens = set([re.sub("LOAD_CODE$", "", t) for t in remain_tokens])
         remain_tokens = set(remain_tokens) - opcode_set
-        assert remain_tokens == set([]), \
-            "Remaining tokens %s\n====\n%s" % (remain_tokens, p.dump_grammar())
+        assert remain_tokens == set([]), "Remaining tokens %s\n====\n%s" % (
+            remain_tokens,
+            p.dump_grammar(),
+        )
 
-    p = get_python_parser(PYTHON_VERSION, is_pypy=IS_PYPY)
-    (lhs, rhs, tokens,
-     right_recursive, dup_rhs) = p.check_sets()
+    p = get_python_parser(PYTHON_VERSION_TRIPLE, is_pypy=IS_PYPY)
+    (lhs, rhs, tokens, right_recursive, dup_rhs) = p.check_sets()
 
     # We have custom rules that create the below
-    expect_lhs = set(['pos_arg', 'get_iter', 'attribute'])
+    expect_lhs = set(["pos_arg"])
 
-    unused_rhs = set(['list', 'mkfunc',
-                      'mklambda',
-                      'unpack',])
-    expect_right_recursive = set([('designList',
-                                   ('store', 'DUP_TOP', 'designList'))])
+    if PYTHON_VERSION_TRIPLE < (3, 8):
+        if PYTHON_VERSION_TRIPLE < (3, 7):
+            expect_lhs.add("attribute")
 
-    if PYTHON_VERSION != 3.7:
-        unused_rhs.add('call')
+        expect_lhs.add("get_iter")
 
-    if PYTHON_VERSION > 2.6:
-        expect_lhs.add('kvlist')
-        expect_lhs.add('kv3')
-        unused_rhs.add('dict')
-
-    if PYTHON3:
-        expect_lhs.add('load_genexpr')
-
-        unused_rhs = unused_rhs.union(set("""
-        except_pop_except generator_exp
-        """.split()))
-        if PYTHON_VERSION >= 3.0:
-            expect_lhs.add("annotate_arg")
-            expect_lhs.add("annotate_tuple")
-            unused_rhs.add("mkfunc_annotate")
-            unused_rhs.add("dict_comp")
-            unused_rhs.add("classdefdeco1")
-            if PYTHON_VERSION != 3.6:
-                if PYTHON_VERSION in (3.5, 3.7):
-                    expect_right_recursive.add((('l_stmts',
-                                                 ('lastl_stmt', 'come_froms', 'l_stmts'))))
-                    pass
-                pass
-            else:
-                expect_right_recursive.add((('l_stmts',
-                                             ('lastl_stmt', 'COME_FROM', 'l_stmts'))))
-                pass
-            pass
-        pass
+        if PYTHON_VERSION_TRIPLE >= (3, 8) or PYTHON_VERSION_TRIPLE < (3, 0):
+            expect_lhs.add("stmts_opt")
     else:
-        expect_lhs.add('kwarg')
+        expect_lhs.add("async_with_as_stmt")
+        expect_lhs.add("async_with_stmt")
+
+    unused_rhs = set(["list", "mkfunc", "lambda_body", "unpack"])
+
+    expect_right_recursive = set([("designList", ("store", "DUP_TOP", "designList"))])
+
+    if PYTHON_VERSION_TRIPLE[:2] <= (3, 6):
+        unused_rhs.add("call")
+
+    if PYTHON_VERSION_TRIPLE >= (2, 7):
+        expect_lhs.add("kvlist")
+        expect_lhs.add("kv3")
+        unused_rhs.add("dict")
+
+        if PYTHON_VERSION_TRIPLE < (3, 7) and PYTHON_VERSION_TRIPLE[:2] != (2, 7):
+            # NOTE: this may disappear
+            expect_lhs.add("except_handler_else")
+
+    expect_lhs.add("load_genexpr")
+
+    unused_rhs = unused_rhs.union(
+        set(
+            """
+    except_pop_except generator_exp
+    """.split()
+        )
+    )
+    if PYTHON_VERSION_TRIPLE < (3, 7):
+        expect_lhs.add("annotate_arg")
+        expect_lhs.add("annotate_tuple")
+        unused_rhs.add("mkfunc_annotate")
+
+    unused_rhs.add("dict_comp")
+    unused_rhs.add("classdefdeco1")
+    unused_rhs.add("tryelsestmtl")
+    if PYTHON_VERSION_TRIPLE >= (3, 5):
+        expect_right_recursive.add(
+            (("l_stmts", ("lastl_stmt", "come_froms", "l_stmts")))
+        )
+        pass
+    pass
+
+    if PYTHON_VERSION_TRIPLE >= (3, 7):
+        expect_lhs.add("set_for")
+        unused_rhs.add("set_iter")
+        pass
+    pass
+    # FIXME
+    if PYTHON_VERSION_TRIPLE < (3, 8):
+        assert expect_lhs == set(lhs)
+        assert unused_rhs == set(rhs)
 
-    assert expect_lhs == set(lhs)
-    assert unused_rhs == set(rhs)
     assert expect_right_recursive == right_recursive
 
-    expect_dup_rhs = frozenset([('COME_FROM',), ('CONTINUE',), ('JUMP_ABSOLUTE',),
-                                ('LOAD_CONST',),
-                                ('JUMP_BACK',), ('JUMP_FORWARD',)])
+    expect_dup_rhs = frozenset(
+        [
+            ("COME_FROM",),
+            ("CONTINUE",),
+            ("JUMP_ABSOLUTE",),
+            ("LOAD_CONST",),
+            ("JUMP_BACK",),
+            ("JUMP_FORWARD",),
+        ]
+    )
     reduced_dup_rhs = dict((k, dup_rhs[k]) for k in dup_rhs if k not in expect_dup_rhs)
-    for k in reduced_dup_rhs:
-        print(k, reduced_dup_rhs[k])
+    if reduced_dup_rhs:
+        print("\nPossible duplicate RHS that might be folded, into one of the LHS symbols")
+        for k in reduced_dup_rhs:
+            print(k, reduced_dup_rhs[k])
     # assert not reduced_dup_rhs, reduced_dup_rhs
 
-    s = get_scanner(PYTHON_VERSION, IS_PYPY)
+    s = get_scanner(PYTHON_VERSION_TRIPLE, IS_PYPY)
     ignore_set = set(
-            """
+        """
             JUMP_BACK CONTINUE
             COME_FROM COME_FROM_EXCEPT
             COME_FROM_EXCEPT_CLAUSE
             COME_FROM_LOOP COME_FROM_WITH
             COME_FROM_FINALLY ELSE
-            LOAD_GENEXPR LOAD_ASSERT LOAD_SETCOMP LOAD_DICTCOMP
+            LOAD_GENEXPR LOAD_ASSERT LOAD_SETCOMP LOAD_DICTCOMP LOAD_STR LOAD_CODE
             LAMBDA_MARKER
             RETURN_END_IF RETURN_END_IF_LAMBDA RETURN_VALUE_LAMBDA RETURN_LAST
-            """.split())
-    if 2.6 <= PYTHON_VERSION <= 2.7:
+            """.split()
+    )
+
+    if (2, 6) <= PYTHON_VERSION_TRIPLE <= (2, 7):
         opcode_set = set(s.opc.opname).union(ignore_set)
-        if PYTHON_VERSION == 2.6:
+        if PYTHON_VERSION_TRIPLE[:2] == (2, 6):
             opcode_set.add("THEN")
         check_tokens(tokens, opcode_set)
-    elif PYTHON_VERSION == 3.4:
-        ignore_set.add('LOAD_CLASSNAME')
-        ignore_set.add('STORE_LOCALS')
+    elif PYTHON_VERSION_TRIPLE[:2] == (3, 4):
+        ignore_set.add("LOAD_CLASSNAME")
+        ignore_set.add("STORE_LOCALS")
         opcode_set = set(s.opc.opname).union(ignore_set)
         check_tokens(tokens, opcode_set)
 
+
 def test_dup_rule():
     import inspect
-    python_parser(PYTHON_VERSION, inspect.currentframe().f_code,
-                  is_pypy=IS_PYPY,
-                  parser_debug={
-                      'dups': True, 'transition': False, 'reduce': False,
-                      'rules': False, 'errorstack': None, 'context': True})
+
+    python_parser(
+        PYTHON_VERSION_TRIPLE,
+        inspect.currentframe().f_code,
+        is_pypy=IS_PYPY,
+        parser_debug={
+            "dups": True,
+            "transition": False,
+            "reduce": False,
+            "rules": False,
+            "errorstack": None,
+            "context": True,
+        },
+    )

pytest/test_pysource.py-notyet

@@ -1,21 +1,15 @@
 import sys
-from uncompyle6 import PYTHON3
 from uncompyle6.scanner import get_scanner
 from uncompyle6.semantics.consts import (
     escape, NONE,
     # RETURN_NONE, PASS, RETURN_LOCALS
 )
 
-if PYTHON3:
-    from io import StringIO
-    def iteritems(d):
-        return d.items()
-else:
-    from StringIO import StringIO
-    def iteritems(d):
-        return d.iteritems()
+from io import StringIO
+def iteritems(d):
+    return d.items()
 
-from uncompyle6.semantics.pysource import SourceWalker as SourceWalker
+from uncompyle6.semantics.pysource import (SourceWalker, deparse_code2str)
 
 def test_template_engine():
     s = StringIO()
@@ -127,11 +121,17 @@ def test_tables():
                             "Full entry: %s" %
                             (name, k, arg, typ, entry[arg], type(entry[arg]), entry)
                             )
-                        assert len(tup) == 2
+                        assert 2 <= len(tup) <= 3
                         for j, x in enumerate(tup):
-                            assert isinstance(x, int), (
-                                "%s[%s][%d][%d] type '%s' is '%s should be an int but is %s. Full entry: %s" %
-                                (name, k, arg, j, typ, x, type(x), entry)
+                            if len(tup) == 3 and j == 1:
+                                assert isinstance(x, str), (
+                                    "%s[%s][%d][%d] type '%s' is '%s should be an string but is %s. Full entry: %s" %
+                                    (name, k, arg, j, typ, x, type(x), entry)
+                                    )
+                            else:
+                                assert isinstance(x, int), (
+                                    "%s[%s][%d][%d] type '%s' is '%s should be an int but is %s. Full entry: %s" %
+                                    (name, k, arg, j, typ, x, type(x), entry)
                                 )
                         pass
                     arg += 1
@@ -179,3 +179,11 @@ def test_tables():
             assert arg == len(entry), (
                 "%s[%s] arg %d should be length of entry %d. Full entry: %s" %
                             (name, k, arg, len(entry), entry))
+
+def test_deparse_code2str():
+    def deparse_test(co):
+        "This is a docstring"
+        s = deparse_code2str(co, debug_opts={"asm": "after", "tree": True})
+        assert s
+        return
+    deparse_test(deparse_test.__code__)

pytest/test_single_compile.py

@@ -1,22 +1,24 @@
 import pytest
-from uncompyle6 import PYTHON_VERSION, deparse_code
-pytestmark = pytest.mark.skip(PYTHON_VERSION < 2.7,
-                              reason="need at least Python 2.7")
+from uncompyle6 import code_deparse
+from xdis.version_info import PYTHON_VERSION_TRIPLE
 
-if PYTHON_VERSION > 2.6:
-    def test_single_mode():
-        single_expressions = (
-            'i = 1',
-            'i and (j or k)',
-            'i += 1',
-            'i = j % 4',
-            'i = {}',
-            'i = []',
-            'for i in range(10):\n    i\n',
-            'for i in range(10):\n    for j in range(10):\n        i + j\n',
-            'try:\n    i\nexcept Exception:\n    j\nelse:\n    k\n'
-        )
+pytest.mark.skip(PYTHON_VERSION_TRIPLE < (2, 7), reason="need Python < 2.7")
 
-        for expr in single_expressions:
-            code = compile(expr + '\n', '<string>', 'single')
-            assert deparse_code(PYTHON_VERSION, code, compile_mode='single').text == expr + '\n'
+
+def test_single_mode():
+    single_expressions = (
+        "i = 1",
+        "i and (j or k)",
+        "i += 1",
+        "i = j % 4",
+        "i = {}",
+        "i = []",
+        "for i in range(10):\n    i\n",
+        "for i in range(10):\n    for j in range(10):\n        i + j\n",
+        # 'try:\n    i\nexcept Exception:\n    j\nelse:\n    k\n'
+    )
+
+    for expr in single_expressions:
+        code = compile(expr + "\n", "<string>", "single")
+        got = code_deparse(code, compile_mode="single").text
+        assert got == expr + "\n"

pytest/test_token.py

@@ -1,23 +1,24 @@
 from uncompyle6.scanners.tok import Token
 
+
 def test_token():
     # Test token formatting of: LOAD_CONST None
-    t = Token('LOAD_CONST', offset=0, attr=None, pattr=None, has_arg=True)
-    expect = '           0  LOAD_CONST               None'
+    t = Token("LOAD_CONST", offset=0, attr=None, pattr=None, has_arg=True)
+    expect = "0  LOAD_CONST               None"
     # print(t.format())
     assert t
-    assert t.format() == expect
+    assert t.format().strip() == expect.strip()
 
     # Make sure equality testing of tokens ignores offset
-    t2 = Token('LOAD_CONST', offset=2, attr=None, pattr=None, has_arg=True)
+    t2 = Token("LOAD_CONST", offset=2, attr=None, pattr=None, has_arg=True)
     assert t2 == t
 
-
     # Make sure formatting of: LOAD_CONST False. We assume False is the 0th index
     # of co_consts.
-    t = Token('LOAD_CONST', offset=1, attr=False, pattr=False, has_arg=True)
-    expect = '           1  LOAD_CONST            0  False'
-    assert t.format() == expect
+    t = Token("LOAD_CONST", offset=1, attr=False, pattr=False, has_arg=True)
+    expect = "1  LOAD_CONST               False"
+    assert t.format().strip() == expect.strip()
 
-if __name__ == '__main__':
+
+if __name__ == "__main__":
     test_token()

pytest/testdata/if-2.7.right

@@ -1,12 +1,12 @@
 # Python 2.7
 # Embedded file name: simple_source/branching/05_if.py
 
-   6       0  LOAD_NAME             0  'True'
-           3  POP_JUMP_IF_FALSE    15  'to 15'
+ L.   6         0  LOAD_NAME             0  'True'
+                3  POP_JUMP_IF_FALSE    15  'to 15'
 
-   7       6  LOAD_NAME             1  'False'
-           9  STORE_NAME            2  'b'
-          12  JUMP_FORWARD          0  'to 15'
-        15_0  COME_FROM            12  '12'
-          15  LOAD_CONST            0  None
-          18  RETURN_VALUE     
+ L.   7         6  LOAD_NAME             1  'False'
+                9  STORE_NAME            2  'b'
+               12  JUMP_FORWARD          0  'to 15'
+             15_0  COME_FROM            12  '12'
+               15  LOAD_CONST               None
+               18  RETURN_VALUE     

pytest/testdata/ifelse-2.7.right

@@ -1,15 +1,15 @@
 # Python 2.7
 # Embedded file name: simple_source/branching/05_ifelse.py
 
-   3       0  LOAD_NAME             0  'True'
-           3  POP_JUMP_IF_FALSE    15  'to 15'
+ L.   3         0  LOAD_NAME             0  'True'
+                3  POP_JUMP_IF_FALSE    15  'to 15'
 
-   4       6  LOAD_CONST            0  1
-           9  STORE_NAME            1  'b'
-          12  JUMP_FORWARD          6  'to 21'
+ L.   4         6  LOAD_CONST               1
+                9  STORE_NAME            1  'b'
+               12  JUMP_FORWARD          6  'to 21'
 
-   6      15  LOAD_CONST            1  2
-          18  STORE_NAME            2  'd'
-        21_0  COME_FROM            12  '12'
-          21  LOAD_CONST            2  None
-          24  RETURN_VALUE     
+ L.   6        15  LOAD_CONST               2
+               18  STORE_NAME            2  'd'
+             21_0  COME_FROM            12  '12'
+               21  LOAD_CONST               None
+               24  RETURN_VALUE     

pytest/validate.py

@@ -1,23 +1,23 @@
 # future
 from __future__ import print_function
+
 # std
 import os
 import difflib
 import subprocess
 import tempfile
 import functools
-# uncompyle6 / xdis
-from uncompyle6 import PYTHON_VERSION, PYTHON3, IS_PYPY, deparse_code
-# TODO : I think we can get xdis to support the dis api (python 3 version) by doing something like this there
-from xdis.bytecode import Bytecode
-from xdis.main import get_opcode
-opc = get_opcode(PYTHON_VERSION, IS_PYPY)
-Bytecode = functools.partial(Bytecode, opc=opc)
 
-if PYTHON3:
-    from io import StringIO
-else:
-    from StringIO import StringIO
+# uncompyle6 / xdis
+from uncompyle6 import code_deparse
+from xdis.version_info import PYTHON_VERSION_TRIPLE, IS_PYPY
+
+# TODO : I think we can get xdis to support the dis api (python 3 version) by doing something like this there
+from xdis import Bytecode, get_opcode
+
+opc = get_opcode(PYTHON_VERSION_TRIPLE, IS_PYPY)
+Bytecode = functools.partial(Bytecode, opc=opc)
+import six
 
 def _dis_to_text(co):
     return Bytecode(co).dis()
@@ -32,36 +32,32 @@ def print_diff(original, uncompyled):
     :param original: Text describing the original code object.
     :param uncompyled: Text describing the uncompyled code object.
     """
-    original_lines = original.split('\n')
-    uncompyled_lines = uncompyled.split('\n')
-    args = original_lines, uncompyled_lines, 'original', 'uncompyled'
+    original_lines = original.split("\n")
+    uncompyled_lines = uncompyled.split("\n")
+    args = original_lines, uncompyled_lines, "original", "uncompyled"
     try:
         from bs4 import BeautifulSoup
+
         diff = difflib.HtmlDiff().make_file(*args)
         diff = BeautifulSoup(diff, "html.parser")
         diff.select_one('table[summary="Legends"]').extract()
     except ImportError:
-        print('\nTo display diff highlighting run:\n    pip install BeautifulSoup4')
+        print("\nTo display diff highlighting run:\n    pip install BeautifulSoup4")
         diff = difflib.HtmlDiff().make_table(*args)
 
     with tempfile.NamedTemporaryFile(delete=False) as f:
-        f.write(str(diff).encode('utf-8'))
+        f.write(str(diff).encode("utf-8"))
 
     try:
         print()
-        html = subprocess.check_output([
-            'elinks',
-            '-dump',
-            '-no-references',
-            '-dump-color-mode',
-            '1',
-            f.name,
-        ]).decode('utf-8')
+        html = subprocess.check_output(
+            ["elinks", "-dump", "-no-references", "-dump-color-mode", "1", f.name]
+        ).decode("utf-8")
         print(html)
     except:
-        print('\nFor side by side diff install elinks')
+        print("\nFor side by side diff install elinks")
         diff = difflib.Differ().compare(original_lines, uncompyled_lines)
-        print('\n'.join(diff))
+        print("\n".join(diff))
     finally:
         os.unlink(f.name)
 
@@ -79,18 +75,19 @@ def are_instructions_equal(i1, i2):
 
     :return: True if the two instructions are approximately equal, otherwise False.
     """
-    result = (1==1
+    result = (
+        1 == 1
         and i1.opname == i2.opname
         and i1.opcode == i2.opcode
         and i1.arg == i2.arg
         # ignore differences due to code objects
         # TODO : Better way of ignoring address
-        and (i1.argval == i2.argval or '<code object' in str(i1.argval))
+        and (i1.argval == i2.argval or "<code object" in str(i1.argval))
         # TODO : Should probably recurse to check code objects
-        and (i1.argrepr == i2.argrepr or '<code object' in i1.argrepr)
+        and (i1.argrepr == i2.argrepr or "<code object" in i1.argrepr)
         and i1.offset == i2.offset
         # ignore differences in line numbers
-        #and i1.starts_line
+        # and i1.starts_line
         and i1.is_jump_target == i2.is_jump_target
     )
     return result
@@ -114,22 +111,21 @@ def are_code_objects_equal(co1, co2):
     return True
 
 
-def validate_uncompyle(text, mode='exec'):
+def validate_uncompyle(text, mode="exec"):
     """
     Validate decompilation of the given source code.
 
     :param text: Source to validate decompilation of.
     """
-    original_code = compile(text, '<string>', mode)
+    original_code = compile(text, "<string>", mode)
     original_dis = _dis_to_text(original_code)
     original_text = text
 
-    deparsed = deparse_code(PYTHON_VERSION, original_code,
-                            compile_mode=mode,
-                            out=six.StringIO(),
-                            is_pypy=IS_PYPY)
+    deparsed = code_deparse(
+        original_code, out=six.StringIO(), version=PYTHON_VERSION_TRIPLE, compile_mode=mode
+    )
     uncompyled_text = deparsed.text
-    uncompyled_code = compile(uncompyled_text, '<string>', 'exec')
+    uncompyled_code = compile(uncompyled_text, "<string>", "exec")
 
     if not are_code_objects_equal(uncompyled_code, original_code):
 
@@ -137,15 +133,17 @@ def validate_uncompyle(text, mode='exec'):
 
         def output(text, dis):
             width = 60
-            return '\n\n'.join([
-                ' SOURCE CODE '.center(width, '#'),
-                text.strip(),
-                ' BYTECODE '.center(width, '#'),
-                dis
-            ])
+            return "\n\n".join(
+                [
+                    " SOURCE CODE ".center(width, "#"),
+                    text.strip(),
+                    " BYTECODE ".center(width, "#"),
+                    dis,
+                ]
+            )
 
         original = output(original_text, original_dis)
         uncompyled = output(uncompyled_text, uncompyled_dis)
         print_diff(original, uncompyled)
 
-        assert 'original' == 'uncompyled'
+        assert "original" == "uncompyled"

requirements-dev.txt

@@ -1,3 +1,3 @@
-pytest>=3.0.0,<=3.0.1
 flake8
-hypothesis<=3.0.0
+six
+pytest==3.2.5 # for 2.7 < PYTHON_VERSION <= 3.2 use pytest 2.9.2; for 3.1 2.10

requirements.txt

@@ -2,3 +2,8 @@
 hypothesis==2.0.0
 pytest
 -e .
+
+Click~=7.0
+xdis>=6.0.4
+configobj~=5.0.6
+setuptools~=65.5.1

setup.cfg

@@ -1,11 +1,57 @@
 [bdist_rpm]
-release = 1
-packager = Mysterie <kajusska@gmail.com>
-doc_files = README
-#		CHANGES.txt
-#		USAGE.txt
+release = 0
+packager = rocky <rb@dustyfeet.com
+doc_files = README.rst
+		ChangeLog
+		COPYING
+		DECOMPYLE-2.4-CHANGELOG.txt
+		HISTORY.md
+		HOW-TO_REPORT-A-BUG.md
+		NEWS.md
 #		doc/
 #		examples/
 
 [bdist_wheel]
 universal=1
+
+[metadata]
+description_file = README.rst
+
+[flake8]
+# max-line-length setting: NO we do not want everyone writing 120-character lines!
+# We are setting the maximum line length big here because there are longer
+# lines allowed by black in some cases that are forbidden by flake8. Since
+# black has the final say about code formatting issues, this setting is here to
+# make sure that flake8 doesn't fail the build on longer lines allowed by
+# black.
+max-line-length = 120
+max-complexity = 12
+select = E,F,W,C,B,B9
+ignore =
+    # E123 closing bracket does not match indentation of opening bracket's line
+    E123
+    # E203 whitespace before ':' (Not PEP8 compliant, Python Black)
+    E203
+    # E501 line too long (82 > 79 characters) (replaced by B950 from flake8-bugbear,
+    # https://github.com/PyCQA/flake8-bugbear)
+    E501
+    # W503 line break before binary operator (Not PEP8 compliant, Python Black)
+    W503
+    # W504 line break after binary operator (Not PEP8 compliant, Python Black)
+    W504
+    # C901 function too complex - since many of zz9 functions are too complex with a lot
+    # of if branching
+    C901
+    # module level import not at top of file. This is too restrictive. Can't even have a
+    # docstring higher.
+    E402
+per-file-ignores =
+    # These are config files. The `c` variable them is injected not defined.
+    pow/ansible/roles/jupyterhub/templates/jupyterhub_config*.py:F821
+    # Ignore some errors in files that are stolen from other projects to avoid lots
+    # of merge problems later .
+    pow/ansible/roles/webtier/files/supervisor_httpgroupok.py:E126,E128,E222,E225,E226,E261,E301,E302,E305,F841,E201,E202
+    silhouette/src/silhouette/gprof2dot.py:E711,E713,E741,F401
+    # Ignore undefined name errors in "expectation" test Python code.
+    # These files get exec'd in an environment that defines the variables.
+    server/tests/files/expectations/*.py:F821

setup.py

@@ -1,42 +1,61 @@
 #!/usr/bin/env python
+import setuptools
 import sys
 
 """Setup script for the 'uncompyle6' distribution."""
 
 SYS_VERSION = sys.version_info[0:2]
-if not ((2, 6) <= SYS_VERSION  <= (3, 7)):
-    mess = "Python Release 2.6 .. 3.7 are supported in this code branch."
-    if ((2, 4) <= SYS_VERSION <= (2, 7)):
-        mess += ("\nFor your Python, version %s, use the python-2.4 code/branch." %
-                 sys.version[0:3])
+if not ((2, 4) <= SYS_VERSION < (3, 12)):
+    mess = "Python Release 2.6 .. 3.11 are supported in this code branch."
+    if (2, 4) <= SYS_VERSION <= (2, 7):
+        mess += (
+            "\nFor your Python, version %s, use the python-2.4 code/branch."
+            % sys.version[0:3]
+        )
+    if (3, 3) <= SYS_VERSION < (3, 6):
+        mess += (
+            "\nFor your Python, version %s, use the python-3.3-to-3.5 code/branch."
+            % sys.version[0:3]
+        )
     elif SYS_VERSION < (2, 4):
-        mess += ("\nThis package is not supported for Python version %s."
-                 % sys.version[0:3])
+        mess += (
+            "\nThis package is not supported for Python version %s." % sys.version[0:3]
+        )
     print(mess)
     raise Exception(mess)
 
-from __pkginfo__ import \
-    author,           author_email,       install_requires,          \
-    license,          long_description,   classifiers,               \
-    entry_points,     modname,            py_modules,                \
-    short_desc,       VERSION,            web,                       \
-    zip_safe
+from __pkginfo__ import (
+    author,
+    author_email,
+    install_requires,
+    license,
+    long_description,
+    classifiers,
+    entry_points,
+    modname,
+    py_modules,
+    short_desc,
+    __version__,
+    web,
+    zip_safe,
+)
 
-from setuptools import setup, find_packages
-setup(
-       author             = author,
-       author_email       = author_email,
-       classifiers        = classifiers,
-       description        = short_desc,
-       entry_points       = entry_points,
-       install_requires   = install_requires,
-       license            = license,
-       long_description   = long_description,
-       name               = modname,
-       packages           = find_packages(),
-       py_modules         = py_modules,
-       test_suite         = 'nose.collector',
-       url                = web,
-       tests_require      = ['nose>=1.0'],
-       version            = VERSION,
-       zip_safe           = zip_safe)
+setuptools.setup(
+    author=author,
+    author_email=author_email,
+    classifiers=classifiers,
+    description=short_desc,
+    entry_points=entry_points,
+    install_requires=install_requires,
+    license=license,
+    long_description=long_description,
+    long_description_content_type="text/x-rst",
+    name=modname,
+    packages=setuptools.find_packages(),
+    py_modules=py_modules,
+    test_suite="nose.collector",
+    url=web,
+    tests_require=["nose>=1.0"],
+    version=__version__,
+    zip_safe=zip_safe,
+)

test/.gitignore

@@ -1 +1,8 @@
+/.coverage
+/.python-version
 /nohup.out
+/pycdc
+/test_pycdc_tests.sh
+/test_uncompyle2.py
+/test_unpy33.py
+/test_unpy37.py

test/Makefile

@@ -1,6 +1,7 @@
 PHONY=check clean dist distclean test test-unit test-functional rmChangeLog clean_pyc nosetests \
-      check-bytecode-1  check-bytecode-1.4 check-bytecode-1.5 \
-      check-bytecode-2 check-bytecode-3 \
+      check-bytecode-1.0 check-bytecode-1.1 check-bytecode-1.2 check-bytecode-1.3 \
+      check-bytecode-1 check-bytecode-1.4 check-bytecode-1.5 check-bytecode-1.6 \
+      check-bytecode-2 check-bytecode-3 check-bytecode-3-short \
       check-bytecode-2.2 check-byteocde-2.3 check-bytecode-2.4 \
       check-short check-2.6 check-2.7 check-3.0 check-3.1 check-3.2 check-3.3 \
       check-3.4 check-3.5 check-3.6 check-3.7 check-5.6 5.6 5.8 \
@@ -12,7 +13,7 @@ PHONY=check clean dist distclean test test-unit test-functional rmChangeLog clea
 GIT2CL ?= git2cl
 PYTHON ?= python
 
-PYTHON_VERSION = $(shell $(PYTHON) -V 2>&1 | cut -d ' ' -f 2 | cut -d'.' -f1,2)
+PYTHON_VERSION = $(shell $(PYTHON) -V 2>&1 | cut -d ' ' -f 2 | cut -d'.' -f1,2 | head -1)
 NATIVE_CHECK = check-$(PYTHON_VERSION)
 
 # Set COMPILE='--compile' to force compilation before check
@@ -21,8 +22,8 @@ COVER_DIR=../tmp/grammar-cover
 
 # Run short tests
 check-short:
-	@$(PYTHON) -V && PYTHON_VERSION=`$(PYTHON) -V 2>&1 | cut -d ' ' -f 2 | cut -d'.' -f1,2`; \
-	$(MAKE) check-bytecode-short
+	@$(PYTHON) -V && PYTHON_VERSION=`$(PYTHON) -V 2>&1 | cut -d ' ' -f 2 | cut -d'.' -f1,2` | head -1; \
+	$(MAKE) check-bytecode-${PYTHON_VERSION}
 
 # Run all tests
 check:
@@ -33,51 +34,68 @@ check-2.6 check-2.7: check-bytecode-2 check-bytecode-3 check-bytecode-1 check-na
 
 #: Run working tests from Python 3.0
 check-3.0: check-bytecode
-	$(PYTHON) test_pythonlib.py --bytecode-3.0 --weak-verify $(COMPILE)
+	$(PYTHON) test_pythonlib.py --bytecode-3.0-run --verify-run
+	$(PYTHON) test_pythonlib.py --bytecode-3.0 --syntax-verify $(COMPILE)
 
 #: Run working tests from Python 3.1
 check-3.1: check-bytecode
-	$(PYTHON) test_pythonlib.py --bytecode-3.1 --weak-verify $(COMPILE)
 	$(PYTHON) test_pythonlib.py --bytecode-3.1-run --verify-run
+	$(PYTHON) test_pythonlib.py --bytecode-3.1 --syntax-verify $(COMPILE)
 
 #: Run working tests from Python 3.2
 check-3.2: check-bytecode
-	$(PYTHON) test_pythonlib.py --bytecode-3.2 --weak-verify $(COMPILE)
 	$(PYTHON) test_pythonlib.py --bytecode-3.2-run --verify-run
+	$(PYTHON) test_pythonlib.py --bytecode-3.2 --syntax-verify $(COMPILE)
 
 #: Run working tests from Python 3.3
 check-3.3: check-bytecode
-	$(PYTHON) test_pythonlib.py --bytecode-3.3 --weak-verify $(COMPILE)
 	$(PYTHON) test_pythonlib.py --bytecode-3.3-run --verify-run
+	$(PYTHON) test_pythonlib.py --bytecode-3.3 --syntax-verify $(COMPILE)
 
 #: Run working tests from Python 3.4
 check-3.4: check-bytecode check-3.4-ok check-2.7-ok
-	$(PYTHON) test_pythonlib.py --bytecode-3.4 --weak-verify $(COMPILE)
 	$(PYTHON) test_pythonlib.py --bytecode-3.4-run --verify-run
+	$(PYTHON) test_pythonlib.py --bytecode-3.4 --syntax-verify $(COMPILE)
 
 #: Run working tests from Python 3.5
 check-3.5: check-bytecode
-	$(PYTHON) test_pythonlib.py --bytecode-3.5 --weak-verify $(COMPILE)
 	$(PYTHON) test_pythonlib.py --bytecode-3.5-run --verify-run
+	$(PYTHON) test_pythonlib.py --bytecode-3.5 --syntax-verify $(COMPILE)
 
 #: Run working tests from Python 3.6
 check-3.6: check-bytecode
-	$(PYTHON) test_pythonlib.py --bytecode-3.6 --weak-verify $(COMPILE)
+	$(PYTHON) test_pythonlib.py --bytecode-3.6-run --verify-run
+	$(PYTHON) test_pythonlib.py --bytecode-3.6 --syntax-verify $(COMPILE)
 
 #: Run working tests from Python 3.7
 check-3.7: check-bytecode
-	$(PYTHON) test_pythonlib.py --bytecode-3.7 --weak-verify $(COMPILE)
+	$(PYTHON) test_pythonlib.py --bytecode-3.7-run --verify-run
+	$(PYTHON) test_pythonlib.py --bytecode-3.7 --syntax-verify $(COMPILE)
+
+check-pypy37: check-bytecode
+	$(PYTHON) test_pythonlib.py --bytecode-pypy37 --verify-run
+
+#: Run working tests from Python 3.8
+check-3.8: check-bytecode
+	$(PYTHON) test_pythonlib.py --bytecode-3.8-run --verify-run
+	$(PYTHON) test_pythonlib.py --bytecode-3.8 --syntax-verify $(COMPILE)
+
+check-3.9: check-bytecode
+	@echo "Note that we do not support decompiling Python 3.9 bytecode - no 3.9 tests run"
+
+check-3.10: check-bytecode
+	@echo "Note that we do not support decompiling Python 3.10 bytecode - no 3.10 tests run"
 
 # FIXME
-#: this is called when running under pypy3.5-5.8.0 or pypy2-5.6.0
-5.8 5.6:
+#: this is called when running under pypy3.5-5.8.0, pypy2-5.6.0, pypy3.6-7.3.0 or pypy3.8-7.3.7
+5.8 5.6 7.3:
 
 #: Check deparsing only, but from a different Python version
 check-disasm:
 	$(PYTHON) dis-compare.py
 
 #: Check deparsing bytecode 1.x only
-check-bytecode-1: check-bytecode-1.4 check-bytecode-1.5
+check-bytecode-1: check-bytecode-1.0 check-bytecode-1.1 check-bytecode-1.2 check-bytecode-1.3 check-bytecode-1.4 check-bytecode-1.5 check-bytecode-1.6
 
 #: Check deparsing bytecode 2.x only
 check-bytecode-2:
@@ -86,10 +104,19 @@ check-bytecode-2:
         --bytecode-2.5 --bytecode-2.6 --bytecode-2.7 --bytecode-pypy2.7
 
 #: Check deparsing bytecode 3.x only
+# check-bytecode-3:
+# 	$(PYTHON) test_pythonlib.py \
+#         --bytecode-3.0 \
+# 	--bytecode-3.1 --bytecode-3.2 --bytecode-3.3 \
+# 	--bytecode-3.4 --bytecode-3.5 --bytecode-3.6 \
+#         --bytecode-3.7  \
+# 	--bytecode-pypy3.2 --bytecode-pypy3.6 --bytecode-3.8
+
+# FIXME: Until we shaked out problems with xdis...
 check-bytecode-3:
-	$(PYTHON) test_pythonlib.py --bytecode-3.0 \
-       --bytecode-3.1 --bytecode-3.2 --bytecode-3.3 \
-       --bytecode-3.4 --bytecode-3.5 --bytecode-3.6 --bytecode-pypy3.2
+	$(PYTHON) test_pythonlib.py \
+	--bytecode-3.4 --bytecode-3.5 --bytecode-3.6 \
+        --bytecode-3.7  --bytecode-3.8
 
 #: Check deparsing on selected bytecode 3.x
 check-bytecode-3-short:
@@ -99,6 +126,7 @@ check-bytecode-3-short:
 #: Check deparsing bytecode on all Python 2 and Python 3 versions
 check-bytecode: check-bytecode-3
 	$(PYTHON) test_pythonlib.py \
+	--bytecode-1.0 --bytecode-1.1 --bytecode-1.2 \
 	--bytecode-1.3 --bytecode-1.4 --bytecode-1.5 \
         --bytecode-2.2 --bytecode-2.3 --bytecode-2.4 \
 	--bytecode-2.1 --bytecode-2.2 --bytecode-2.3 --bytecode-2.4 \
@@ -112,6 +140,18 @@ check-bytecode-short: check-bytecode-3-short
         --bytecode-2.6 --bytecode-2.7 --bytecode-pypy2.7
 
 
+#: Check deparsing bytecode 1.0 only
+check-bytecode-1.0:
+	$(PYTHON) test_pythonlib.py --bytecode-1.0
+
+#: Check deparsing bytecode 1.1 only
+check-bytecode-1.1:
+	$(PYTHON) test_pythonlib.py --bytecode-1.1
+
+#: Check deparsing bytecode 1.2 only
+check-bytecode-1.2:
+	$(PYTHON) test_pythonlib.py --bytecode-1.2
+
 #: Check deparsing bytecode 1.3 only
 check-bytecode-1.3:
 	$(PYTHON) test_pythonlib.py --bytecode-1.3
@@ -124,6 +164,10 @@ check-bytecode-1.4:
 check-bytecode-1.5:
 	$(PYTHON) test_pythonlib.py --bytecode-1.5
 
+#: Check deparsing bytecode 1.6 only
+check-bytecode-1.6:
+	$(PYTHON) test_pythonlib.py --bytecode-1.6
+
 #: Check deparsing Python 2.1
 check-bytecode-2.1:
 	$(PYTHON) test_pythonlib.py --bytecode-2.1
@@ -138,12 +182,24 @@ check-bytecode-2.3:
 
 #: Check deparsing Python 2.4
 check-bytecode-2.4:
+	$(PYTHON) test_pythonlib.py --bytecode-2.4-run --verify-run
 	$(PYTHON) test_pythonlib.py --bytecode-2.4
 
 #: Check deparsing Python 2.5
 check-bytecode-2.5:
+	$(PYTHON) test_pythonlib.py --bytecode-2.5-run --verify-run
 	$(PYTHON) test_pythonlib.py --bytecode-2.5
 
+#: Check deparsing Python 2.6
+check-bytecode-2.6:
+	$(PYTHON) test_pythonlib.py --bytecode-2.6-run --verify-run
+	$(PYTHON) test_pythonlib.py --bytecode-2.6 --syntax-verify
+
+#: Check deparsing Python 2.7
+check-bytecode-2.7:
+	$(PYTHON) test_pythonlib.py --bytecode-2.7-run --verify-run
+	$(PYTHON) test_pythonlib.py --bytecode-2.7 --syntax-verify
+
 #: Get grammar coverage for Python 2.4
 grammar-coverage-2.4:
 	-rm $(COVER_DIR)/spark-grammar-24.cover
@@ -166,7 +222,7 @@ grammar-coverage-2.6:
 grammar-coverage-2.7:
 	-rm $(COVER_DIR)/spark-grammar-2.7.cover || true
 	SPARK_PARSER_COVERAGE=$(COVER_DIR)/spark-grammar-2.7.cover $(PYTHON) test_pythonlib.py --bytecode-2.7
-	SPARK_PARSER_COVERAGE=$(COVER_DIR)/spark-grammar-2.7.cover $(PYTHON) test_pyenvlib.py --2.7.14 --max=600
+	SPARK_PARSER_COVERAGE=$(COVER_DIR)/spark-grammar-2.7.cover $(PYTHON) test_pyenvlib.py --2.7.16 --max=600
 
 #: Get grammar coverage for Python 3.0
 grammar-coverage-3.0:
@@ -209,81 +265,78 @@ grammar-coverage-3.5:
 grammar-coverage-3.6:
 	rm $(COVER_DIR)/spark-grammar-3.6.cover || /bin/true
 	SPARK_PARSER_COVERAGE=$(COVER_DIR)/spark-grammar-3.6.cover $(PYTHON) test_pythonlib.py --bytecode-3.6
-	SPARK_PARSER_COVERAGE=$(COVER_DIR)/spark-grammar-3.6.cover $(PYTHON) test_pyenvlib.py --3.6.4 --max=280
+	SPARK_PARSER_COVERAGE=$(COVER_DIR)/spark-grammar-3.6.cover $(PYTHON) test_pyenvlib.py --3.6.8 --max=280
 
-#: Check deparsing Python 2.6
-check-bytecode-2.6:
-	$(PYTHON) test_pythonlib.py --bytecode-2.6 --weak-verify
-	$(PYTHON) test_pythonlib.py --bytecode-2.6-run --verify-run
-
-#: Check deparsing Python 2.7
-check-bytecode-2.7:
-	$(PYTHON) test_pythonlib.py --bytecode-2.7 --weak-verify
-	$(PYTHON) test_pythonlib.py --bytecode-2.7-run --verify-run
+#: Get grammar coverage for Python 3.7
+grammar-coverage-3.7:
+	rm $(COVER_DIR)/spark-grammar-3.7.cover || /bin/true
+	SPARK_PARSER_COVERAGE=$(COVER_DIR)/spark-grammar-3.7.cover $(PYTHON) test_pyenvlib.py --3.7.3 --max=500
 
 #: Check deparsing Python 3.0
 check-bytecode-3.0:
-	$(PYTHON) test_pythonlib.py --bytecode-3.0 --weak-verify
-
-#: Check deparsing Python 3.0
-check-bytecode-3.0:
-	$(PYTHON) test_pythonlib.py --bytecode-3.0 --weak-verify
-        # $(PYTHON) test_pythonlib.py --bytecode-3.0-run --verify-run
+	$(PYTHON) test_pythonlib.py --bytecode-3.0-run --verify-run
+	$(PYTHON) test_pythonlib.py --bytecode-3.0 --syntax-verify
 
 #: Check deparsing Python 3.1
 check-bytecode-3.1:
-	$(PYTHON) test_pythonlib.py --bytecode-3.1 --weak-verify
 	$(PYTHON) test_pythonlib.py --bytecode-3.1-run --verify-run
+	$(PYTHON) test_pythonlib.py --bytecode-3.1 --syntax-verify
 
 #: Check deparsing Python 3.2
 check-bytecode-3.2:
-	$(PYTHON) test_pythonlib.py --bytecode-3.2 --weak-verify
 	$(PYTHON) test_pythonlib.py --bytecode-3.2-run --verify-run
+	$(PYTHON) test_pythonlib.py --bytecode-3.2 --syntax-verify
 
 #: Check deparsing Python 3.3
 check-bytecode-3.3:
-	$(PYTHON) test_pythonlib.py --bytecode-3.3 --weak-verify
 	$(PYTHON) test_pythonlib.py --bytecode-3.3-run --verify-run
+	$(PYTHON) test_pythonlib.py --bytecode-3.3 --syntax-verify
 
 #: Check deparsing Python 3.4
 check-bytecode-3.4:
-	$(PYTHON) test_pythonlib.py --bytecode-3.4 --weak-verify
 	$(PYTHON) test_pythonlib.py --bytecode-3.4-run --verify-run
+	$(PYTHON) test_pythonlib.py --bytecode-3.4 --syntax-verify
 
 #: Check deparsing Python 3.5
 check-bytecode-3.5:
-	$(PYTHON) test_pythonlib.py --bytecode-3.5 --weak-verify
 	$(PYTHON) test_pythonlib.py --bytecode-3.5-run --verify-run
+	$(PYTHON) test_pythonlib.py --bytecode-3.5 --syntax-verify
 
 #: Check deparsing Python 3.6
 check-bytecode-3.6:
-	$(PYTHON) test_pythonlib.py --bytecode-3.6 --weak-verify
 	$(PYTHON) test_pythonlib.py --bytecode-3.6-run --verify-run
+	$(PYTHON) test_pythonlib.py --bytecode-3.6 --syntax-verify
 
 #: Check deparsing Python 3.7
 check-bytecode-3.7:
-	$(PYTHON) test_pythonlib.py --bytecode-3.7 --weak-verify
+	$(PYTHON) test_pythonlib.py --bytecode-3.7-run --verify-run
+	$(PYTHON) test_pythonlib.py --bytecode-3.7 --syntax-verify
+
+#: Check deparsing Python 3.8
+check-bytecode-3.8:
+	$(PYTHON) test_pythonlib.py --bytecode-3.8-run --verify-run
+	$(PYTHON) test_pythonlib.py --bytecode-3.8 --syntax-verify
 
 #: short tests for bytecodes only for this version of Python
 check-native-short:
-	$(PYTHON) test_pythonlib.py --bytecode-$(PYTHON_VERSION) --weak-verify $(COMPILE)
+	$(PYTHON) test_pythonlib.py --bytecode-$(PYTHON_VERSION) --syntax-verify $(COMPILE)
 	$(PYTHON) test_pythonlib.py --bytecode-$(PYTHON_VERSION)-run --verify-run $(COMPILE)
 
 #: Run longer Python 2.6's lib files known to be okay
 check-2.6-ok:
-	$(PYTHON) test_pythonlib.py --ok-2.6 --weak-verify $(COMPILE)
+	$(PYTHON) test_pythonlib.py --ok-2.6 --syntax-verify $(COMPILE)
 
 #: Run longer Python 2.7's lib files known to be okay
 check-2.7-ok:
-	$(PYTHON) test_pythonlib.py --ok-2.7 --weak-verify $(COMPILE)
+	$(PYTHON) test_pythonlib.py --ok-2.7 --syntax-verify $(COMPILE)
 
 #: Run longer Python 3.2's lib files known to be okay
 check-3.2-ok:
-	$(PYTHON) test_pythonlib.py --ok-3.2 --weak-verify $(COMPILE)
+	$(PYTHON) test_pythonlib.py --ok-3.2 --syntax-verify $(COMPILE)
 
 #: Run longer Python 3.4's lib files known to be okay
 check-3.4-ok:
-	$(PYTHON) test_pythonlib.py --ok-3.4 --weak-verify $(COMPILE)
+	$(PYTHON) test_pythonlib.py --ok-3.4 --syntax-verify $(COMPILE)
 
 #: PyPy of some sort. E.g. [PyPy 5.0.1 with GCC 4.8.4]
 # Skip for now
@@ -297,6 +350,25 @@ pypy-2.7 5.0 5.3 6.0:
 pypy-3.2 2.4:
 	$(PYTHON) test_pythonlib.py --bytecode-pypy3.2 --verify
 
+#: PyPy 5.0.x with Python 3.6.1 ...
+check-bytecode-pypy3.6: 7.1
+7.1:
+	$(PYTHON) test_pythonlib.py --bytecode-pypy3.6-run --verify-run
+	$(PYTHON) test_pythonlib.py --bytecode-pypy3.6 --verify
+
+#: PyPy 5.0.x with Python 3.6.9
+check-bytecode-pypy3.6: 7.2 7.3
+
+7.2:
+	$(PYTHON) test_pythonlib.py --bytecode-pypy3.6-run --verify-run
+	$(PYTHON) test_pythonlib.py --bytecode-pypy3.6 --verify
+
+7.3:
+	$(PYTHON) test_pythonlib.py --bytecode-pypy3.7 --verify-run
+
+# Pyston 2.3
+2.3: check-3.8
+
 clean: clean-py-dis clean-dis clean-unverified
 
 clean-dis:

test/add-test.py

@@ -1,16 +1,49 @@
 #!/usr/bin/env python
-""" Trivial helper program to bytecompile and run an uncompile
+""" Trivial helper program to byte compile and uncompile the bytecode file.
 """
 import os, sys, py_compile
-assert len(sys.argv) >= 2
+from xdis.version_info import version_tuple_to_str, PYTHON_VERSION_TRIPLE
+
+if len(sys.argv) < 2:
+    print("Usage: add-test.py [--run] *python-source*... [optimize-level]")
+    sys.exit(1)
+
+assert 2 <= len(sys.argv) <= 4
 version = sys.version[0:3]
-for path in sys.argv[1:]:
+vers = sys.version_info[:2]
+if sys.argv[1] in ("--run", "-r"):
+    suffix = "_run"
+    assert len(sys.argv) >= 3
+    py_source = sys.argv[2:]
+    i = 2
+else:
+    suffix = ""
+    py_source = sys.argv[1:]
+    i = 1
+try:
+    optimize = int(sys.argv[-1])
+    assert sys.argv >= i + 2
+    py_source = sys.argv[i:-1]
+    i = 2
+
+except:
+    optimize = 2
+
+for path in py_source:
     short = os.path.basename(path)
-    if hasattr(sys, 'pypy_version_info'):
-        cfile =  "bytecode_pypy%s/%s" % (version, short) + 'c'
+    if short.endswith(".py"):
+        short = short[: -len(".py")]
+    if hasattr(sys, "pypy_version_info"):
+        version = version_tuple_to_str(end=2, delimiter="")
+        bytecode = "bytecode_pypy%s%s/%spy%s.pyc" % (version, suffix, short, version)
     else:
-        cfile =  "bytecode_%s/%s" % (version, short) + 'c'
-    print("byte-compiling %s to %s" % (path, cfile))
-    py_compile.compile(path, cfile)
-    if isinstance(version, str) or version >= (2, 6, 0):
-        os.system("../bin/uncompyle6 -a -t %s" % cfile)
+        version = version_tuple_to_str(end=2)
+        bytecode = "bytecode_%s%s/%s.pyc" % (version, suffix, short)
+
+    print("byte-compiling %s to %s" % (path, bytecode))
+    if PYTHON_VERSION_TRIPLE >= (3, 2):
+        py_compile.compile(path, bytecode, optimize=optimize)
+    else:
+        py_compile.compile(path, bytecode)
+    if PYTHON_VERSION_TRIPLE >= (2, 6):
+        os.system("../bin/uncompyle6 -a -t %s" % bytecode)