Readme link typo

until we can more fully untangle if stmts in loops.
Current tests break urllib2.pyc and cgi.pyc

.circleci/config.yml

@@ -1,11 +1,3 @@
-# This configuration was automatically generated from a CircleCI 1.0 config.
-# It should include any build commands you had along with commands that CircleCI
-# inferred from your project structure. We strongly recommend you read all the
-# comments in this file to understand the structure of CircleCI 2.0, as the idiom
-# for configuration has changed substantially in 2.0 to allow arbitrary jobs rather
-# than the prescribed lifecycle of 1.0. In general, we recommend using this generated
-# configuration as a reference rather than using it in production, though in most
-# cases it should duplicate the execution of your original 1.0 config.
 version: 2
 jobs:
   build:
@@ -18,18 +10,10 @@ jobs:
       CIRCLE_ARTIFACTS: /tmp/circleci-artifacts
       CIRCLE_TEST_REPORTS: /tmp/circleci-test-results
       COMPILE: --compile
-    # In CircleCI 1.0 we used a pre-configured image with a large number of languages and other packages.
-    # In CircleCI 2.0 you can now specify your own image, or use one of our pre-configured images.
-    # The following configuration line tells CircleCI to use the specified docker image as the runtime environment for you job.
-    # We have selected a pre-built image that mirrors the build environment we use on
-    # the 1.0 platform, but we recommend you choose an image more tailored to the needs
-    # of each job. For more information on choosing an image (or alternatively using a
-    # VM instead of a container) see https://circleci.com/docs/2.0/executor-types/
     # To see the list of pre-built images that CircleCI provides for most common languages see
     # https://circleci.com/docs/2.0/circleci-images/
     docker:
-    - image: circleci/build-image:ubuntu-14.04-XXL-upstart-1189-5614f37
+      - image: circleci/python:3.6.9
-      command: /sbin/init
     steps:
     # Machine Setup
     #   If you break your build into multiple jobs with workflows, you will probably want to do the parts of this that are relevant in each
@@ -42,25 +26,24 @@ jobs:
     # This is based on your 1.0 configuration file or project settings
     - run:
         working_directory: ~/rocky/python-uncompyle6
-        command: pyenv local 2.7.11 && pyenv rehash && pip install virtualenv && pip install nose && pip install pep8 && pip install six && pyenv rehash
+        command: pip install --user virtualenv && pip install --user nose && pip install --user pep8
     # Dependencies
     #   This would typically go in either a build or a build-and-test job when using workflows
     # Restore the dependency cache
     - restore_cache:
         keys:
-        # This branch if available
+          - v2-dependencies-{{ .Branch }}-
-        - v1-dep-{{ .Branch }}-
+          # fallback to using the latest cache if no exact match is found
-        # Default branch if not
+          - v2-dependencies-
-        - v1-dep-master-
+
-        # Any branch if there are none on the default branch - this should be unnecessary if you have your default branch configured correctly
-        - v1-dep-
     # This is based on your 1.0 configuration file or project settings
-    - run: pip install --upgrade setuptools
+    - run: pip install --user --upgrade setuptools
-    - run: pip install -e .
+    - run: pip install --user -e .
-    - run: pip install pytest==3.2.5 hypothesis==3.0.0
+    - run: pip install --user -r requirements-dev.txt
+
     # Save dependency cache
     - save_cache:
-        key: v1-dep-{{ .Branch }}-{{ epoch }}
+        key: v2-dependencies-{{ .Branch }}-{{ epoch }}
         paths:
         # This is a broad list of cache paths to include many possible development environments
         # You can probably delete some of these entries
@@ -69,14 +52,13 @@ jobs:
         - ~/.m2
         - ~/.ivy2
         - ~/.bundle
-        - ~/.go_workspace
-        - ~/.gradle
         - ~/.cache/bower
+
     # Test
     #   This would typically be a build job when using workflows, possibly combined with build
     # This is based on your 1.0 configuration file or project settings
-    - run: python ./setup.py develop && make check-2.7
+    - run: sudo python ./setup.py develop && make check-3.6
-    - run: cd ./test/stdlib && pyenv local 2.7.11 && bash ./runtests.sh 'test_[p-z]*.py'
+    - run: cd ./test/stdlib && bash ./runtests.sh 'test_[p-z]*.py'
     # Teardown
     #   If you break your build into multiple jobs with workflows, you will probably want to do the parts of this that are relevant in each
     # Save test results

.github/ISSUE_TEMPLATE/bug-report.md

@@ -11,7 +11,7 @@ Please remove any of the optional sections if they are not applicable.
 Prerequisites
 
 * Make sure the bytecode you have can be disassembled with a
-  disassembler.
+  disassembler and produces valid results.
 * Don't put bytecode and corresponding source code on any service that
   requires registration to download.
 * When you open a bug report there is no privacy. If the legitimacy of
@@ -35,7 +35,7 @@ decompiler service for versions of Python up to 2.6.
 
 ## How to Reproduce
 
-<!-- Please show both the input you gave and the
+<!-- Please show both the *input* you gave and the
 output you got in describing how to reproduce the bug:
 
 or give a complete console log with input and output
@@ -63,7 +63,7 @@ can add that too.
 Please modify for your setup
 
 - Uncompyle6 version: output from  `uncompyle6 --version` or `pip show uncompyle6`
-- Python version: `python -V`
+- Python version for the version of Python the byte-compiled the file: `python -c "import sys; print(sys.version)"` where `python` is the correct Cpython or Pypy binary.
 - OS and Version: [e.g. Ubuntu bionic]
 
 -->

.gitignore

@@ -6,6 +6,7 @@
 /.eggs
 /.hypothesis
 /.idea
+/.mypy_cache
 /.pytest_cache
 /.python-version
 /.tox

.travis.yml

@@ -3,9 +3,9 @@ language: python
 python:
   - '3.5'
   - '2.7'
-  - '2.6'
   - '3.4'
   - '3.6'
+  - '3.8'
 
 matrix:
   include:

Makefile

@@ -47,8 +47,12 @@ check-3.8:
 # Skip for now
 2.6 5.0 5.3 5.6 5.8:
 
-#:PyPy pypy3-2.4.0 Python 3:
+#:PyPy pypy3-2.4.0 Python 3.6.1:
-pypy-3.2 2.4:
+7.1 pypy-3.2 2.4:
+	$(MAKE) -C test $@
+
+#:PyPy pypy3-2.4.0 Python 3.6.9:
+7.2:
 	$(MAKE) -C test $@
 
 #: Run py.test tests

NEWS.md

@@ -1,21 +1,267 @@
+3.6.4: 2020-2-9 Plateau
+=======================
+
+The main focus in this release was fix some of the more glaring problems creapt in from the last release due to that refactor.
+
+`uncompyle6` code is at a plateau where what is most needed is a code refactoring. In doing this, until everything refactored and replaced, decomplation may get worse.
+Therefore, this release largely serves as a checkpoint before more major upheaval.
+
+The upheaval, in  started last release, I believe the pinnicle was around c90ff51 which wasn't a release. I suppose I should tag that.
+
+After c90ff5, I started down the road of redoing control flow in a more comprehensible, debuggable, and scalable way. See [The Control Flow Mess](https://github.com/rocky/python-uncompyle6/wiki/The-Control-Flow-Mess)
+
+The bulk of the refactoring going on in the [decompyle3](https://github.com/rocky/python-decompil3) project, but I try to trickle down the changes.
+
+It is tricky because the changes are large and I have to figure decompose things so that little testable pieces can be done. And there is also the problem that what is in decompyle3 is incomplete as well.
+
+Other than control flow, another change that will probably happen in the next release is to redo the grammar for lambda expressions. Right now, we treat them as Python statements, you know, things with compound statements in them. But lambda aren't that. And so there is hackery to paper over difference making a statement out of an expression the wrong thing to do. For example, a return of an "and" expression can be expressed as nested "if" statements with return inside them, but the "if" variant of the bytecode is not valid in a lambda.
+
+In the decompyle3 code, I've gone down the road making the grammar goal symbol be an expression. This also offers the opportunity to split the grammar making parsing inside lambda not only more reliable because the wrong choices don't exist, but also simpler and faster because all those rules just need don't need to exist in parsing.
+
+I cringe in thinking about how the code has lived for so long without noticing such a simple stupidity, and lapse of sufficient thought.
+
+Some stats from testing. The below give numbers of decompiled tests from Python's test suite which succesfully ran
+
+```
+   Version  test-suites passing
+   -------  -------------------
+   2.4.6     243
+   2.5.6     265
+   2.6.9     305
+   3.3.7     300
+   3.4.10    304
+   3.5.9     260
+   3.6.10    236
+   3.7.6     306
+   3.8.1     114
+```
+
+Decompiled bytecode files distributed with Python (syntax check only):
+
+```
+2.7.17  647 files:   0 failed
+3.2.6   900 files:   0 failed
+3.3.7  1256 files:   0 failed
+3.4.10  800 files:   0 failed
+3.5.9   900 files:   0 failed
+3.6.10 1300 files:  28 failed
+```
+
+
+3.6.3: 2020-1-26 Martin and Susanne
+===================================
+
+Of late, every release fixes major gaps and embarrassments of the last release....
+
+And in some cases, like this one, exposes lacuna and rot.
+
+I now have [control] flow under control, even if it isn't the most optimal way.
+
+I now have greatly expanded automated testing.
+
+On the most recent Python versions I regularly decompile thousands of Python programs that are distributed with Python.  when it is possible, I then decompile Python's standard test suite distributed with Python and run the decompiled source code which basically checks itself. This amounts to about 250 test programs per version. This is in addition to the 3 CI testing services which do different things.
+
+Does this mean the decompiler works perfectly? No. There are still a dozen or so failing programs, although the actual number of bugs is probably smaller though.
+
+However, in perparation of a more major refactoring of the parser grammar, this release was born.
+
+In many cases, decompilation is better. But there are some cases where decompilation has gotten worse. For lack of time (and interest) 3.0 bytecode suffered a hit. Possibly some code in the 3.x range did too. In time and with cleaner refactored code, this will come back.
+
+Commit c90ff51 was a local maxiumum before, I started reworking the grammar to separate productions that were specific to loops versus those that are not in loops.
+In the middle of that I added another grammar simplication to remove singleton productions of the form `sstmts-> stmts`. These were always was a bit ugly, and complicated output.
+
+At any rate if decompilation fails, you can try c90ff51. Or another decompiler. `unpyc37` is pretty good for 3.7. wibiti `uncompyle2` is great for 2.7. `pycdc` is mediocre for Python before 3.5 or so, and not that good for the most recent Python. Geerally these programs will give some sort of answer even if it isn't correct.
+
+decompyle3 isn't that good for 3.7 and worse for 3.8, but right now it does things no other Python decompiler like `unpyc37` or `pycdc` does. For example, `decompyle3` handles variable annotations. As always, the issue trackers for the various programs will give you a sense for what needs to be done. For now, I've given up on reporting issues in the other decompilers because there are already enough issues reported, and they are just not getting fixed anyway.
+
+
+3.6.2: 2020-1-5 Samish
+======================
+
+Yet again the focus has been on just fixing bugs, mostly geared in the
+later 3.x range. To get some sense what sill needs fixing, consult
+test/stdlib/runtests.sh. And that only has a portion of what's known.
+
+`make_function.py` has gotten so complex that it was split out into 3 parts
+to handle different version ranges: Python <3, Python 3.0..3.6 and Python 3.7+.
+
+An important fix is that we had been dropping docstrings in Python 3 code as a result
+of a incomplete merge from the decompile3 base with respect to the transform phase.
+
+Also important (at least to me) is that we can now handle 3.6+
+variable type annotations.  Some of the decompile3 code uses that in
+its source code, and I now use variable annotations in conjunction
+with mypy in some of my other Python projects
+
+Code generation for imports, especially where the import is dotted
+changed a bit in 3.7; with this release are just now tracking that
+change better. For this I've added pseudo instruction
+`IMPORT_NAME_ATTR`, derived from the `IMPORT_NAME` instruction, to
+indicate when an import contains a dotted import. Similarly, code for
+3.7 `import .. as ` is basically the same as `from .. import`, the
+only difference is the target of the name changes to an "alias" in the
+former. As a result, the disambiguation is now done on the semantic
+action side, rathero than in parsing grammar rules.
+
+Some small specific fixes:
+
+* 3.7+ some chained compare parsing has been fixed. Other remain.
+* better if/else rule checking in the 3.4 and below range.
+* 3.4+ keyword-only parameter handling was fixed more generally
+* 3.3 .. 3.5 keyword-only parameter args in lambda was fixed
+
+
+3.6.1: 2019-12-10 Christmas Hannukah
+====================================
+
+Overall, as in the past, the focus has been on just fixing bugs, more geared
+in the later 3.x range. Handling "async for/with" in 3.8+ works better.
+
+Numerous bugs around handling `lambda` with keyword-only and `*` args in the
+3.0-3.8 have been fixed. However many still remain.
+
+`binary_expr` and `unary_expr` have been renamed to `bin_op` and
+`unary_op` to better correspond the Python AST names.
+
+Some work was done Python 3.7+ to handle `and` better; less was done
+along the lines of handling `or`. Much more is needed to improve
+parsing stability of 3.7+. More of what was done with `and` needs to
+be done with `or` and this will happen first in the "decompyle3"
+project.
+
+Later this will probably be extended backwards to handle the 3.6-
+versions better. This however comes with a big decompilation speed
+penalty. When we redo control flow this should go back to normal, but
+for now, accuracy is more important than speed.
+
+Another `assert` transform rule was added. Parser rules to distingish
+`try/finally` in 3.8 were added and we are more stringent about what
+can be turned into an `assert`. There was some grammar cleanup here
+too.
+
+A number of small bugs were fixed, and some administrative changes to
+make `make check-short` really be short, but check more throughly what
+it checks. minimum xdis version needed was bumped to include in the
+newer 3.6-3.9 releases. See the `ChangeLog` for details.
+
+
+3.6.0: 2019-12-10 gecko gecko
+=============================
+
+The main focus in this release was more accurate decompilation especially
+for 3.7 and 3.8. However there are some improvments to Python 2.x as well,
+including one of the long-standing problems of detecting the difference between
+`try ... ` and `try else ...`.
+
+With this release we now rebase Python 3.7 on off of a 3.7 base; This
+is also as it is (now) in decompyle3.  This facilitates removing some of the
+cruft in control-flow detection in the 2.7 uncompyle2 base.
+
+Alas, decompilation speed for 3.7 on is greatly increased. Hopefull
+this is temporary (cough, cough) until we can do a static control flow
+pass.
+
+Finally, runing in 3.9-dev is tolerated. We can disassemble, but no parse tables yet.
+
+
+3.5.1 2019-11-17 JNC
+====================
+
+- Pypy 3.3, 3.5, 3.6, and 3.6.9 support
+- bump xdis version to handle newer Python releases, e.g. 2.7.17, 3.5.8, and 3.5.9
+- Improve 3.0 decompilation
+    - no parse errors on stlib bytecode. However accurate translation in
+	  control-flow and and/or detection needs work
+- Remove extraneous iter() in "for" of list comprehension  Fixes #272
+- "for" block without a `POP_BLOCK `and confusing `JUMP_BACK` for `CONTINUE`. Fixes #293
+- Fix unmarshal incompletness detected in Pypy 3.6
+- Miscellaneous bugs fixed
+
+3.5.0 2019-10-12 Stony Brook Ride
+=================================
+
+- Fix fragment bugs
+   * missing `RETURN_LAST` introduced when adding transformation layer
+   * more parent entries on tokens
+- Preliminary support for decompiling Python 1.0, 1.1, 1.2, and 1.6
+   * Newer _xdis_ version needed
+
+3.4.1 2019-10-02
+================
+
+- Correct assert{,2} transforms Fixes #289
+- Fragment parsing fixes:
+     * Wasn't handling 3-arg `%p`
+   	 * fielding error in `code_deparse()`
+- Use newer _xdis_ to better track Python 3.8.0
+
+
+3.4.0 2019-08-24 Totoro
+=======================
+
+The main change is to add a tree-transformation phase. This simplifies the
+code a little and allows us to turn `if ...: raise AssertionError` into
+`assert`, and many `if ..: else if ...` into `if ... elif ..`
+
+Use options `--show=before` and `--show=after` to see the before the tree transformation phase and after the tree transformation phase.
+
+Most of the heavy lifting for this was done by x0ret.
+
+Other changes:
+
+- Fix issue #275, #283 (process to fix this bug is documented on wiki), #284
+- blacken more code
+- CircleCI adjustments for a changing CircleCi
+- Require more recent `xdis` for Python 3.8
+- Fix bugs in code using `BUILD_LIST_UNPACK` and variants
+
+3.3.5 2019-07-03 Pre Independence Day
+=====================================
+
+Again, most of the work in this is release is thanks to x0ret.
+
+- Handle annotation arguments in Python 3.x
+- Fix _vararg_ and function signatures in 3.x
+- Some 3.x < 3.6 `while` (1)/`if` fixes &mdash; others remain
+- Start reinstating `else if` -> `elif`
+- `LOAD_CONST` -> `LOAD_CODE` where appropriate
+- option `--weak-verify` is now `--syntax-verify`
+- code cleanups, start using [black](https://github.com/python/black) to reformat text
+
+
+3.3.4 2019-06-19 Fleetwood at 65
+================================
+
+Most of the work in this is release is thanks to x0ret.
+
+- Major work was done by x0ret to correct function signatures and include annotation types
+- Handle Python 3.6 `STORE_ANNOTATION` [#58](https://github.com/rocky/python-uncompyle6/issues/58)
+- Friendlier assembly output
+- `LOAD_CONST` replaced by `LOAD_STR` where appropriate to simplify parsing and improve clarity
+- remove unneeded parenthesis in a generator expression when it is the single argument to the function [#247](https://github.com/rocky/python-uncompyle6/issues/246)
+- Bug in noting an async function [#246](https://github.com/rocky/python-uncompyle6/issues/246)
+- Handle Unicode docstrings and fix docstring bugs [#241](https://github.com/rocky/python-uncompyle6/issues/241)
+- Add short option -T as an alternate for --tree+
+- Some grammar cleanup
+
 3.3.3 2019-05-19 Henry and Lewis
 ================================
 
 As before, decomplation bugs fixed. The focus has primarily been on
 Python 3.7. But with this release, releases will be put on hold,as a
-better control-flow detection is worked on . Tis has been needed for a
+better control-flow detection is worked on . This has been needed for a
 while, and is long overdue. It will probably also take a while to get
 done as good as what we have now.
 
 However this work will be done in a new project
 [decompyle3](https://github.com/rocky/python-decompile3).  In contrast
-to _uncompyle6_ the code wil be written assuming a modern Python 3,
+to _uncompyle6_ the code will be written assuming a modern Python 3,
 e.g. 3.7. It is originally intended to decompile Python version 3.7
 and greater.
 
 * A number of Python 3.7+ chained comparisons were fixed
 * Revise Python 3.6ish format string handling
-* Go over operator precedence, e.g. for AST IfExp
+* Go over operator precedence, e.g. for AST `IfExp`
 
 Reported Bug Fixes
 ------------------
@@ -38,7 +284,7 @@ some span back as far as 2.x
 But as before, many more remain in the 3.7 and 3.8 range which will
 get addressed in future releases
 
-Pypy 3.6 support was started. Pypy 3.x detection fixed (via xdis)
+Pypy 3.6 support was started. Pypy 3.x detection fixed (via `xdis`)
 
 3.3.1 2019-04-19 Good Friday
 ==========================
@@ -47,7 +293,7 @@ Lots of decomplation bugs, especially in the 3.x series fixed. Don't worry thoug
 
 * Add annotation return values in 3.6+
 * Fix 3.6+ lambda parameter handling decompilation
-* Fix 3.7+ chained comparision decompilation
+* Fix 3.7+ chained comparison decompilation
 * split out semantic-action customization into more separate files
 * Add 3.8 try/else
 * Fix 2.7 generator decompilation
@@ -57,8 +303,7 @@ Lots of decomplation bugs, especially in the 3.x series fixed. Don't worry thoug
 3.3.0 2019-04-14 Holy Week
 ==========================
 
-* First cut at Python 3.8 (many bugs remain)
+* First cut at Python 3.8 (many bug remain)
-* Reinstate -c | --compile (compile before disassembly) option
 * The usual smattering of bug and doc fixes
 
 3.2.6 2019-03-23 Mueller Report
@@ -80,14 +325,14 @@ Bug Fixes
 Pull Requests
 ----------------
 
-* [#202: Better "assert" statement detemination in Python 2.7](https://github.com/rocky/python-uncompyle6/pull/211)
+* [#202: Better "assert" statement determination in Python 2.7](https://github.com/rocky/python-uncompyle6/pull/211)
 * [#204: Python 3.7 testing](https://github.com/rocky/python-uncompyle6/pull/204)
 * [#205: Run more f-string tests on Python 3.7](https://github.com/rocky/python-uncompyle6/pull/205)
 * [#211: support utf-8 chars in Python 3 sourcecode](https://github.com/rocky/python-uncompyle6/pull/202)
 
 
 
-3.2.5 2018-12-30 Clearout sale
+3.2.5 2018-12-30 Clear-out sale
 ======================================
 
 - 3.7.2 Remove deprecation warning on regexp string that isn't raw
@@ -122,7 +367,7 @@ Pull Requests
 
 - Add rudimentary 1.4 support (still a bit buggy)
 - add --tree+ option to show formatting rule, when it is constant
-- Python 2.7.15candidate1 support (via xdis)
+- Python 2.7.15candidate1 support (via `xdis`)
 - bug fixes, especially for 3.7 (but 2.7 and 3.6 and others as well)
 
 3.1.3 2018-04-16
@@ -152,14 +397,14 @@ Jesus on Friday's New York Times puzzle: "I'm stuck on 2A"
 - reduce 3.5, 3.6 control-flow bugs
 - reduce ambiguity in rules that lead to long (exponential?) parses
 - limit/isolate some 2.6/2.7,3.x grammar rules
-- more runtime testing of decompiled code
+- more run-time testing of decompiled code
-- more removal of parenthesis around calls via setting precidence
+- more removal of parenthesis around calls via setting precedence
 
 3.1.0 2018-03-21 Equinox
 ==============================
 
 - Add code_deparse_with_offset() fragment function.
-- Correct paramenter call fragment deparse_code()
+- Correct parameter call fragment deparse_code()
 - Lots of 3.6, 3.x, and 2.7 bug fixes
   About 5% of 3.6 fail parsing now. But
   semantics still needs much to be desired.
@@ -227,24 +472,24 @@ function calls and definitions.
 2.15.1 2018-01-27
 =====================
 
-- Add --linemap option to give line correspondences
+- Add `--linemap` option to give line correspondences
   between original source lines and reconstructed line sources.
   It is far from perfect, but it is a start
 - Add a new class of tests: tests which when decompiled check themselves
 - Split off Python version semantic action customizations into its own file
-- Fix 2.7 bug in ifelse loop statement
+- Fix 2.7 bug in `if`/`else` loop statement
-- Handle 3.6+ EXTENDED_ARGs for POP_JUMP_IF... instructions
+- Handle 3.6+ `EXTENDED_ARG`s for `POP_JUMP_IF..` instructions
-- Correct 3.6+ calls with kwargs
+- Correct 3.6+ calls with `kwargs`
 - Describe the difficulty of 3.6 in README
 
 2.14.3 2018-01-19
 =====================
 
-- Fix bug in 3.5+ await stmt
+- Fix bug in 3.5+ `await` statement
 - Better version to magic handling; handle 3.5.2 .. 3.5.4 versions
 - Improve/correct test_pyenvlib.py status messages
 - Fix some 2.7 and 2.6 parser bugs
-- Fix whilelse parsing bugs
+- Fix `whilelse` parsing bugs
 - Correct 2.5- decorator parsing
 - grammar for decorators matches AST a little more
 - better tests in setup.py for running the right version of Python
@@ -255,15 +500,15 @@ function calls and definitions.
 
 Decompilation bug fixes, mostly 3.6 and pre 2.7
 
-- 3.6 FUNCTION_EX (somewhat)
+- 3.6 `FUNCTION_EX` (somewhat)
-- 3.6 FUNCTION_EX_KW fixes
+- 3.6 `FUNCTION_EX_KW` fixes
-- 3.6 MAKE_FUNCTION fixes
+- 3.6 `MAKE_FUNCTION` fixes
-- correct 3.5 CALL_FUNCTION_VAR
+- correct 3.5 `CALL_FUNCTION_VAR`
 - stronger 3.x "while 1" testing
 - Fix bug in if's with "pass" bodies. Fixes #104
 - try/else and try/finally fixes on 2.6-
 - limit pypy customization to pypy
-- Add addr fields in COME_FROMS
+- Add addr fields in `COME_FROM`S
 - Allow use of full instructions in parser reduction routines
 - Reduce grammar in Python 3 by specialization more to specific
   Python versions
@@ -272,11 +517,11 @@ Decompilation bug fixes, mostly 3.6 and pre 2.7
 2.14.1 2017-12-10 Dr. Gecko
 ===================================
 
-- Many decompilation bugfixes
+- Many decompilation bug fixes
 - Grammar rule reduction and version isolation
 - Match higher-level nonterminal names more closely
   with Python AST
-- Start automated Python stdlib testing - full round trip
+- Start automated Python _stdlib_ testing &mdash; full round trip
 
 2.14.0 2017-11-26 johnnybamazing
 =========================================
@@ -285,7 +530,7 @@ Decompilation bug fixes, mostly 3.6 and pre 2.7
   and remove used grammar rules
 - Fix a number of bytecode decompile problems
   (many more remain)
-- Add stdlib/runtests.sh for even more rigorous testing
+- Add `stdlib/runtests.sh` for even more rigorous testing
 
 2.13.3 2017-11-13
 =====================
@@ -301,16 +546,16 @@ Overall: better 3.6 decompiling and some much needed code refactoring and cleanu
   rather trying to parse the bytecode array. This largely been done in for versions 3.x;
   3.0 custom mangling code has been reduced;
   some 2.x conversion has been done, but more is desired. This make it possible to...
-- Handle EXTENDED_ARGS better. While relevant to all Python versions it is most noticeable in
+- Handle `EXTENDED_ARGS` better. While relevant to all Python versions it is most noticeable in
-  version 3.6+ where in switching to wordcodes the size of operands has been reduced from 2**16
+  version 3.6+ where in switching to wordcodes the size of operands has been reduced from 2^16
-  to 2**8. JUMP instruction then often need EXTENDED_ARGS.
+  to 2^8. `JUMP` instruction then often need EXTENDED_ARGS.
 - Refactor find_jump_targets() with via working of of instructions rather the bytecode array.
-- use --weak-verify more and additional fuzzing on verify()
+- use `--weak-verify` more and additional fuzzing on verify()
 - fragment parser now ignores errors in nested function definitions; an parameter was
   added to assist here. Ignoring errors may be okay because the fragment parser often just needs,
   well, *fragments*.
-- Distinguish RETURN_VALUE from RETURN_END_IF in exception bodies better in 3.6
+- Distinguish `RETURN_VALUE` from `RETURN_END_IF` in exception bodies better in 3.6
-- bug in 3.x language changes: import queue via import Queue
+- bug in 3.x language changes: import queue via `import Queue`
 - reinstate some bytecode tests since decompiling has gotten better
 - Revise how to report a bug
 
@@ -330,12 +575,12 @@ Overall: better 3.6 decompiling and some much needed code refactoring and cleanu
 - Fixes in deparsing lambda expressions
 - Improve table-semantics descriptions
 - Document hacky customize arg count better (until we can remove it)
-- Update to use xdis 3.7.0 or greater
+- Update to use `xdis` 3.7.0 or greater
 
 2.12.0 2017-09-26
 =====================
 
-- Use xdis 3.6.0 or greater now
+- Use `xdis` 3.6.0 or greater now
 - Small semantic table cleanups
 - Python 3.4's terms a little names better
 - Slightly more Python 3.7, but still failing a lot
@@ -349,13 +594,13 @@ Overall: better 3.6 decompiling and some much needed code refactoring and cleanu
 2.11.4 2017-08-15
 =====================
 
-* scanner and parser now allow 3-part version string lookups,
+* scanner and parser now allow 3-part version string look ups,
   e.g. 2.7.1 We allow a float here, but if passed a string like '2.7'. or
-* unpin 3.5.1. xdis 3.5.4 has been release and fixes the problems we had. Use that.
+* unpin 3.5.1. `xdis` 3.5.4 has been release and fixes the problems we had. Use that.
-* some routines here moved to xdis. Use the xdis version
+* some routines here moved to `xdis`. Use the `xdis` version
-* README.rst: Link typo Name is trepan2 now not trepan
+* `README.rst`: Link typo Name is _trepan2_ now not _trepan_
-* xdis-forced change adjust for COMPARE_OP "is-not" in
+* xdis-forced change adjust for `COMPARE_OP` "is-not" in
-  semanatic routines. We need "is not".
+  semantic routines. We need "is not".
 * Some PyPy tolerance in validate testing.
 * Some pyston tolerance
 
@@ -365,15 +610,15 @@ Overall: better 3.6 decompiling and some much needed code refactoring and cleanu
 Very minor changes
 
 - RsT doc fixes and updates
-- use newer xdis, but not too new; 3.5.2 breaks uncompyle6
+- use newer `xdis`, but not too new; 3.5.2 breaks uncompyle6
-- use xdis opcode sets
+- use `xdis` opcode sets
-- xdis "exception match" is now "exception-match"
+- `xdis` "exception match" is now "exception-match"
 
 2.11.2 2017-07-09
 =====================
 
 - Start supporting Pypy 3.5 (5.7.1-beta)
-- use xdis 3.5.0's opcode sets and require xdis 3.5.0
+- use `xdis` 3.5.0's opcode sets and require `xdis` 3.5.0
 - Correct some Python 2.4-2.6 loop detection
 - guard against badly formatted bytecode
 
@@ -381,55 +626,55 @@ Very minor changes
 =====================
 
 - Python 3.x annotation and function signature fixes
-- Bump xdis version
+- Bump `xdis` version
-- Small pysource bug fixes
+- Small `pysource.py` bug fixes
 
 2.11.0 2017-06-18 Fleetwood
 ==================================
 
 - Major improvements in fragment tracking
-  * Add nonterminal node in extractInfo
+  * Add nonterminal node in `extractInfo()`
   * tag more offsets in expressions
   * tag array subscripts
-  * set YIELD value offset in a <yield> expr
+  * set `YIELD` value offset in a _yield expr_
   * fix a long-standing bug in not adjusting final AST when melding other deparse ASTs
 - Fixes yet again for make_function node handling; document what's up here
-- Fix bug in snowflake Python 3.5 *args kwargs
+- Fix bug in snowflake Python 3.5 `*args`, `kwargs`
 
 2.10.1 2017-06-3 Marylin Frankel
 ========================================
 
 - fix some fragments parsing bugs
-  - was returning the wrong type sometimes in deparse_code_around_offset()
+  - was returning the wrong type sometimes in `deparse_code_around_offset()`
   - capture function name in offsets
-  - track changes to ifelstrmtr node from pysource into fragments
+  - track changes to `ifelstrmtr` node from `pysource.py` into fragments
 
 2.10.0 2017-05-30 Elaine Gordon
 =======================================
 
 - Add fuzzy offset deparse look up
 - 3.6 bug fixes
-   - fix EXTENDED_ARGS handling (and in 2.6 and others)
+   - fix `EXTENDED_ARGS` handling (and in 2.6 and others)
    - semantic routine make_function fragments.py
-   - MAKE_FUNCTION handling
+   - `MAKE_FUNCTION` handling
-   - CALL_FUNCTION_EX handling
+   - `CALL_FUNCTION_EX` handling
-   - async property on defs
+   - `async` property on `defs`
-   - support for CALL_FUNCTION_KW (moagstar)
+   - support for `CALL_FUNCTION_KW` (moagstar)
-- 3.5+ UNMAP_PACK and BUILD_UNMAP_PACK handling
+- 3.5+ `UNMAP_PACK` and` BUILD_UNMAP_PACK` handling
 - 3.5 FUNCTION_VAR bug
-- 3.x pass statement insdie while True
+- 3.x pass statement inside `while True`
 - Improve 3.2 decompilation
-- Fixed -o argument processing (grkov90)
+- Fixed `-o` argument processing (grkov90)
 - Reduce scope of LOAD_ASSERT as expr to 3.4+
-- "await" statement fixes
+- `await` statement fixes
 - 2.3, 2.4 "if 1 .." fixes
 - 3.x annotation fixes
 
 2.9.11 2017-04-06
 =====================
 
-- Better support for Python 3.5+ BUILD_MAP_UNPACK
+- Better support for Python 3.5+ `BUILD_MAP_UNPACK`
-- Start 3.6 CALL_FUNCTION_EX support
+- Start 3.6 `CALL_FUNCTION_EX` support
 - Many decompilation bug fixes. (Many more remain). See ChangeLog
 
 2.9.10 2017-02-25
@@ -437,7 +682,7 @@ Very minor changes
 
 - Python grammar rule fixes
 - Add ability to get grammar coverage on runs
-- Handle Python 3.6 opcode BUILD_CONST_KEYMAP
+- Handle Python 3.6 opcode `BUILD_CONST_KEYMAP`
 
 2.9.9 2016-12-16
 
@@ -453,13 +698,13 @@ Very minor changes
 ====================
 
 - Better control-flow detection
-- pseudo instruction THEN in 2.x
+- pseudo instruction `THEN` in 2.x
   to disambiguate if from and
-- fix bug in --verify option
+- fix bug in `--verify` option
 - DRY (a little) control-flow detection
 - fix syntax in tuples with one element
 - if AST rule inheritance in Python 2.5
-- NAME_MODULE removal for Python <= 2.4
+- `NAME_MODULE` removal for Python <= 2.4
 - verify call fixes for Python <= 2.4
 - more Python lint
 
@@ -470,9 +715,9 @@ Very minor changes
 - Some Python 3.6 bytecode to wordcode conversion fixes
 - option -g: show start-end range when possible
 - track print_docstring move to help (used in python 3.1)
-- verify: allow RETURN_VALUE to match RETURN_END_IF
+- verify: allow `RETURN_VALUE` to match `RETURN_END_IF`
 - some 3.2 compatibility
-- Better Python 3 control flow detection by adding Pseudo ELSE opcodes
+- Better Python 3 control flow detection by adding Pseudo `ELSE` opcodes
 
 2.9.6 2016-12-04
 ====================
@@ -488,7 +733,7 @@ Very minor changes
 
 - Correct MANIFEST.in
 - More AST grammar checking
-- --linemapping option or linenumbers.line_number_mapping()
+- `--linemapping` option or _linenumbers.line_number_mapping()_
   Shows correspondence of lines between source
   and decompiled source
 - Some control flow adjustments in code for 2.x.
@@ -508,7 +753,7 @@ Very minor changes
   * improper while 1 else
   * docstring indent
   * 3.3 default values in lambda expressions
-  * start 3.0 decompilation (needs newer xdis)
+  * start 3.0 decompilation (needs newer `xdis`)
 - Start grammar misparse checking
 
 
@@ -522,12 +767,12 @@ Very minor changes
 2.9.3 2016-10-26
 ====================
 
-Release forced by incompatibility change in xdis 3.2.0.
+Release forced by incompatibility change in` xdis` 3.2.0.
 
 - Python 3.1 bugs:
-  * handle "with ... as"
+  * handle `with`... `as`
-  * handle "with"
+  * handle `with`
-  * Start handling def (...) -> yy (has bugs still)
+  * Start handling `def` (...) -> _yy_ (has bugs still)
 
 - DRY Python 3.x via inheritance
 - Python 3.6 work (from Daniel Bradburn)
@@ -553,12 +798,12 @@ Release forced by incompatibility change in xdis 3.2.0.
 2.9.0 2016-10-09
 ====================
 
-- Use xdis 3.0.0 protocol load_module.
+- Use `xdis` 3.0.0 protocol `load_module`.
   this Forces change in requirements.txt and _pkg_info_.py
 - Start Python 1.5 decompiling; another round of work is needed to
   remove bugs
 - Simplify python 2.1 grammar
-- Fix bug with -t ...  Wasn't showing source text when -t option was given
+- Fix bug with `-t` ...  Wasn't showing source text when `-t` option was given
 - Fix 2.1-2.6 bug in list comprehension
 
 2.8.4 2016-10-08
@@ -567,8 +812,8 @@ Release forced by incompatibility change in xdis 3.2.0.
 - Python 3 disassembly bug fixes
 - Python 3.6 fstring bug fixes (from moagstar)
 - Python 2.1 disassembly
-- COME_FROM suffixes added in Python3
+- `COME_FROM` suffixes added in Python3
-- use .py extension in verification disassembly
+- use `.py` extension in verification disassembly
 
 2.8.3 2016-09-11 live from NYC!
 =======================================
@@ -613,8 +858,8 @@ control-flow structure detection is done.
 - Add Python 2.2 decompilation
 
 - Fix bugs
- * PyPy LOOKUP_METHOD bug
+ * PyPy `LOOKUP_METHOD` bug
- * Python 3.6 FORMAT_VALUE handles expressions now
+ * Python 3.6 `FORMAT_VALUE` handles expressions now
 
 2.8.0 2016-08-03
 ====================
@@ -664,7 +909,7 @@ control-flow structure detection is done.
 ====================
 
 - Improve Python 2.6 bytecode deparsing:
-  stdlib now will deparse something
+  _stdlib_ now will deparse something
 - Better <2.6 vs. 2.7 grammar separation
 - Fix some 2.7 deparsing bugs
 - Fix bug in installing uncompyle6 script
@@ -727,9 +972,9 @@ control-flow structure detection is done.
 2.3.2 2016-05-1
 ===================
 
-- Add --version option standalone scripts
+- Add `--version` option standalone scripts
 - Correct License information in package
-- expose fns uncompyle_file, load_file, and load_module
+- expose functions `uncompyle_file()`, `load_file()`, and `load_module()`
 - Start to DRY Python2 and Python3 grammars Separate out 3.2, and 3.5+
   specific grammar code
 - Fix bug in 3.5+ constant map parsing
@@ -737,12 +982,12 @@ control-flow structure detection is done.
 2.3.0, 2.3.1 2016-04-30
 =============================
 
-- Require spark_parser  >= 1.1.0
+- Require `spark_parser` >= 1.1.0
 
 2.2.0 2016-04-30
 ====================
 
-- Spark is no longer here but pulled separate package spark_parse
+- Spark is no longer here but pulled separate package [spark_parser](https://pypi.org/project/spark_parser/)
 - Python 3 parsing fixes
 - More tests
 
@@ -752,7 +997,7 @@ control-flow structure detection is done.
 - Support single-mode (in addition to exec-mode) compilation
 - Start to DRY Python 2 and Python 3 grammars
 - Fix bug in if else ternary  construct
-- Fix bug in uncomplye6 -d and -r options (via lelicopter)
+- Fix bug in uncomplye6 `-d` and `-r` options (via lelicopter)
 
 
 2.1.3 2016-01-02
@@ -760,7 +1005,7 @@ control-flow structure detection is done.
 
 - Limited support for decompiling Python 3.5
 - Improve Python 3 class deparsing
-- Handle MAKE_CLOSURE opcode
+- Handle `MAKE_CLOSURE` opcode
 - Start to DRY opcode code.
 - increase test coverage
 - fix misc small bugs and some improvements
@@ -802,5 +1047,5 @@ Changes from uncompyle2
 
 SPARK:
   add option to show grammar rules applied
-  allow Python-style # comments in grammar
+  allow Python-style `#` comments in grammar
   Runs on Python 3 and Python 2

README.rst

@@ -1,4 +1,6 @@
-|buildstatus| |Latest Version| |Supported Python Versions|
+|buildstatus|  |Pypi Installs| |Latest Version| |Supported Python Versions|
+
+|packagestatus|
 
 uncompyle6
 ==========
@@ -11,9 +13,9 @@ Introduction
 ------------
 
 *uncompyle6* translates Python bytecode back into equivalent Python
-source code. It accepts bytecodes from Python version 1.3 to version
+source code. It accepts bytecodes from Python version 1.0 to version
 3.8, spanning over 24 years of Python releases. We include Dropbox's
-Python 2.5 bytecode and some PyPy bytecode.
+Python 2.5 bytecode and some PyPy bytecodes.
 
 Why this?
 ---------
@@ -44,14 +46,15 @@ not exist and there is just bytecode. Again, my debuggers make use of
 this.
 
 There were (and still are) a number of decompyle, uncompyle,
-uncompyle2, uncompyle3 forks around. Almost all of them come basically
+uncompyle2, uncompyle3 forks around. Many of them come basically from
-from the same code base, and (almost?) all of them are no longer
+the same code base, and (almost?) all of them are no longer actively
-actively maintained. One was really good at decompiling Python 1.5-2.3
+maintained. One was really good at decompiling Python 1.5-2.3, another
-or so, another really good at Python 2.7, but that only. Another
+really good at Python 2.7, but that only. Another handles Python 3.2
-handles Python 3.2 only; another patched that and handled only 3.3.
+only; another patched that and handled only 3.3.  You get the
-You get the idea. This code pulls all of these forks together and
+idea. This code pulls all of these forks together and *moves
-*moves forward*. There is some serious refactoring and cleanup in this
+forward*. There is some serious refactoring and cleanup in this code
-code base over those old forks.
+base over those old forks. Even more experimental refactoring is going
+on in decompyle3_.
 
 This demonstrably does the best in decompiling Python across all
 Python versions. And even when there is another project that only
@@ -73,11 +76,11 @@ fixed in the other decompilers.
 Requirements
 ------------
 
-The code here can be run on Python versions 2.6 or later, PyPy 3-2.4,
+The code here can be run on Python versions 2.6 or later, PyPy 3-2.4
-or PyPy-5.0.1.  Python versions 2.4-2.7 are supported in the
+and later.  Python versions 2.4-2.7 are supported in the python-2.4
-python-2.4 branch.  The bytecode files it can read have been tested on
+branch.  The bytecode files it can read have been tested on Python
-Python bytecodes from versions 1.4, 2.1-2.7, and 3.0-3.8 and the
+bytecodes from versions 1.4, 2.1-2.7, and 3.0-3.8 and later PyPy
-above-mentioned PyPy versions.
+versions.
 
 Installation
 ------------
@@ -86,15 +89,15 @@ This uses setup.py, so it follows the standard Python routine:
 
 ::
 
-    pip install -e .  # set up to run from source tree
+    $ pip install -e .  # set up to run from source tree
-                      # Or if you want to install instead
+                        # Or if you want to install instead
-    python setup.py install # may need sudo
+    $ python setup.py install # may need sudo
 
 A GNU makefile is also provided so :code:`make install` (possibly as root or
 sudo) will do the steps above.
 
-Testing
+Running Tests
--------
+-------------
 
 ::
 
@@ -122,16 +125,39 @@ For usage help:
 
    $ uncompyle6 -h
 
-If you want strong verification of the correctness of the
+Verification
-decompilation process, add the `--verify` option. But there are
+------------
-situations where this will indicate a failure, although the generated
-program is semantically equivalent. Using option `--weak-verify` will
-tell you if there is something definitely wrong. Generally, large
-swaths of code are decompiled correctly, if not the entire program.
 
-You can also cross compare the results with pycdc_ . Since they work
+In older versions of Python it was possible to verify bytecode by
-differently, bugs here often aren't in that, and vice versa.
+decompiling bytecode, and then compiling using the Python interpreter
+for that bytecode version. Having done this the bytecode produced
+could be compared with the original bytecode. However as Python's code
+generation got better, this no longer was feasible.
 
+If you want Python syntax verification of the correctness of the
+decompilation process, add the :code:`--syntax-verify` option. However since
+Python syntax changes, you should use this option if the bytecode is
+the right bytecode for the Python interpreter that will be checking
+the syntax.
+
+You can also cross compare the results with either another version of
+`uncompyle6` since there are are sometimes regressions in decompiling
+specific bytecode as the overall quality improves.
+
+For Python 3.7 and above, the code in decompyle3_ is generally
+better.
+
+Or try specific another python decompiler like uncompyle2_, unpyc37_,
+or pycdc_.  Since the later two work differently, bugs here often
+aren't in that, and vice versa.
+
+There is an interesting class of these programs that is readily
+available give stronger verification: those programs that when run
+test themselves. Our test suite includes these.
+
+And Python comes with another a set of programs like this: its test
+suite for the standard library. We have some code in :code:`test/stdlib` to
+facilitate this kind of checking too.
 
 Known Bugs/Restrictions
 -----------------------
@@ -146,27 +172,6 @@ All of the Python decompilers that I have looked at have problems
 decompiling Python's control flow. In some cases we can detect an
 erroneous decompilation and report that.
 
-In older versions of Python it was possible to verify bytecode by
-decompiling bytecode, and then compiling using the Python interpreter
-for that bytecode version. Having done this the bytecode produced
-could be compared with the original bytecode. However as Python's code
-generation got better, this is no longer feasible.
-
-The verification that we use that doesn't check bytecode for
-equivalence but does check to see if the resulting decompiled source
-is a valid Python program by running the Python interpreter. Because
-the Python language has changed so much, for best results you should
-use the same Python version in checking as was used in creating the
-bytecode.
-
-There are however an interesting class of these programs that is
-readily available give stronger verification: those programs that
-when run check some computation, or even better themselves.
-
-And already Python has a set of programs like this: the test suite
-for the standard library that comes with Python. We have some
-code in `test/stdlib` to facilitate this kind of checking.
-
 Python support is strongest in Python 2 for 2.7 and drops off as you
 get further away from that. Support is also probably pretty good for
 python 2.3-2.4 since a lot of the goodness of early the version of the
@@ -182,22 +187,28 @@ In the Python 3 series, Python support is is strongest around 3.4 or
 3.0 is weird in that it in some ways resembles 2.6 more than it does
 3.1 or 2.7. Python 3.6 changes things drastically by using word codes
 rather than byte codes. As a result, the jump offset field in a jump
-instruction argument has been reduced. This makes the `EXTENDED_ARG`
+instruction argument has been reduced. This makes the :code:`EXTENDED_ARG`
 instructions are now more prevalent in jump instruction; previously
 they had been rare.  Perhaps to compensate for the additional
-`EXTENDED_ARG` instructions, additional jump optimization has been
+:code:`EXTENDED_ARG` instructions, additional jump optimization has been
 added. So in sum handling control flow by ad hoc means as is currently
 done is worse.
 
-Between Python 3.5, 3.6 and 3.7 there have been major changes to the
+Between Python 3.5, 3.6, 3.7 there have been major changes to the
-`MAKE_FUNCTION` and `CALL_FUNCTION` instructions.
+:code:`MAKE_FUNCTION` and :code:`CALL_FUNCTION` instructions.  Python
+
+Python 3.8 removes :code:`SETUP_LOOP`, :code:`SETUP_EXCEPT`,
+:code:`BREAK_LOOP`, and :code:`CONTINUE_LOOP`, instructions which may
+make control-flow detection harder, lacking the more sophisticated
+control-flow analysis that is planned. We'll see.
 
 Currently not all Python magic numbers are supported. Specifically in
 some versions of Python, notably Python 3.6, the magic number has
-changes several times within a version. 
+changes several times within a version.
 
-**We support only released versions, not candidate versions.** Note however
+**We support only released versions, not candidate versions.** Note
-that the magic of a released version is usually the same as the *last* candidate version prior to release.
+however that the magic of a released version is usually the same as
+the *last* candidate version prior to release.
 
 There are also customized Python interpreters, notably Dropbox,
 which use their own magic and encrypt bytcode. With the exception of
@@ -219,15 +230,16 @@ There is lots to do, so please dig in and help.
 See Also
 --------
 
-* https://github.com/zrax/pycdc : purports to support all versions of Python. It is written in C++ and is most accurate for Python versions around 2.7 and 3.3 when the code was more actively developed. Accuracy for more recent versions of Python 3 and early versions of Python are especially lacking. See its `issue tracker <https://github.com/zrax/pycdc/issues>`_ for details. Currently lightly maintained.
+* https://github.com/rocky/python-decompile3 : Much smaller and more modern code, focusing on 3.7+. Changes in that will get migrated back ehre.
 * https://code.google.com/archive/p/unpyc3/ : supports Python 3.2 only. The above projects use a different decompiling technique than what is used here. Currently unmaintained.
 * https://github.com/figment/unpyc3/ : fork of above, but supports Python 3.3 only. Includes some fixes like supporting function annotations. Currently unmaintained.
-* https://github.com/wibiti/uncompyle2 : supports Python 2.7 only, but does that fairly well. There are situtations where `uncompyle6` results are incorrect while `uncompyle2` results are not, but more often uncompyle6 is correct when uncompyle2 is not. Because `uncompyle6` adheres to accuracy over idiomatic Python, `uncompyle2` can produce more natural-looking code when it is correct. Currently `uncompyle2` is lightly maintained. See its issue `tracker <https://github.com/wibiti/uncompyle2/issues>`_ for more details
+* https://github.com/wibiti/uncompyle2 : supports Python 2.7 only, but does that fairly well. There are situations where :code:`uncompyle6` results are incorrect while :code:`uncompyle2` results are not, but more often uncompyle6 is correct when uncompyle2 is not. Because :code:`uncompyle6` adheres to accuracy over idiomatic Python, :code:`uncompyle2` can produce more natural-looking code when it is correct. Currently :code:`uncompyle2` is lightly maintained. See its issue `tracker <https://github.com/wibiti/uncompyle2/issues>`_ for more details
 * `How to report a bug <https://github.com/rocky/python-uncompyle6/blob/master/HOW-TO-REPORT-A-BUG.md>`_
 * The HISTORY_ file.
 * https://github.com/rocky/python-xdis : Cross Python version disassembler
 * https://github.com/rocky/python-xasm : Cross Python version assembler
 * https://github.com/rocky/python-uncompyle6/wiki : Wiki Documents which describe the code and aspects of it in more detail
+* https://github.com/zrax/pycdc : The README for this C++ code says it aims to support all versions of Python. It is best for Python versions around 2.7 and 3.3 when the code was initially developed. Accuracy for current versions of Python3 and early versions of Python is lacking. Without major effort, it is unlikely it can be made to support current Python 3. See its `issue tracker <https://github.com/zrax/pycdc/issues>`_ for details. Currently lightly maintained.
 
 
 .. _trepan: https://pypi.python.org/pypi/trepan2g
@@ -236,12 +248,18 @@ See Also
 .. _debuggers: https://pypi.python.org/pypi/trepan3k
 .. _remake: https://bashdb.sf.net/remake
 .. _pycdc: https://github.com/zrax/pycdc
+.. _decompyle3: https://github.com/rocky/python-decompile3
+.. _uncompyle2: https://github.com/wibiti/uncompyle2
+.. _unpyc37: https://github.com/andrew-tavera/unpyc37
 .. _this: https://github.com/rocky/python-uncompyle6/wiki/Deparsing-technology-and-its-use-in-exact-location-reporting
 .. |buildstatus| image:: https://travis-ci.org/rocky/python-uncompyle6.svg
 		 :target: https://travis-ci.org/rocky/python-uncompyle6
+.. |packagestatus| image:: https://repology.org/badge/vertical-allrepos/python:uncompyle6.svg
+		 :target: https://repology.org/project/python:uncompyle6/versions
 .. _PJOrion: http://www.koreanrandom.com/forum/topic/15280-pjorion-%D1%80%D0%B5%D0%B4%D0%B0%D0%BA%D1%82%D0%B8%D1%80%D0%BE%D0%B2%D0%B0%D0%BD%D0%B8%D0%B5-%D0%BA%D0%BE%D0%BC%D0%BF%D0%B8%D0%BB%D1%8F%D1%86%D0%B8%D1%8F-%D0%B4%D0%B5%D0%BA%D0%BE%D0%BC%D0%BF%D0%B8%D0%BB%D1%8F%D1%86%D0%B8%D1%8F-%D0%BE%D0%B1%D1%84
 .. _Deobfuscator: https://github.com/extremecoders-re/PjOrion-Deobfuscator
 .. _Py2EXE: https://en.wikipedia.org/wiki/Py2exe
 .. |Supported Python Versions| image:: https://img.shields.io/pypi/pyversions/uncompyle6.svg
 .. |Latest Version| image:: https://badge.fury.io/py/uncompyle6.svg
 		 :target: https://badge.fury.io/py/uncompyle6
+.. |Pypi Installs| image:: https://pepy.tech/badge/uncompyle6/month

__pkginfo__.py

@@ -26,46 +26,46 @@ copyright   = """
 Copyright (C) 2015-2019 Rocky Bernstein <rb@dustyfeet.com>.
 """
 
-classifiers =  ['Development Status :: 5 - Production/Stable',
+classifiers =  ["Development Status :: 5 - Production/Stable",
-                'Intended Audience :: Developers',
+                "Intended Audience :: Developers",
-                'License :: OSI Approved :: GNU General Public License v3 (GPLv3)',
+                "License :: OSI Approved :: GNU General Public License v3 (GPLv3)",
-                'Operating System :: OS Independent',
+                "Operating System :: OS Independent",
-                'Programming Language :: Python',
+                "Programming Language :: Python",
-                'Programming Language :: Python :: 2.4',
+                "Programming Language :: Python :: 2.4",
-                'Programming Language :: Python :: 2.5',
+                "Programming Language :: Python :: 2.5",
-                'Programming Language :: Python :: 2.6',
+                "Programming Language :: Python :: 2.6",
-                'Programming Language :: Python :: 2.7',
+                "Programming Language :: Python :: 2.7",
-                'Programming Language :: Python :: 3.0',
+                "Programming Language :: Python :: 3.0",
-                'Programming Language :: Python :: 3.1',
+                "Programming Language :: Python :: 3.1",
-                'Programming Language :: Python :: 3.2',
+                "Programming Language :: Python :: 3.2",
-                'Programming Language :: Python :: 3.3',
+                "Programming Language :: Python :: 3.3",
-                'Programming Language :: Python :: 3.4',
+                "Programming Language :: Python :: 3.4",
-                'Programming Language :: Python :: 3.5',
+                "Programming Language :: Python :: 3.5",
-                'Programming Language :: Python :: 3.6',
+                "Programming Language :: Python :: 3.6",
-                'Programming Language :: Python :: 3.7',
+                "Programming Language :: Python :: 3.7",
-                'Programming Language :: Python :: 3.8',
+                "Programming Language :: Python :: 3.8",
-                'Topic :: Software Development :: Debuggers',
+                "Topic :: Software Development :: Debuggers",
-                'Topic :: Software Development :: Libraries :: Python Modules',
+                "Topic :: Software Development :: Libraries :: Python Modules",
                 ]
 
 # The rest in alphabetic order
 author             = "Rocky Bernstein, Hartmut Goebel, John Aycock, and others"
 author_email       = "rb@dustyfeet.com"
 entry_points       = {
-    'console_scripts': [
+    "console_scripts": [
-        'uncompyle6=uncompyle6.bin.uncompile:main_bin',
+        "uncompyle6=uncompyle6.bin.uncompile:main_bin",
-        'pydisassemble=uncompyle6.bin.pydisassemble:main',
+        "pydisassemble=uncompyle6.bin.pydisassemble:main",
     ]}
 ftp_url            = None
-install_requires   = ['spark-parser >= 1.8.7, < 1.9.0',
+install_requires   = ["spark-parser >= 1.8.9, < 1.9.0",
-                      'xdis >= 4.0.1, < 4.1.0']
+                      "xdis >= 4.2.2, < 4.3.0"]
 
-license            = 'GPL3'
+license            = "GPL3"
-mailing_list       = 'python-debugger@googlegroups.com'
+mailing_list       = "python-debugger@googlegroups.com"
-modname            = 'uncompyle6'
+modname            = "uncompyle6"
 py_modules         = None
-short_desc         = 'Python cross-version byte-code decompiler'
+short_desc         = "Python cross-version byte-code decompiler"
-web                = 'https://github.com/rocky/python-uncompyle6/'
+web                = "https://github.com/rocky/python-uncompyle6/"
 
 # tracebacks in zip files are funky and not debuggable
 zip_safe = True
@@ -82,5 +82,5 @@ def read(*rnames):
     return open(os.path.join(srcdir, *rnames)).read()
 
 # Get info from files; set: long_description and VERSION
-long_description   = ( read("README.rst") + '\n' )
+long_description   = ( read("README.rst") + "\n" )
-exec(read('uncompyle6/version.py'))
+exec(read("uncompyle6/version.py"))

admin-tools/README.md

@@ -8,4 +8,15 @@ They are customized to my environment:
 - I have git repos for xdis, and spark parser at the same level as uncompyle6
 
 There may be other rocky-specific things that need customization.
-how-to-make-a-release.txt has overall how I make a release
+how-to-make-a-release.md has overall how I make a release
+
+Since this project uses python over a wide variety of release, some versions
+of projects that should be used for specific Python versions
+
+for 3.2.6:
+   pytest==2.9.2
+
+for 3.1.5
+   pytset==2.1.0
+   py=1.8.0 and comment out line 10 of _builtin.py # callable = callable
+   six==1.10.0

admin-tools/check-newer-versions.sh

@@ -21,8 +21,9 @@ for version in $PYVERSIONS; do
 	exit $?
     fi
     make clean && pip install -e .
-    if ! make check; then
+    if ! make check-short; then
 	exit $?
     fi
     echo === $version ===
 done
+make check

admin-tools/how-to-make-a-release.md

@@ -56,19 +56,21 @@
 
     $ . ./admin-tools/make-dist-older.sh
     $ git tag release-python-2.4-$VERSION
-
+	$ twine check dist/uncompyle6-$VERSION*
     $ . ./admin-tools/make-dist-newer.sh
-
+	$ twine check dist/uncompyle6-$VERSION*
-Goto https://github.com/rocky/python-uncompyle6/releases
 
 # Upload single package and look at Rst Formating
 
+	$ twine check dist/uncompyle6-${VERSION}*
     $ twine upload dist/uncompyle6-${VERSION}-py3.3.egg
 
 # Upload rest of versions
 
     $ twine upload dist/uncompyle6-${VERSION}*
 
+Goto https://github.com/rocky/python-uncompyle6/releases
+
 # Push tags:
 
     $ git push --tags

admin-tools/pyenv-newer-versions

@@ -5,4 +5,4 @@ if [[ $0 == ${BASH_SOURCE[0]} ]] ; then
     echo "This script should be *sourced* rather than run directly through bash"
     exit 1
 fi
-export PYVERSIONS='3.6.8 3.7.3 2.6.9 3.3.7 2.7.16 3.2.6 3.1.5 3.4.8'
+export PYVERSIONS='3.5.9 3.6.10 2.6.9 3.3.7 2.7.17 3.2.6 3.1.5 3.4.10 3.7.6 3.8.1'

admin-tools/setup-master.sh

@@ -1,5 +1,5 @@
 #!/bin/bash
-PYTHON_VERSION=3.6.8
+PYTHON_VERSION=3.7.6
 
 # FIXME put some of the below in a common routine
 function finish {
@@ -20,3 +20,4 @@ cd $fulldir/..
     (cd ../python-xdis && git checkout master && pyenv local $PYTHON_VERSION) && git pull && \
     git checkout master && pyenv local $PYTHON_VERSION && git pull
 cd $owd
+rm -v */.python-version || true

admin-tools/setup-python-2.4.sh

@@ -14,3 +14,4 @@ cd $fulldir/..
     (cd ../python-xdis && git checkout python-2.4 && pyenv local $PYTHON_VERSION) && git pull && \
     git checkout python-2.4 &&  pyenv local $PYTHON_VERSION && git pull
 cd $owd
+rm -v */.python-version || true

appveyor.yml

@@ -61,7 +61,7 @@ build_script:
 
 test_script:
   # Run the project tests
-  - "%CMD_IN_ENV% python test/test_pyenvlib.py --native --weak-verify"
+  - "%CMD_IN_ENV% python test/test_pyenvlib.py --native --syntax-verify"
 
 after_test:
   # If tests are successful, create binary packages for the project.

pytest/Makefile

@@ -1,7 +1,11 @@
 PHONY=check test pytest
+SHELL=/bin/bash
 
 PYTHON ?= python
 
 #: Run all tests
 test check pytest:
-	py.test
+	@PYTHON_VERSION=`$(PYTHON) -V 2>&1 | cut -d ' ' -f 2 | cut -d'.' -f1,2`; \
+	 if [[ $$PYTHON_VERSION > 3.2 ]] || [[ $$PYTHON_VERSION == 2.7 ]] || [[ $$PYTHON_VERSION == 2.6 ]]; then \
+		py.test; \
+	fi

pytest/test_docstring.py

@@ -1,78 +0,0 @@
-import sys
-from uncompyle6 import PYTHON3
-if PYTHON3:
-    from io import StringIO
-    minint = -sys.maxsize-1
-    maxint = sys.maxsize
-else:
-    from StringIO import StringIO
-    minint = -sys.maxint-1
-    maxint = sys.maxint
-from uncompyle6.semantics.helper import print_docstring
-
-class PrintFake():
-    def __init__(self):
-        self.pending_newlines = 0
-        self.f = StringIO()
-
-    def write(self, *data):
-        if (len(data) == 0) or (len(data) == 1 and data[0] == ''):
-            return
-        out = ''.join((str(j) for j in data))
-        n = 0
-        for i in out:
-            if i == '\n':
-                n += 1
-                if n == len(out):
-                    self.pending_newlines = max(self.pending_newlines, n)
-                    return
-            elif n:
-                self.pending_newlines = max(self.pending_newlines, n)
-                out = out[n:]
-                break
-            else:
-                break
-
-        if self.pending_newlines > 0:
-            self.f.write('\n'*self.pending_newlines)
-            self.pending_newlines = 0
-
-        for i in out[::-1]:
-            if i == '\n':
-                self.pending_newlines += 1
-            else:
-                break
-
-        if self.pending_newlines:
-            out = out[:-self.pending_newlines]
-        self.f.write(out)
-    def println(self, *data):
-        if data and not(len(data) == 1 and data[0] == ''):
-            self.write(*data)
-        self.pending_newlines = max(self.pending_newlines, 1)
-        return
-    pass
-
-def test_docstring():
-
-    for doc, expect in (
-            ("Now is the time",
-             '  """Now is the time"""'),
-           ("""
-Now is the time
-""",
-            '''  """
-  Now is the time
-  """''')
-
-            # (r'''func placeholder - ' and with ("""\nstring\n  """)''',
-            #  """  r'''func placeholder - ' and with (\"\"\"\nstring\n\"\"\")'''"""),
-            # (r"""func placeholder - ' and with ('''\nstring\n''') and \"\"\"\nstring\n\"\"\" """,
-            # """  r\"\"\"func placeholder - ' and with ('''\nstring\n''') and \"\"\"\nstring\n\"\"\" \"\"\"""")
-            ):
-
-        o = PrintFake()
-        # print(doc)
-        # print(expect)
-        print_docstring(o, '  ', doc)
-        assert expect == o.f.getvalue()

pytest/test_fstring.py

@@ -1,158 +0,0 @@
-# std
-# test
-from uncompyle6 import PYTHON_VERSION, deparse_code
-import pytest
-pytestmark = pytest.mark.skipif(PYTHON_VERSION <= 2.6,
-                                reason='hypothesis needs 2.7 or later')
-if PYTHON_VERSION > 2.6:
-
-    import hypothesis
-    from hypothesis import strategies as st
-
-    # uncompyle6
-
-
-    @st.composite
-    def expressions(draw):
-        # todo : would be nice to generate expressions using hypothesis however
-        # this is pretty involved so for now just use a corpus of expressions
-        # from which to select.
-        return draw(st.sampled_from((
-            'abc',
-            'len(items)',
-            'x + 1',
-            'lineno',
-            'container',
-            'self.attribute',
-            'self.method()',
-            # These expressions are failing, I think these are control
-            # flow problems rather than problems with FORMAT_VALUE,
-            # however I need to confirm this...
-            #'sorted(items, key=lambda x: x.name)',
-            #'func(*args, **kwargs)',
-            #'text or default',
-            #'43 if life_the_universe and everything else None'
-        )))
-
-
-    @st.composite
-    def format_specifiers(draw):
-        """
-        Generate a valid format specifier using the rules:
-
-        format_spec ::=  [[fill]align][sign][#][0][width][,][.precision][type]
-        fill        ::=  <any character>
-        align       ::=  "<" | ">" | "=" | "^"
-        sign        ::=  "+" | "-" | " "
-        width       ::=  integer
-        precision   ::=  integer
-        type        ::=  "b" | "c" | "d" | "e" | "E" | "f" | "F" | "g" | "G" | "n" | "o" | "s" | "x" | "X" | "%"
-
-        See https://docs.python.org/2/library/string.html
-
-        :param draw: Let hypothesis draw from other strategies.
-
-        :return: An example format_specifier.
-        """
-        alphabet_strategy = st.characters(min_codepoint=ord('a'), max_codepoint=ord('z'))
-        fill = draw(st.one_of(alphabet_strategy, st.none()))
-        align = draw(st.sampled_from(list('<>=^')))
-        fill_align = (fill + align or '') if fill else ''
-
-        type_ = draw(st.sampled_from('bcdeEfFgGnosxX%'))
-        can_have_sign = type_ in 'deEfFgGnoxX%'
-        can_have_comma = type_ in 'deEfFgG%'
-        can_have_precision = type_ in 'fFgG'
-        can_have_pound = type_ in 'boxX%'
-        can_have_zero = type_ in 'oxX'
-
-        sign = draw(st.sampled_from(list('+- ') + [''])) if can_have_sign else ''
-        pound = draw(st.sampled_from(('#', '',))) if can_have_pound else ''
-        zero = draw(st.sampled_from(('0', '',))) if can_have_zero else ''
-
-        int_strategy = st.integers(min_value=1, max_value=1000)
-
-        width = draw(st.one_of(int_strategy, st.none()))
-        width = str(width) if width is not None else ''
-
-        comma = draw(st.sampled_from((',', '',))) if can_have_comma else ''
-        if can_have_precision:
-            precision = draw(st.one_of(int_strategy, st.none()))
-            precision = '.' + str(precision) if precision else ''
-        else:
-            precision = ''
-
-        return ''.join((fill_align, sign, pound, zero, width, comma, precision, type_,))
-
-
-    @st.composite
-    def fstrings(draw):
-        """
-        Generate a valid f-string.
-        See https://www.python.org/dev/peps/pep-0498/#specification
-
-        :param draw: Let hypothsis draw from other strategies.
-
-        :return: A valid f-string.
-        """
-        character_strategy = st.characters(
-            blacklist_characters='\r\n\'\\s{}',
-            min_codepoint=1,
-            max_codepoint=1000,
-        )
-        is_raw = draw(st.booleans())
-        integer_strategy = st.integers(min_value=0, max_value=3)
-        expression_count = draw(integer_strategy)
-        content = []
-        for _ in range(expression_count):
-            expression = draw(expressions())
-            conversion = draw(st.sampled_from(('', '!s', '!r', '!a',)))
-            has_specifier = draw(st.booleans())
-            specifier = ':' + draw(format_specifiers()) if has_specifier else ''
-            content.append('{{{}{}}}'.format(expression, conversion, specifier))
-            content.append(draw(st.text(character_strategy)))
-        content = ''.join(content)
-        return "f{}'{}'".format('r' if is_raw else '', content)
-
-
-    @pytest.mark.skipif(PYTHON_VERSION != 3.6, reason='need Python 3.6')
-    @hypothesis.given(format_specifiers())
-    def test_format_specifiers(format_specifier):
-        """Verify that format_specifiers generates valid specifiers"""
-        try:
-            exec('"{:' + format_specifier + '}".format(0)')
-        except ValueError as e:
-            if 'Unknown format code' not in str(e):
-                raise
-
-
-    def run_test(text):
-        hypothesis.assume(len(text))
-        hypothesis.assume("f'{" in text)
-        expr = text + '\n'
-        code = compile(expr, '<string>', 'single')
-        deparsed = deparse_code(PYTHON_VERSION, code, compile_mode='single')
-        recompiled = compile(deparsed.text, '<string>', 'single')
-        if recompiled != code:
-            print(recompiled)
-            print('================')
-            print(code)
-            print('----------------')
-            assert 'dis(' + deparsed.text.strip('\n') + ')' == 'dis(' + expr.strip('\n') + ')'
-
-
-    @pytest.mark.skipif(PYTHON_VERSION != 3.6, reason='need Python 3.6')
-    @hypothesis.given(fstrings())
-    def test_uncompyle_fstring(fstring):
-        """Verify uncompyling fstring bytecode"""
-        run_test(fstring)
-
-
-    @pytest.mark.skipif(PYTHON_VERSION != 3.6, reason='need Python 3.6+')
-    @pytest.mark.parametrize('fstring', [
-        "f'{abc}{abc!s}'",
-        "f'{abc}0'",
-    ])
-    def test_uncompyle_direct(fstring):
-        """useful for debugging"""
-        run_test(fstring)

pytest/test_function_call.py

@@ -1,185 +0,0 @@
-import string
-from uncompyle6 import PYTHON_VERSION
-import pytest
-pytestmark = pytest.mark.skip(PYTHON_VERSION < 2.7,
-                              reason="need at least Python 2.7")
-
-if PYTHON_VERSION > 2.6:
-    from hypothesis import given, assume, example, settings, strategies as st
-    from validate import validate_uncompyle
-    from test_fstring import expressions
-
-    alpha = st.sampled_from(string.ascii_lowercase)
-    numbers = st.sampled_from(string.digits)
-    alphanum = st.sampled_from(string.ascii_lowercase + string.digits)
-
-
-    @st.composite
-    def function_calls(draw,
-                       min_keyword_args=0, max_keyword_args=5,
-                       min_positional_args=0, max_positional_args=5,
-                       min_star_args=0, max_star_args=1,
-                       min_double_star_args=0, max_double_star_args=1):
-        """
-        Strategy factory for generating function calls.
-
-        :param draw: Callable which draws examples from other strategies.
-
-        :return: The function call text.
-        """
-        st_positional_args = st.lists(
-            alpha,
-            min_size=min_positional_args,
-            max_size=max_positional_args
-        )
-        st_keyword_args = st.lists(
-            alpha,
-            min_size=min_keyword_args,
-            max_size=max_keyword_args
-        )
-        st_star_args = st.lists(
-            alpha,
-            min_size=min_star_args,
-            max_size=max_star_args
-        )
-        st_double_star_args = st.lists(
-            alpha,
-            min_size=min_double_star_args,
-            max_size=max_double_star_args
-        )
-
-        positional_args = draw(st_positional_args)
-        keyword_args = draw(st_keyword_args)
-        st_values = st.lists(
-            expressions(),
-            min_size=len(keyword_args),
-            max_size=len(keyword_args)
-        )
-        keyword_args = [
-            x + '=' + e
-            for x, e in
-            zip(keyword_args, draw(st_values))
-        ]
-        star_args = ['*' + x for x in draw(st_star_args)]
-        double_star_args = ['**' + x for x in draw(st_double_star_args)]
-
-        arguments = positional_args + keyword_args + star_args + double_star_args
-        draw(st.randoms()).shuffle(arguments)
-        arguments = ','.join(arguments)
-
-        function_call = 'fn({arguments})'.format(arguments=arguments)
-        try:
-            # TODO: Figure out the exact rules for ordering of positional, keyword,
-            # star args, double star args and in which versions the various
-            # types of arguments are supported so we don't need to check that the
-            # expression compiles like this.
-            compile(function_call, '<string>', 'single')
-        except:
-            assume(False)
-        return function_call
-
-
-    def test_function_no_args():
-        validate_uncompyle("fn()")
-
-    @pytest.mark.skipif(PYTHON_VERSION < 2.7,
-                        reason="need at least Python 2.7")
-    def isolated_function_calls(which):
-        """
-        Returns a strategy for generating function calls, but isolated to
-        particular types of arguments, for example only positional arguments.
-
-        This can help reason about debugging errors in specific types of function
-        calls.
-
-        :param which: One of 'keyword', 'positional', 'star', 'double_star'
-
-        :return: Strategy for generating an function call isolated to specific
-                 argument types.
-        """
-        kwargs = dict(
-            max_keyword_args=0,
-            max_positional_args=0,
-            max_star_args=0,
-            max_double_star_args=0,
-        )
-        kwargs['_'.join(('min', which, 'args'))] = 1
-        kwargs['_'.join(('max', which, 'args'))] = 5 if 'star' not in which else 1
-        return function_calls(**kwargs)
-
-
-    with settings(max_examples=25):
-
-        @pytest.mark.skipif(PYTHON_VERSION < 2.7,
-                            reason="need at least Python 2.7")
-        @given(isolated_function_calls('positional'))
-        @example("fn(0)")
-        def test_function_positional_only(expr):
-            validate_uncompyle(expr)
-
-        @pytest.mark.skipif(PYTHON_VERSION < 2.7,
-                            reason="need at least Python 2.7")
-        @given(isolated_function_calls('keyword'))
-        @example("fn(a=0)")
-        def test_function_call_keyword_only(expr):
-            validate_uncompyle(expr)
-
-        @pytest.mark.skipif(PYTHON_VERSION < 2.7,
-                            reason="need at least Python 2.7")
-        @given(isolated_function_calls('star'))
-        @example("fn(*items)")
-        def test_function_call_star_only(expr):
-            validate_uncompyle(expr)
-
-        @pytest.mark.skipif(PYTHON_VERSION < 2.7,
-                            reason="need at least Python 2.7")
-        @given(isolated_function_calls('double_star'))
-        @example("fn(**{})")
-        def test_function_call_double_star_only(expr):
-            validate_uncompyle(expr)
-
-
-    @pytest.mark.xfail()
-    def test_BUILD_CONST_KEY_MAP_BUILD_MAP_UNPACK_WITH_CALL_BUILD_TUPLE_CALL_FUNCTION_EX():
-        validate_uncompyle("fn(w=0,m=0,**v)")
-
-
-    @pytest.mark.xfail()
-    def test_BUILD_MAP_BUILD_MAP_UNPACK_WITH_CALL_BUILD_TUPLE_CALL_FUNCTION_EX():
-        validate_uncompyle("fn(a=0,**g)")
-
-
-    @pytest.mark.xfail()
-    def test_CALL_FUNCTION_EX():
-        validate_uncompyle("fn(*g,**j)")
-
-
-    @pytest.mark.xfail()
-    def test_BUILD_MAP_CALL_FUNCTION_EX():
-        validate_uncompyle("fn(*z,u=0)")
-
-
-    @pytest.mark.xfail()
-    def test_BUILD_TUPLE_CALL_FUNCTION_EX():
-        validate_uncompyle("fn(**a)")
-
-
-    @pytest.mark.xfail()
-    def test_BUILD_MAP_BUILD_TUPLE_BUILD_TUPLE_UNPACK_WITH_CALL_CALL_FUNCTION_EX():
-        validate_uncompyle("fn(b,b,b=0,*a)")
-
-
-    @pytest.mark.xfail()
-    def test_BUILD_TUPLE_BUILD_TUPLE_UNPACK_WITH_CALL_CALL_FUNCTION_EX():
-        validate_uncompyle("fn(*c,v)")
-
-
-    @pytest.mark.xfail()
-    def test_BUILD_CONST_KEY_MAP_CALL_FUNCTION_EX():
-        validate_uncompyle("fn(i=0,y=0,*p)")
-
-
-    @pytest.mark.skip(reason='skipping property based test until all individual tests are passing')
-    @given(function_calls())
-    def test_function_call(function_call):
-        validate_uncompyle(function_call)

pytest/test_grammar.py

@@ -1,112 +1,146 @@
 import re
-from uncompyle6 import PYTHON_VERSION, PYTHON3, IS_PYPY # , PYTHON_VERSION
+from uncompyle6 import PYTHON_VERSION, PYTHON3, IS_PYPY  # , PYTHON_VERSION
 from uncompyle6.parser import get_python_parser, python_parser
 from uncompyle6.scanner import get_scanner
 
-def test_grammar():
 
+def test_grammar():
     def check_tokens(tokens, opcode_set):
         remain_tokens = set(tokens) - opcode_set
-        remain_tokens = set([re.sub(r'_\d+$','', t) for t in remain_tokens])
+        remain_tokens = set([re.sub(r"_\d+$", "", t) for t in remain_tokens])
-        remain_tokens = set([re.sub('_CONT$','', t) for t in remain_tokens])
+        remain_tokens = set([re.sub("_CONT$", "", t) for t in remain_tokens])
+        remain_tokens = set([re.sub("LOAD_CODE$", "", t) for t in remain_tokens])
         remain_tokens = set(remain_tokens) - opcode_set
-        assert remain_tokens == set([]), \
+        assert remain_tokens == set([]), "Remaining tokens %s\n====\n%s" % (
-            "Remaining tokens %s\n====\n%s" % (remain_tokens, p.dump_grammar())
+            remain_tokens,
+            p.dump_grammar(),
+        )
 
     p = get_python_parser(PYTHON_VERSION, is_pypy=IS_PYPY)
-    (lhs, rhs, tokens,
+    (lhs, rhs, tokens, right_recursive, dup_rhs) = p.check_sets()
-     right_recursive, dup_rhs) = p.check_sets()
 
     # We have custom rules that create the below
-    expect_lhs = set(['pos_arg', 'attribute'])
+    expect_lhs = set(["pos_arg"])
+
     if PYTHON_VERSION < 3.8:
-        expect_lhs.add('get_iter')
+        if PYTHON_VERSION < 3.7:
+            expect_lhs.add("attribute")
 
+        expect_lhs.add("get_iter")
 
-    unused_rhs = set(['list', 'mkfunc',
+        if PYTHON_VERSION > 3.7 or PYTHON_VERSION < 3.0:
-                      'mklambda',
+            expect_lhs.add("stmts_opt")
-                      'unpack',])
+    else:
+        expect_lhs.add("async_with_as_stmt")
+        expect_lhs.add("async_with_stmt")
 
-    expect_right_recursive = set([('designList',
+    unused_rhs = set(["list", "mkfunc", "mklambda", "unpack"])
-                                   ('store', 'DUP_TOP', 'designList'))])
 
-    if PYTHON_VERSION < 3.7:
+    expect_right_recursive = set([("designList", ("store", "DUP_TOP", "designList"))])
-        unused_rhs.add('call')
+
+    if PYTHON_VERSION <= 3.6:
+        unused_rhs.add("call")
 
     if PYTHON_VERSION > 2.6:
-        expect_lhs.add('kvlist')
+        expect_lhs.add("kvlist")
-        expect_lhs.add('kv3')
+        expect_lhs.add("kv3")
-        unused_rhs.add('dict')
+        unused_rhs.add("dict")
+
+        if PYTHON_VERSION < 3.7 and PYTHON_VERSION != 2.7:
+            # NOTE: this may disappear
+            expect_lhs.add("except_handler_else")
 
     if PYTHON3:
-        expect_lhs.add('load_genexpr')
+        expect_lhs.add("load_genexpr")
 
-        unused_rhs = unused_rhs.union(set("""
+        unused_rhs = unused_rhs.union(
+            set(
+                """
         except_pop_except generator_exp
-        """.split()))
+        """.split()
+            )
+        )
         if PYTHON_VERSION >= 3.0:
-            expect_lhs.add("annotate_arg")
+            if PYTHON_VERSION < 3.7:
-            expect_lhs.add("annotate_tuple")
+                expect_lhs.add("annotate_arg")
-            unused_rhs.add("mkfunc_annotate")
+                expect_lhs.add("annotate_tuple")
+                unused_rhs.add("mkfunc_annotate")
+
             unused_rhs.add("dict_comp")
             unused_rhs.add("classdefdeco1")
             unused_rhs.add("tryelsestmtl")
             if PYTHON_VERSION >= 3.5:
-                expect_right_recursive.add((('l_stmts',
+                expect_right_recursive.add(
-                                             ('lastl_stmt', 'come_froms', 'l_stmts'))))
+                    (("l_stmts", ("lastl_stmt", "come_froms", "l_stmts")))
-                pass
+                )
-            elif 3.0 < PYTHON_VERSION < 3.3:
-                expect_right_recursive.add((('l_stmts',
-                                             ('lastl_stmt', 'COME_FROM', 'l_stmts'))))
                 pass
             pass
         pass
     else:
-        expect_lhs.add('kwarg')
+        expect_lhs.add("kwarg")
-
-    assert expect_lhs == set(lhs)
 
     # FIXME
-    if PYTHON_VERSION != 3.8:
+    if PYTHON_VERSION < 3.8:
+        assert expect_lhs == set(lhs)
         assert unused_rhs == set(rhs)
 
     assert expect_right_recursive == right_recursive
 
-    expect_dup_rhs = frozenset([('COME_FROM',), ('CONTINUE',), ('JUMP_ABSOLUTE',),
+    expect_dup_rhs = frozenset(
-                                ('LOAD_CONST',),
+        [
-                                ('JUMP_BACK',), ('JUMP_FORWARD',)])
+            ("COME_FROM",),
+            ("CONTINUE",),
+            ("JUMP_ABSOLUTE",),
+            ("LOAD_CONST",),
+            ("JUMP_BACK",),
+            ("JUMP_FORWARD",),
+        ]
+    )
     reduced_dup_rhs = dict((k, dup_rhs[k]) for k in dup_rhs if k not in expect_dup_rhs)
-    for k in reduced_dup_rhs:
+    if reduced_dup_rhs:
-        print(k, reduced_dup_rhs[k])
+        print("\nPossible duplicate RHS that might be folded, into one of the LHS symbols")
+        for k in reduced_dup_rhs:
+            print(k, reduced_dup_rhs[k])
     # assert not reduced_dup_rhs, reduced_dup_rhs
 
     s = get_scanner(PYTHON_VERSION, IS_PYPY)
     ignore_set = set(
-            """
+        """
             JUMP_BACK CONTINUE
             COME_FROM COME_FROM_EXCEPT
             COME_FROM_EXCEPT_CLAUSE
             COME_FROM_LOOP COME_FROM_WITH
             COME_FROM_FINALLY ELSE
-            LOAD_GENEXPR LOAD_ASSERT LOAD_SETCOMP LOAD_DICTCOMP
+            LOAD_GENEXPR LOAD_ASSERT LOAD_SETCOMP LOAD_DICTCOMP LOAD_STR LOAD_CODE
             LAMBDA_MARKER
             RETURN_END_IF RETURN_END_IF_LAMBDA RETURN_VALUE_LAMBDA RETURN_LAST
-            """.split())
+            """.split()
+    )
     if 2.6 <= PYTHON_VERSION <= 2.7:
         opcode_set = set(s.opc.opname).union(ignore_set)
         if PYTHON_VERSION == 2.6:
             opcode_set.add("THEN")
         check_tokens(tokens, opcode_set)
     elif PYTHON_VERSION == 3.4:
-        ignore_set.add('LOAD_CLASSNAME')
+        ignore_set.add("LOAD_CLASSNAME")
-        ignore_set.add('STORE_LOCALS')
+        ignore_set.add("STORE_LOCALS")
         opcode_set = set(s.opc.opname).union(ignore_set)
         check_tokens(tokens, opcode_set)
 
+
 def test_dup_rule():
     import inspect
-    python_parser(PYTHON_VERSION, inspect.currentframe().f_code,
+
-                  is_pypy=IS_PYPY,
+    python_parser(
-                  parser_debug={
+        PYTHON_VERSION,
-                      'dups': True, 'transition': False, 'reduce': False,
+        inspect.currentframe().f_code,
-                      'rules': False, 'errorstack': None, 'context': True})
+        is_pypy=IS_PYPY,
+        parser_debug={
+            "dups": True,
+            "transition": False,
+            "reduce": False,
+            "rules": False,
+            "errorstack": None,
+            "context": True,
+        },
+    )

pytest/test_pysource.py-notyet

@@ -15,7 +15,7 @@ else:
     def iteritems(d):
         return d.iteritems()
 
-from uncompyle6.semantics.pysource import SourceWalker as SourceWalker
+from uncompyle6.semantics.pysource import (SourceWalker, deparse_code2str)
 
 def test_template_engine():
     s = StringIO()
@@ -185,3 +185,11 @@ def test_tables():
             assert arg == len(entry), (
                 "%s[%s] arg %d should be length of entry %d. Full entry: %s" %
                             (name, k, arg, len(entry), entry))
+
+def test_deparse_code2str():
+    def deparse_test(co):
+        "This is a docstring"
+        s = deparse_code2str(co, debug_opts={"asm": "after", "tree": True})
+        assert s
+        return
+    deparse_test(deparse_test.__code__)

pytest/test_token.py

@@ -1,23 +1,25 @@
+from uncompyle6 import PYTHON_VERSION
 from uncompyle6.scanners.tok import Token
 
+
 def test_token():
     # Test token formatting of: LOAD_CONST None
-    t = Token('LOAD_CONST', offset=0, attr=None, pattr=None, has_arg=True)
+    t = Token("LOAD_CONST", offset=0, attr=None, pattr=None, has_arg=True)
-    expect = '           0  LOAD_CONST               None'
+    expect = "             0  LOAD_CONST               None"
     # print(t.format())
     assert t
     assert t.format() == expect
 
     # Make sure equality testing of tokens ignores offset
-    t2 = Token('LOAD_CONST', offset=2, attr=None, pattr=None, has_arg=True)
+    t2 = Token("LOAD_CONST", offset=2, attr=None, pattr=None, has_arg=True)
     assert t2 == t
 
-
     # Make sure formatting of: LOAD_CONST False. We assume False is the 0th index
     # of co_consts.
-    t = Token('LOAD_CONST', offset=1, attr=False, pattr=False, has_arg=True)
+    t = Token("LOAD_CONST", offset=1, attr=False, pattr=False, has_arg=True)
-    expect = '           1  LOAD_CONST            0  False'
+    expect = "             1  LOAD_CONST               False"
     assert t.format() == expect
 
-if __name__ == '__main__':
+
+if __name__ == "__main__":
     test_token()

pytest/testdata/if-2.7.right

@@ -1,12 +1,12 @@
 # Python 2.7
 # Embedded file name: simple_source/branching/05_if.py
 
-   6       0  LOAD_NAME             0  'True'
+   6         0  LOAD_NAME             0  'True'
-           3  POP_JUMP_IF_FALSE    15  'to 15'
+             3  POP_JUMP_IF_FALSE    15  'to 15'
 
-   7       6  LOAD_NAME             1  'False'
+   7         6  LOAD_NAME             1  'False'
-           9  STORE_NAME            2  'b'
+             9  STORE_NAME            2  'b'
-          12  JUMP_FORWARD          0  'to 15'
+            12  JUMP_FORWARD          0  'to 15'
-        15_0  COME_FROM            12  '12'
+          15_0  COME_FROM            12  '12'
-          15  LOAD_CONST            0  None
+            15  LOAD_CONST               None
-          18  RETURN_VALUE     
+            18  RETURN_VALUE     

pytest/testdata/ifelse-2.7.right

@@ -1,15 +1,15 @@
 # Python 2.7
 # Embedded file name: simple_source/branching/05_ifelse.py
 
-   3       0  LOAD_NAME             0  'True'
+   3         0  LOAD_NAME             0  'True'
-           3  POP_JUMP_IF_FALSE    15  'to 15'
+             3  POP_JUMP_IF_FALSE    15  'to 15'
 
-   4       6  LOAD_CONST            0  1
+   4         6  LOAD_CONST               1
-           9  STORE_NAME            1  'b'
+             9  STORE_NAME            1  'b'
-          12  JUMP_FORWARD          6  'to 21'
+            12  JUMP_FORWARD          6  'to 21'
 
-   6      15  LOAD_CONST            1  2
+   6        15  LOAD_CONST               2
-          18  STORE_NAME            2  'd'
+            18  STORE_NAME            2  'd'
-        21_0  COME_FROM            12  '12'
+          21_0  COME_FROM            12  '12'
-          21  LOAD_CONST            2  None
+            21  LOAD_CONST               None
-          24  RETURN_VALUE     
+            24  RETURN_VALUE     

pytest/validate.py

@@ -1,16 +1,20 @@
 # future
 from __future__ import print_function
+
 # std
 import os
 import difflib
 import subprocess
 import tempfile
 import functools
+
 # uncompyle6 / xdis
-from uncompyle6 import PYTHON_VERSION, PYTHON3, IS_PYPY, deparse_code
+from uncompyle6 import PYTHON_VERSION, PYTHON3, IS_PYPY, code_deparse
+
 # TODO : I think we can get xdis to support the dis api (python 3 version) by doing something like this there
 from xdis.bytecode import Bytecode
 from xdis.main import get_opcode
+
 opc = get_opcode(PYTHON_VERSION, IS_PYPY)
 Bytecode = functools.partial(Bytecode, opc=opc)
 import six
@@ -20,6 +24,7 @@ if PYTHON3:
 else:
     from StringIO import StringIO
 
+
 def _dis_to_text(co):
     return Bytecode(co).dis()
 
@@ -33,36 +38,32 @@ def print_diff(original, uncompyled):
     :param original: Text describing the original code object.
     :param uncompyled: Text describing the uncompyled code object.
     """
-    original_lines = original.split('\n')
+    original_lines = original.split("\n")
-    uncompyled_lines = uncompyled.split('\n')
+    uncompyled_lines = uncompyled.split("\n")
-    args = original_lines, uncompyled_lines, 'original', 'uncompyled'
+    args = original_lines, uncompyled_lines, "original", "uncompyled"
     try:
         from bs4 import BeautifulSoup
+
         diff = difflib.HtmlDiff().make_file(*args)
         diff = BeautifulSoup(diff, "html.parser")
         diff.select_one('table[summary="Legends"]').extract()
     except ImportError:
-        print('\nTo display diff highlighting run:\n    pip install BeautifulSoup4')
+        print("\nTo display diff highlighting run:\n    pip install BeautifulSoup4")
         diff = difflib.HtmlDiff().make_table(*args)
 
     with tempfile.NamedTemporaryFile(delete=False) as f:
-        f.write(str(diff).encode('utf-8'))
+        f.write(str(diff).encode("utf-8"))
 
     try:
         print()
-        html = subprocess.check_output([
+        html = subprocess.check_output(
-            'elinks',
+            ["elinks", "-dump", "-no-references", "-dump-color-mode", "1", f.name]
-            '-dump',
+        ).decode("utf-8")
-            '-no-references',
-            '-dump-color-mode',
-            '1',
-            f.name,
-        ]).decode('utf-8')
         print(html)
     except:
-        print('\nFor side by side diff install elinks')
+        print("\nFor side by side diff install elinks")
         diff = difflib.Differ().compare(original_lines, uncompyled_lines)
-        print('\n'.join(diff))
+        print("\n".join(diff))
     finally:
         os.unlink(f.name)
 
@@ -80,18 +81,19 @@ def are_instructions_equal(i1, i2):
 
     :return: True if the two instructions are approximately equal, otherwise False.
     """
-    result = (1 == 1
+    result = (
+        1 == 1
         and i1.opname == i2.opname
         and i1.opcode == i2.opcode
         and i1.arg == i2.arg
         # ignore differences due to code objects
         # TODO : Better way of ignoring address
-        and (i1.argval == i2.argval or '<code object' in str(i1.argval))
+        and (i1.argval == i2.argval or "<code object" in str(i1.argval))
         # TODO : Should probably recurse to check code objects
-        and (i1.argrepr == i2.argrepr or '<code object' in i1.argrepr)
+        and (i1.argrepr == i2.argrepr or "<code object" in i1.argrepr)
         and i1.offset == i2.offset
         # ignore differences in line numbers
-        #and i1.starts_line
+        # and i1.starts_line
         and i1.is_jump_target == i2.is_jump_target
     )
     return result
@@ -115,22 +117,21 @@ def are_code_objects_equal(co1, co2):
     return True
 
 
-def validate_uncompyle(text, mode='exec'):
+def validate_uncompyle(text, mode="exec"):
     """
     Validate decompilation of the given source code.
 
     :param text: Source to validate decompilation of.
     """
-    original_code = compile(text, '<string>', mode)
+    original_code = compile(text, "<string>", mode)
     original_dis = _dis_to_text(original_code)
     original_text = text
 
-    deparsed = deparse_code(PYTHON_VERSION, original_code,
+    deparsed = code_deparse(
-                            compile_mode=mode,
+        original_code, out=six.StringIO(), version=PYTHON_VERSION, compile_mode=mode
-                            out=six.StringIO(),
+    )
-                            is_pypy=IS_PYPY)
     uncompyled_text = deparsed.text
-    uncompyled_code = compile(uncompyled_text, '<string>', 'exec')
+    uncompyled_code = compile(uncompyled_text, "<string>", "exec")
 
     if not are_code_objects_equal(uncompyled_code, original_code):
 
@@ -138,15 +139,17 @@ def validate_uncompyle(text, mode='exec'):
 
         def output(text, dis):
             width = 60
-            return '\n\n'.join([
+            return "\n\n".join(
-                ' SOURCE CODE '.center(width, '#'),
+                [
-                text.strip(),
+                    " SOURCE CODE ".center(width, "#"),
-                ' BYTECODE '.center(width, '#'),
+                    text.strip(),
-                dis
+                    " BYTECODE ".center(width, "#"),
-            ])
+                    dis,
+                ]
+            )
 
         original = output(original_text, original_dis)
         uncompyled = output(uncompyled_text, uncompyled_dis)
         print_diff(original, uncompyled)
 
-        assert 'original' == 'uncompyled'
+        assert "original" == "uncompyled"

requirements-dev.txt

@@ -1,2 +1,3 @@
 flake8
-hypothesis<=3.0.0
+six
+pytest==3.2.5 # for 2.7 < PYTHON_VERSION <= 3.2 use pytest 2.9.2; for 3.1 2.10

setup.cfg

@@ -1,11 +1,11 @@
 [bdist_rpm]
 release = 1
-packager = Mysterie <kajusska@gmail.com>
+packager = rocky <rb@dustyfeet.com>
 doc_files = README
-#		CHANGES.txt
-#		USAGE.txt
-#		doc/
-#		examples/
 
 [bdist_wheel]
-universal=1
+
+[egg_info]
+tag_build = 
+tag_date = 0
+

setup.py

@@ -4,8 +4,8 @@ import sys
 """Setup script for the 'uncompyle6' distribution."""
 
 SYS_VERSION = sys.version_info[0:2]
-if not ((2, 6) <= SYS_VERSION  <= (3, 8)):
+if not ((2, 6) <= SYS_VERSION  <= (3, 9)):
-    mess = "Python Release 2.6 .. 3.8 are supported in this code branch."
+    mess = "Python Release 2.6 .. 3.9 are supported in this code branch."
     if ((2, 4) <= SYS_VERSION <= (2, 7)):
         mess += ("\nFor your Python, version %s, use the python-2.4 code/branch." %
                  sys.version[0:3])
@@ -32,6 +32,7 @@ setup(
        install_requires   = install_requires,
        license            = license,
        long_description   = long_description,
+       long_description_content_type = "text/x-rst",
        name               = modname,
        packages           = find_packages(),
        py_modules         = py_modules,

test/.gitignore

@@ -1 +1,3 @@
+/.coverage
+/.python-version
 /nohup.out

test/Makefile

@@ -1,5 +1,6 @@
 PHONY=check clean dist distclean test test-unit test-functional rmChangeLog clean_pyc nosetests \
-      check-bytecode-1  check-bytecode-1.3 check-bytecode-1.4 check-bytecode-1.5 \
+      check-bytecode-1.0 check-bytecode-1.1 check-bytecode-1.2 check-bytecode-1.3 \
+      check-bytecode-1 check-bytecode-1.4 check-bytecode-1.5 check-bytecode-1.6 \
       check-bytecode-2 check-bytecode-3 check-bytecode-3-short \
       check-bytecode-2.2 check-byteocde-2.3 check-bytecode-2.4 \
       check-short check-2.6 check-2.7 check-3.0 check-3.1 check-3.2 check-3.3 \
@@ -22,7 +23,7 @@ COVER_DIR=../tmp/grammar-cover
 # Run short tests
 check-short:
 	@$(PYTHON) -V && PYTHON_VERSION=`$(PYTHON) -V 2>&1 | cut -d ' ' -f 2 | cut -d'.' -f1,2`; \
-	$(MAKE) check-bytecode-short
+	$(MAKE) check-bytecode-$${PYTHON_VERSION}
 
 # Run all tests
 check:
@@ -34,47 +35,47 @@ check-2.6 check-2.7: check-bytecode-2 check-bytecode-3 check-bytecode-1 check-na
 #: Run working tests from Python 3.0
 check-3.0: check-bytecode
 	$(PYTHON) test_pythonlib.py --bytecode-3.0-run --verify-run
-	$(PYTHON) test_pythonlib.py --bytecode-3.0 --weak-verify $(COMPILE)
+	$(PYTHON) test_pythonlib.py --bytecode-3.0 --syntax-verify $(COMPILE)
 
 #: Run working tests from Python 3.1
 check-3.1: check-bytecode
 	$(PYTHON) test_pythonlib.py --bytecode-3.1-run --verify-run
-	$(PYTHON) test_pythonlib.py --bytecode-3.1 --weak-verify $(COMPILE)
+	$(PYTHON) test_pythonlib.py --bytecode-3.1 --syntax-verify $(COMPILE)
 
 #: Run working tests from Python 3.2
 check-3.2: check-bytecode
 	$(PYTHON) test_pythonlib.py --bytecode-3.2-run --verify-run
-	$(PYTHON) test_pythonlib.py --bytecode-3.2 --weak-verify $(COMPILE)
+	$(PYTHON) test_pythonlib.py --bytecode-3.2 --syntax-verify $(COMPILE)
 
 #: Run working tests from Python 3.3
 check-3.3: check-bytecode
 	$(PYTHON) test_pythonlib.py --bytecode-3.3-run --verify-run
-	$(PYTHON) test_pythonlib.py --bytecode-3.3 --weak-verify $(COMPILE)
+	$(PYTHON) test_pythonlib.py --bytecode-3.3 --syntax-verify $(COMPILE)
 
 #: Run working tests from Python 3.4
 check-3.4: check-bytecode check-3.4-ok check-2.7-ok
 	$(PYTHON) test_pythonlib.py --bytecode-3.4-run --verify-run
-	$(PYTHON) test_pythonlib.py --bytecode-3.4 --weak-verify $(COMPILE)
+	$(PYTHON) test_pythonlib.py --bytecode-3.4 --syntax-verify $(COMPILE)
 
 #: Run working tests from Python 3.5
 check-3.5: check-bytecode
 	$(PYTHON) test_pythonlib.py --bytecode-3.5-run --verify-run
-	$(PYTHON) test_pythonlib.py --bytecode-3.5 --weak-verify $(COMPILE)
+	$(PYTHON) test_pythonlib.py --bytecode-3.5 --syntax-verify $(COMPILE)
 
 #: Run working tests from Python 3.6
 check-3.6: check-bytecode
 	$(PYTHON) test_pythonlib.py --bytecode-3.6-run --verify-run
-	$(PYTHON) test_pythonlib.py --bytecode-3.6 --weak-verify $(COMPILE)
+	$(PYTHON) test_pythonlib.py --bytecode-3.6 --syntax-verify $(COMPILE)
 
 #: Run working tests from Python 3.7
 check-3.7: check-bytecode
 	$(PYTHON) test_pythonlib.py --bytecode-3.7-run --verify-run
-	$(PYTHON) test_pythonlib.py --bytecode-3.7 --weak-verify $(COMPILE)
+	$(PYTHON) test_pythonlib.py --bytecode-3.7 --syntax-verify $(COMPILE)
 
 #: Run working tests from Python 3.8
 check-3.8: check-bytecode
 	$(PYTHON) test_pythonlib.py --bytecode-3.8-run --verify-run
-	$(PYTHON) test_pythonlib.py --bytecode-3.8 --weak-verify $(COMPILE)
+	$(PYTHON) test_pythonlib.py --bytecode-3.8 --syntax-verify $(COMPILE)
 
 # FIXME
 #: this is called when running under pypy3.5-5.8.0 or pypy2-5.6.0
@@ -85,7 +86,7 @@ check-disasm:
 	$(PYTHON) dis-compare.py
 
 #: Check deparsing bytecode 1.x only
-check-bytecode-1: check-bytecode-1.4 check-bytecode-1.5
+check-bytecode-1: check-bytecode-1.0 check-bytecode-1.1 check-bytecode-1.2 check-bytecode-1.3 check-bytecode-1.4 check-bytecode-1.5 check-bytecode-1.6
 
 #: Check deparsing bytecode 2.x only
 check-bytecode-2:
@@ -98,8 +99,8 @@ check-bytecode-3:
 	$(PYTHON) test_pythonlib.py --bytecode-3.0 \
 	--bytecode-3.1 --bytecode-3.2 --bytecode-3.3 \
 	--bytecode-3.4 --bytecode-3.5 --bytecode-3.6 \
-        --bytecode-3.7 --bytecode-3.8 \
+        --bytecode-3.7  \
-	--bytecode-pypy3.2
+	--bytecode-pypy3.2 --bytecode-pypy3.6 --bytecode-3.8
 
 #: Check deparsing on selected bytecode 3.x
 check-bytecode-3-short:
@@ -109,6 +110,7 @@ check-bytecode-3-short:
 #: Check deparsing bytecode on all Python 2 and Python 3 versions
 check-bytecode: check-bytecode-3
 	$(PYTHON) test_pythonlib.py \
+	--bytecode-1.0 --bytecode-1.1 --bytecode-1.2 \
 	--bytecode-1.3 --bytecode-1.4 --bytecode-1.5 \
         --bytecode-2.2 --bytecode-2.3 --bytecode-2.4 \
 	--bytecode-2.1 --bytecode-2.2 --bytecode-2.3 --bytecode-2.4 \
@@ -122,6 +124,18 @@ check-bytecode-short: check-bytecode-3-short
         --bytecode-2.6 --bytecode-2.7 --bytecode-pypy2.7
 
 
+#: Check deparsing bytecode 1.0 only
+check-bytecode-1.0:
+	$(PYTHON) test_pythonlib.py --bytecode-1.0
+
+#: Check deparsing bytecode 1.1 only
+check-bytecode-1.1:
+	$(PYTHON) test_pythonlib.py --bytecode-1.1
+
+#: Check deparsing bytecode 1.2 only
+check-bytecode-1.2:
+	$(PYTHON) test_pythonlib.py --bytecode-1.2
+
 #: Check deparsing bytecode 1.3 only
 check-bytecode-1.3:
 	$(PYTHON) test_pythonlib.py --bytecode-1.3
@@ -134,6 +148,10 @@ check-bytecode-1.4:
 check-bytecode-1.5:
 	$(PYTHON) test_pythonlib.py --bytecode-1.5
 
+#: Check deparsing bytecode 1.6 only
+check-bytecode-1.6:
+	$(PYTHON) test_pythonlib.py --bytecode-1.6
+
 #: Check deparsing Python 2.1
 check-bytecode-2.1:
 	$(PYTHON) test_pythonlib.py --bytecode-2.1
@@ -176,7 +194,7 @@ grammar-coverage-2.6:
 grammar-coverage-2.7:
 	-rm $(COVER_DIR)/spark-grammar-2.7.cover || true
 	SPARK_PARSER_COVERAGE=$(COVER_DIR)/spark-grammar-2.7.cover $(PYTHON) test_pythonlib.py --bytecode-2.7
-	SPARK_PARSER_COVERAGE=$(COVER_DIR)/spark-grammar-2.7.cover $(PYTHON) test_pyenvlib.py --2.7.14 --max=600
+	SPARK_PARSER_COVERAGE=$(COVER_DIR)/spark-grammar-2.7.cover $(PYTHON) test_pyenvlib.py --2.7.16 --max=600
 
 #: Get grammar coverage for Python 3.0
 grammar-coverage-3.0:
@@ -219,83 +237,88 @@ grammar-coverage-3.5:
 grammar-coverage-3.6:
 	rm $(COVER_DIR)/spark-grammar-3.6.cover || /bin/true
 	SPARK_PARSER_COVERAGE=$(COVER_DIR)/spark-grammar-3.6.cover $(PYTHON) test_pythonlib.py --bytecode-3.6
-	SPARK_PARSER_COVERAGE=$(COVER_DIR)/spark-grammar-3.6.cover $(PYTHON) test_pyenvlib.py --3.6.4 --max=280
+	SPARK_PARSER_COVERAGE=$(COVER_DIR)/spark-grammar-3.6.cover $(PYTHON) test_pyenvlib.py --3.6.8 --max=280
+
+#: Get grammar coverage for Python 3.7
+grammar-coverage-3.7:
+	rm $(COVER_DIR)/spark-grammar-3.7.cover || /bin/true
+	SPARK_PARSER_COVERAGE=$(COVER_DIR)/spark-grammar-3.7.cover $(PYTHON) test_pyenvlib.py --3.7.3 --max=500
 
 #: Check deparsing Python 2.6
 check-bytecode-2.6:
 	$(PYTHON) test_pythonlib.py --bytecode-2.6-run --verify-run
-	$(PYTHON) test_pythonlib.py --bytecode-2.6 --weak-verify
+	$(PYTHON) test_pythonlib.py --bytecode-2.6 --syntax-verify
 
 #: Check deparsing Python 2.7
 check-bytecode-2.7:
 	$(PYTHON) test_pythonlib.py --bytecode-2.7-run --verify-run
-	$(PYTHON) test_pythonlib.py --bytecode-2.7 --weak-verify
+	$(PYTHON) test_pythonlib.py --bytecode-2.7 --syntax-verify
 
 #: Check deparsing Python 3.0
 check-bytecode-3.0:
 	$(PYTHON) test_pythonlib.py --bytecode-3.0-run --verify-run
-	$(PYTHON) test_pythonlib.py --bytecode-3.0 --weak-verify
+	$(PYTHON) test_pythonlib.py --bytecode-3.0 --syntax-verify
 
 #: Check deparsing Python 3.1
 check-bytecode-3.1:
 	$(PYTHON) test_pythonlib.py --bytecode-3.1-run --verify-run
-	$(PYTHON) test_pythonlib.py --bytecode-3.1 --weak-verify
+	$(PYTHON) test_pythonlib.py --bytecode-3.1 --syntax-verify
 
 #: Check deparsing Python 3.2
 check-bytecode-3.2:
 	$(PYTHON) test_pythonlib.py --bytecode-3.2-run --verify-run
-	$(PYTHON) test_pythonlib.py --bytecode-3.2 --weak-verify
+	$(PYTHON) test_pythonlib.py --bytecode-3.2 --syntax-verify
 
 #: Check deparsing Python 3.3
 check-bytecode-3.3:
 	$(PYTHON) test_pythonlib.py --bytecode-3.3-run --verify-run
-	$(PYTHON) test_pythonlib.py --bytecode-3.3 --weak-verify
+	$(PYTHON) test_pythonlib.py --bytecode-3.3 --syntax-verify
 
 #: Check deparsing Python 3.4
 check-bytecode-3.4:
 	$(PYTHON) test_pythonlib.py --bytecode-3.4-run --verify-run
-	$(PYTHON) test_pythonlib.py --bytecode-3.4 --weak-verify
+	$(PYTHON) test_pythonlib.py --bytecode-3.4 --syntax-verify
 
 #: Check deparsing Python 3.5
 check-bytecode-3.5:
 	$(PYTHON) test_pythonlib.py --bytecode-3.5-run --verify-run
-	$(PYTHON) test_pythonlib.py --bytecode-3.5 --weak-verify
+	$(PYTHON) test_pythonlib.py --bytecode-3.5 --syntax-verify
 
 #: Check deparsing Python 3.6
 check-bytecode-3.6:
 	$(PYTHON) test_pythonlib.py --bytecode-3.6-run --verify-run
-	$(PYTHON) test_pythonlib.py --bytecode-3.6 --weak-verify
+	$(PYTHON) test_pythonlib.py --bytecode-3.6 --syntax-verify
 
 #: Check deparsing Python 3.7
 check-bytecode-3.7:
 	$(PYTHON) test_pythonlib.py --bytecode-3.7-run --verify-run
-	$(PYTHON) test_pythonlib.py --bytecode-3.7 --weak-verify
+	$(PYTHON) test_pythonlib.py --bytecode-3.7 --syntax-verify
 
 #: Check deparsing Python 3.8
 check-bytecode-3.8:
 	$(PYTHON) test_pythonlib.py --bytecode-3.8-run --verify-run
-	$(PYTHON) test_pythonlib.py --bytecode-3.8 --weak-verify
+	$(PYTHON) test_pythonlib.py --bytecode-3.8 --syntax-verify
 
 #: short tests for bytecodes only for this version of Python
 check-native-short:
-	$(PYTHON) test_pythonlib.py --bytecode-$(PYTHON_VERSION) --weak-verify $(COMPILE)
+	$(PYTHON) test_pythonlib.py --bytecode-$(PYTHON_VERSION) --syntax-verify $(COMPILE)
 	$(PYTHON) test_pythonlib.py --bytecode-$(PYTHON_VERSION)-run --verify-run $(COMPILE)
 
 #: Run longer Python 2.6's lib files known to be okay
 check-2.6-ok:
-	$(PYTHON) test_pythonlib.py --ok-2.6 --weak-verify $(COMPILE)
+	$(PYTHON) test_pythonlib.py --ok-2.6 --syntax-verify $(COMPILE)
 
 #: Run longer Python 2.7's lib files known to be okay
 check-2.7-ok:
-	$(PYTHON) test_pythonlib.py --ok-2.7 --weak-verify $(COMPILE)
+	$(PYTHON) test_pythonlib.py --ok-2.7 --syntax-verify $(COMPILE)
 
 #: Run longer Python 3.2's lib files known to be okay
 check-3.2-ok:
-	$(PYTHON) test_pythonlib.py --ok-3.2 --weak-verify $(COMPILE)
+	$(PYTHON) test_pythonlib.py --ok-3.2 --syntax-verify $(COMPILE)
 
 #: Run longer Python 3.4's lib files known to be okay
 check-3.4-ok:
-	$(PYTHON) test_pythonlib.py --ok-3.4 --weak-verify $(COMPILE)
+	$(PYTHON) test_pythonlib.py --ok-3.4 --syntax-verify $(COMPILE)
 
 #: PyPy of some sort. E.g. [PyPy 5.0.1 with GCC 4.8.4]
 # Skip for now
@@ -309,8 +332,16 @@ pypy-2.7 5.0 5.3 6.0:
 pypy-3.2 2.4:
 	$(PYTHON) test_pythonlib.py --bytecode-pypy3.2 --verify
 
-#: PyPy 5.0.x with Python 3.6 ...
+#: PyPy 5.0.x with Python 3.6.1 ...
+check-bytecode-pypy3.6: 7.1
 7.1:
+	$(PYTHON) test_pythonlib.py --bytecode-pypy3.6-run --verify-run
+	$(PYTHON) test_pythonlib.py --bytecode-pypy3.6 --verify
+
+#: PyPy 5.0.x with Python 3.6.9
+check-bytecode-pypy3.6: 7.2
+7.2:
+	$(PYTHON) test_pythonlib.py --bytecode-pypy3.6-run --verify-run
 	$(PYTHON) test_pythonlib.py --bytecode-pypy3.6 --verify
 
 

test/add-test.py

@@ -2,22 +2,23 @@
 """ Trivial helper program to bytecompile and run an uncompile
 """
 import os, sys, py_compile
+
 assert len(sys.argv) >= 2
 version = sys.version[0:3]
-if sys.argv[1] == '--run':
+if sys.argv[1] in ("--run", "-r"):
-    suffix = '_run'
+    suffix = "_run"
     py_source = sys.argv[2:]
 else:
-    suffix = ''
+    suffix = ""
     py_source = sys.argv[1:]
 
 for path in py_source:
     short = os.path.basename(path)
-    if hasattr(sys, 'pypy_version_info'):
+    if hasattr(sys, "pypy_version_info"):
-        cfile =  "bytecode_pypy%s%s/%s" % (version, suffix, short) + 'c'
+        cfile = "bytecode_pypy%s%s/%s" % (version, suffix, short) + "c"
     else:
-        cfile =  "bytecode_%s%s/%s" % (version, suffix, short) + 'c'
+        cfile = "bytecode_%s%s/%s" % (version, suffix, short) + "c"
     print("byte-compiling %s to %s" % (path, cfile))
     py_compile.compile(path, cfile)
     if isinstance(version, str) or version >= (2, 6, 0):
-        os.system("../bin/uncompyle6 -a -t %s" % cfile)
+        os.system("../bin/uncompyle6 -a -T %s" % cfile)