Get ready for release 3.6.0

With grammar reduction tests. Same as tryelsestmt. Lots of stdlib tests work
now. (More remain though.)

.circleci/config.yml

@@ -1,11 +1,3 @@
-# This configuration was automatically generated from a CircleCI 1.0 config.
-# It should include any build commands you had along with commands that CircleCI
-# inferred from your project structure. We strongly recommend you read all the
-# comments in this file to understand the structure of CircleCI 2.0, as the idiom
-# for configuration has changed substantially in 2.0 to allow arbitrary jobs rather
-# than the prescribed lifecycle of 1.0. In general, we recommend using this generated
-# configuration as a reference rather than using it in production, though in most
-# cases it should duplicate the execution of your original 1.0 config.
 version: 2
 jobs:
   build:
@@ -27,9 +19,9 @@ jobs:
     # VM instead of a container) see https://circleci.com/docs/2.0/executor-types/
     # To see the list of pre-built images that CircleCI provides for most common languages see
     # https://circleci.com/docs/2.0/circleci-images/
-    docker:
-    - image: circleci/build-image:ubuntu-14.04-XXL-upstart-1189-5614f37
-      command: /sbin/init
+    machine:
+      python:
+        version: 2.7.14
     steps:
     # Machine Setup
     #   If you break your build into multiple jobs with workflows, you will probably want to do the parts of this that are relevant in each
@@ -42,25 +34,24 @@ jobs:
     # This is based on your 1.0 configuration file or project settings
     - run:
         working_directory: ~/rocky/python-uncompyle6
-        command: pyenv local 2.7.11 && pyenv rehash && pip install virtualenv && pip install nose && pip install pep8 && pip install six && pyenv rehash
+        command: pip install virtualenv && pip install nose && pip install pep8 && pyenv rehash
     # Dependencies
     #   This would typically go in either a build or a build-and-test job when using workflows
     # Restore the dependency cache
     - restore_cache:
         keys:
-        # This branch if available
-        - v1-dep-{{ .Branch }}-
-        # Default branch if not
-        - v1-dep-master-
-        # Any branch if there are none on the default branch - this should be unnecessary if you have your default branch configured correctly
-        - v1-dep-
+          - v2-dependencies-{{ .Branch }}-
+          # fallback to using the latest cache if no exact match is found
+          - v2-dependencies-
+
     # This is based on your 1.0 configuration file or project settings
     - run: pip install --upgrade setuptools
     - run: pip install -e .
-    - run: pip install pytest==3.2.5 hypothesis==3.0.0
+    - run: pip install -r requirements-dev.txt
+
     # Save dependency cache
     - save_cache:
-        key: v1-dep-{{ .Branch }}-{{ epoch }}
+        key: v2-dependencies-{{ .Branch }}-{{ epoch }}
         paths:
         # This is a broad list of cache paths to include many possible development environments
         # You can probably delete some of these entries
@@ -69,14 +60,13 @@ jobs:
         - ~/.m2
         - ~/.ivy2
         - ~/.bundle
-        - ~/.go_workspace
-        - ~/.gradle
         - ~/.cache/bower
+
     # Test
     #   This would typically be a build job when using workflows, possibly combined with build
     # This is based on your 1.0 configuration file or project settings
     - run: python ./setup.py develop && make check-2.7
-    - run: cd ./test/stdlib && pyenv local 2.7.11 && bash ./runtests.sh 'test_[p-z]*.py'
+    - run: cd ./test/stdlib && bash ./runtests.sh 'test_[p-z]*.py'
     # Teardown
     #   If you break your build into multiple jobs with workflows, you will probably want to do the parts of this that are relevant in each
     # Save test results

.gitignore

@@ -6,6 +6,7 @@
 /.eggs
 /.hypothesis
 /.idea
+/.mypy_cache
 /.pytest_cache
 /.python-version
 /.tox

.travis.yml

@@ -5,6 +5,7 @@ python:
   - '2.7'
   - '3.4'
   - '3.6'
+  - '3.8'
 
 matrix:
   include:

Makefile

@@ -47,8 +47,12 @@ check-3.8:
 # Skip for now
 2.6 5.0 5.3 5.6 5.8:
 
-#:PyPy pypy3-2.4.0 Python 3:
-pypy-3.2 2.4:
+#:PyPy pypy3-2.4.0 Python 3.6.1:
+7.1 pypy-3.2 2.4:
+	$(MAKE) -C test $@
+
+#:PyPy pypy3-2.4.0 Python 3.6.9:
+7.2:
 	$(MAKE) -C test $@
 
 #: Run py.test tests

NEWS.md

@@ -1,15 +1,85 @@
+3.6.0: 2019-12-10 gecko gecko
+=============================
+
+The main focus in this release was more accurate decompilation especially
+for 3.7 and 3.8. However there are some improvments to Python 2.x as well,
+including one of the long-standing problems of detecting the difference between
+`try ... ` and `try else ...`.
+
+With this release we now rebase Python 3.7 on off of a 3.7 base; This
+is also as it is (now) in decompyle3.  This facilitates removing some of the
+cruft in control-flow detection in the 2.7 uncompyle2 base.
+
+Alas, decompilation speed for 3.7 on is greatly increased. Hopefull
+this is temporary (cough, cough) until we can do a static control flow
+pass.
+
+Finally, runing in 3.9-dev is tolerated. We can disassemble, but no parse tables yet.
+
+
+3.5.1 2019-11-17 JNC
+====================
+
+- Pypy 3.3, 3.5, 3.6, and 3.6.9 support
+- bump xdis version to handle newer Python releases, e.g. 2.7.17, 3.5.8, and 3.5.9
+- Improve 3.0 decompilation
+    - no parse errors on stlib bytecode. However accurate translation in
+	  control-flow and and/or detection needs work
+- Remove extraneous iter() in "for" of list comprehension  Fixes #272
+- "for" block without a `POP_BLOCK `and confusing `JUMP_BACK` for `CONTINUE`. Fixes #293
+- Fix unmarshal incompletness detected in Pypy 3.6
+- Miscellaneous bugs fixed
+
+3.5.0 2019-10-12 Stony Brook Ride
+=================================
+
+- Fix fragment bugs
+   * missing `RETURN_LAST` introduced when adding transformation layer
+   * more parent entries on tokens
+- Preliminary support for decompiling Python 1.0, 1.1, 1.2, and 1.6
+   * Newer _xdis_ version needed
+
+3.4.1 2019-10-02
+================
+
+- Correct assert{,2} transforms Fixes #289
+- Fragment parsing fixes:
+     * Wasn't handling 3-arg `%p`
+   	 * fielding error in `code_deparse()`
+- Use newer _xdis_ to better track Python 3.8.0
+
+
+3.4.0 2019-08-24 Totoro
+=======================
+
+The main change is to add a tree-transformation phase. This simplifies the
+code a little and allows us to turn `if ...: raise AssertionError` into
+`assert`, and many `if ..: else if ...` into `if ... elif ..`
+
+Use options `--show=before` and `--show=after` to see the before the tree transformation phase and after the tree transformation phase.
+
+Most of the heavy lifting for this was done by x0ret.
+
+Other changes:
+
+- Fix issue #275, #283 (process to fix this bug is documented on wiki), #284
+- blacken more code
+- CircleCI adjustments for a changing CircleCi
+- Require more recent `xdis` for Python 3.8
+- Fix bugs in code using `BUILD_LIST_UNPACK` and variants
+
 3.3.5 2019-07-03 Pre Independence Day
 =====================================
 
 Again, most of the work in this is release is thanks to x0ret.
 
-- Handle annotation args in Python 3.x
-- Fix vararg and function signatures in 3.x
-- Some 3.x < 3.6 while(1)/if fixes - others remain
-- Start reinstating else if -> elif
-- LOAD_CONST -> LOAD_CODE where appropriate
-- option `weak-verify` is now `syntax-verify`
-- code cleanups, start using "blacken" to reformat text
+- Handle annotation arguments in Python 3.x
+- Fix _vararg_ and function signatures in 3.x
+- Some 3.x < 3.6 `while` (1)/`if` fixes &mdash; others remain
+- Start reinstating `else if` -> `elif`
+- `LOAD_CONST` -> `LOAD_CODE` where appropriate
+- option `--weak-verify` is now `--syntax-verify`
+- code cleanups, start using [black](https://github.com/python/black) to reformat text
 
 
 3.3.4 2019-06-19 Fleetwood at 65
@@ -18,12 +88,12 @@ Again, most of the work in this is release is thanks to x0ret.
 Most of the work in this is release is thanks to x0ret.
 
 - Major work was done by x0ret to correct function signatures and include annotation types
-- Handle Python 3.6 STORE_ANNOTATION [#58](https://github.com/rocky/python-uncompyle6/issues/58)
+- Handle Python 3.6 `STORE_ANNOTATION` [#58](https://github.com/rocky/python-uncompyle6/issues/58)
 - Friendlier assembly output
 - `LOAD_CONST` replaced by `LOAD_STR` where appropriate to simplify parsing and improve clarity
 - remove unneeded parenthesis in a generator expression when it is the single argument to the function [#247](https://github.com/rocky/python-uncompyle6/issues/246)
 - Bug in noting an async function [#246](https://github.com/rocky/python-uncompyle6/issues/246)
-- Handle unicode docstrings and fix docstring bugs [#241](https://github.com/rocky/python-uncompyle6/issues/241)
+- Handle Unicode docstrings and fix docstring bugs [#241](https://github.com/rocky/python-uncompyle6/issues/241)
 - Add short option -T as an alternate for --tree+
 - Some grammar cleanup
 
@@ -67,7 +137,7 @@ some span back as far as 2.x
 But as before, many more remain in the 3.7 and 3.8 range which will
 get addressed in future releases
 
-Pypy 3.6 support was started. Pypy 3.x detection fixed (via xdis)
+Pypy 3.6 support was started. Pypy 3.x detection fixed (via `xdis`)
 
 3.3.1 2019-04-19 Good Friday
 ==========================
@@ -151,7 +221,7 @@ Pull Requests
 
 - Add rudimentary 1.4 support (still a bit buggy)
 - add --tree+ option to show formatting rule, when it is constant
-- Python 2.7.15candidate1 support (via xdis)
+- Python 2.7.15candidate1 support (via `xdis`)
 - bug fixes, especially for 3.7 (but 2.7 and 3.6 and others as well)
 
 3.1.3 2018-04-16
@@ -256,24 +326,24 @@ function calls and definitions.
 2.15.1 2018-01-27
 =====================
 
-- Add --linemap option to give line correspondences
+- Add `--linemap` option to give line correspondences
   between original source lines and reconstructed line sources.
   It is far from perfect, but it is a start
 - Add a new class of tests: tests which when decompiled check themselves
 - Split off Python version semantic action customizations into its own file
-- Fix 2.7 bug in ifelse loop statement
-- Handle 3.6+ EXTENDED_ARGs for POP_JUMP_IF... instructions
-- Correct 3.6+ calls with kwargs
+- Fix 2.7 bug in `if`/`else` loop statement
+- Handle 3.6+ `EXTENDED_ARG`s for `POP_JUMP_IF..` instructions
+- Correct 3.6+ calls with `kwargs`
 - Describe the difficulty of 3.6 in README
 
 2.14.3 2018-01-19
 =====================
 
-- Fix bug in 3.5+ await stmt
+- Fix bug in 3.5+ `await` statement
 - Better version to magic handling; handle 3.5.2 .. 3.5.4 versions
 - Improve/correct test_pyenvlib.py status messages
 - Fix some 2.7 and 2.6 parser bugs
-- Fix whilelse parsing bugs
+- Fix `whilelse` parsing bugs
 - Correct 2.5- decorator parsing
 - grammar for decorators matches AST a little more
 - better tests in setup.py for running the right version of Python
@@ -284,15 +354,15 @@ function calls and definitions.
 
 Decompilation bug fixes, mostly 3.6 and pre 2.7
 
-- 3.6 FUNCTION_EX (somewhat)
-- 3.6 FUNCTION_EX_KW fixes
-- 3.6 MAKE_FUNCTION fixes
-- correct 3.5 CALL_FUNCTION_VAR
+- 3.6 `FUNCTION_EX` (somewhat)
+- 3.6 `FUNCTION_EX_KW` fixes
+- 3.6 `MAKE_FUNCTION` fixes
+- correct 3.5 `CALL_FUNCTION_VAR`
 - stronger 3.x "while 1" testing
 - Fix bug in if's with "pass" bodies. Fixes #104
 - try/else and try/finally fixes on 2.6-
 - limit pypy customization to pypy
-- Add addr fields in COME_FROMS
+- Add addr fields in `COME_FROM`S
 - Allow use of full instructions in parser reduction routines
 - Reduce grammar in Python 3 by specialization more to specific
   Python versions
@@ -301,11 +371,11 @@ Decompilation bug fixes, mostly 3.6 and pre 2.7
 2.14.1 2017-12-10 Dr. Gecko
 ===================================
 
-- Many decompilation bugfixes
+- Many decompilation bug fixes
 - Grammar rule reduction and version isolation
 - Match higher-level nonterminal names more closely
   with Python AST
-- Start automated Python stdlib testing - full round trip
+- Start automated Python _stdlib_ testing &mdash; full round trip
 
 2.14.0 2017-11-26 johnnybamazing
 =========================================
@@ -314,7 +384,7 @@ Decompilation bug fixes, mostly 3.6 and pre 2.7
   and remove used grammar rules
 - Fix a number of bytecode decompile problems
   (many more remain)
-- Add stdlib/runtests.sh for even more rigorous testing
+- Add `stdlib/runtests.sh` for even more rigorous testing
 
 2.13.3 2017-11-13
 =====================
@@ -330,16 +400,16 @@ Overall: better 3.6 decompiling and some much needed code refactoring and cleanu
   rather trying to parse the bytecode array. This largely been done in for versions 3.x;
   3.0 custom mangling code has been reduced;
   some 2.x conversion has been done, but more is desired. This make it possible to...
-- Handle EXTENDED_ARGS better. While relevant to all Python versions it is most noticeable in
-  version 3.6+ where in switching to wordcodes the size of operands has been reduced from 2**16
-  to 2**8. JUMP instruction then often need EXTENDED_ARGS.
+- Handle `EXTENDED_ARGS` better. While relevant to all Python versions it is most noticeable in
+  version 3.6+ where in switching to wordcodes the size of operands has been reduced from 2^16
+  to 2^8. `JUMP` instruction then often need EXTENDED_ARGS.
 - Refactor find_jump_targets() with via working of of instructions rather the bytecode array.
-- use --weak-verify more and additional fuzzing on verify()
+- use `--weak-verify` more and additional fuzzing on verify()
 - fragment parser now ignores errors in nested function definitions; an parameter was
   added to assist here. Ignoring errors may be okay because the fragment parser often just needs,
   well, *fragments*.
-- Distinguish RETURN_VALUE from RETURN_END_IF in exception bodies better in 3.6
-- bug in 3.x language changes: import queue via import Queue
+- Distinguish `RETURN_VALUE` from `RETURN_END_IF` in exception bodies better in 3.6
+- bug in 3.x language changes: import queue via `import Queue`
 - reinstate some bytecode tests since decompiling has gotten better
 - Revise how to report a bug
 
@@ -359,12 +429,12 @@ Overall: better 3.6 decompiling and some much needed code refactoring and cleanu
 - Fixes in deparsing lambda expressions
 - Improve table-semantics descriptions
 - Document hacky customize arg count better (until we can remove it)
-- Update to use xdis 3.7.0 or greater
+- Update to use `xdis` 3.7.0 or greater
 
 2.12.0 2017-09-26
 =====================
 
-- Use xdis 3.6.0 or greater now
+- Use `xdis` 3.6.0 or greater now
 - Small semantic table cleanups
 - Python 3.4's terms a little names better
 - Slightly more Python 3.7, but still failing a lot
@@ -378,13 +448,13 @@ Overall: better 3.6 decompiling and some much needed code refactoring and cleanu
 2.11.4 2017-08-15
 =====================
 
-* scanner and parser now allow 3-part version string lookups,
+* scanner and parser now allow 3-part version string look ups,
   e.g. 2.7.1 We allow a float here, but if passed a string like '2.7'. or
-* unpin 3.5.1. xdis 3.5.4 has been release and fixes the problems we had. Use that.
-* some routines here moved to xdis. Use the xdis version
-* README.rst: Link typo Name is trepan2 now not trepan
-* xdis-forced change adjust for COMPARE_OP "is-not" in
-  semanatic routines. We need "is not".
+* unpin 3.5.1. `xdis` 3.5.4 has been release and fixes the problems we had. Use that.
+* some routines here moved to `xdis`. Use the `xdis` version
+* `README.rst`: Link typo Name is _trepan2_ now not _trepan_
+* xdis-forced change adjust for `COMPARE_OP` "is-not" in
+  semantic routines. We need "is not".
 * Some PyPy tolerance in validate testing.
 * Some pyston tolerance
 
@@ -394,15 +464,15 @@ Overall: better 3.6 decompiling and some much needed code refactoring and cleanu
 Very minor changes
 
 - RsT doc fixes and updates
-- use newer xdis, but not too new; 3.5.2 breaks uncompyle6
-- use xdis opcode sets
-- xdis "exception match" is now "exception-match"
+- use newer `xdis`, but not too new; 3.5.2 breaks uncompyle6
+- use `xdis` opcode sets
+- `xdis` "exception match" is now "exception-match"
 
 2.11.2 2017-07-09
 =====================
 
 - Start supporting Pypy 3.5 (5.7.1-beta)
-- use xdis 3.5.0's opcode sets and require xdis 3.5.0
+- use `xdis` 3.5.0's opcode sets and require `xdis` 3.5.0
 - Correct some Python 2.4-2.6 loop detection
 - guard against badly formatted bytecode
 
@@ -410,55 +480,55 @@ Very minor changes
 =====================
 
 - Python 3.x annotation and function signature fixes
-- Bump xdis version
-- Small pysource bug fixes
+- Bump `xdis` version
+- Small `pysource.py` bug fixes
 
 2.11.0 2017-06-18 Fleetwood
 ==================================
 
 - Major improvements in fragment tracking
-  * Add nonterminal node in extractInfo
+  * Add nonterminal node in `extractInfo()`
   * tag more offsets in expressions
   * tag array subscripts
-  * set YIELD value offset in a <yield> expr
+  * set `YIELD` value offset in a _yield expr_
   * fix a long-standing bug in not adjusting final AST when melding other deparse ASTs
 - Fixes yet again for make_function node handling; document what's up here
-- Fix bug in snowflake Python 3.5 *args kwargs
+- Fix bug in snowflake Python 3.5 `*args`, `kwargs`
 
 2.10.1 2017-06-3 Marylin Frankel
 ========================================
 
 - fix some fragments parsing bugs
-  - was returning the wrong type sometimes in deparse_code_around_offset()
+  - was returning the wrong type sometimes in `deparse_code_around_offset()`
   - capture function name in offsets
-  - track changes to ifelstrmtr node from pysource into fragments
+  - track changes to `ifelstrmtr` node from `pysource.py` into fragments
 
 2.10.0 2017-05-30 Elaine Gordon
 =======================================
 
 - Add fuzzy offset deparse look up
 - 3.6 bug fixes
-   - fix EXTENDED_ARGS handling (and in 2.6 and others)
+   - fix `EXTENDED_ARGS` handling (and in 2.6 and others)
    - semantic routine make_function fragments.py
-   - MAKE_FUNCTION handling
-   - CALL_FUNCTION_EX handling
-   - async property on defs
-   - support for CALL_FUNCTION_KW (moagstar)
-- 3.5+ UNMAP_PACK and BUILD_UNMAP_PACK handling
+   - `MAKE_FUNCTION` handling
+   - `CALL_FUNCTION_EX` handling
+   - `async` property on `defs`
+   - support for `CALL_FUNCTION_KW` (moagstar)
+- 3.5+ `UNMAP_PACK` and` BUILD_UNMAP_PACK` handling
 - 3.5 FUNCTION_VAR bug
-- 3.x pass statement insdie while True
+- 3.x pass statement inside `while True`
 - Improve 3.2 decompilation
-- Fixed -o argument processing (grkov90)
+- Fixed `-o` argument processing (grkov90)
 - Reduce scope of LOAD_ASSERT as expr to 3.4+
-- "await" statement fixes
+- `await` statement fixes
 - 2.3, 2.4 "if 1 .." fixes
 - 3.x annotation fixes
 
 2.9.11 2017-04-06
 =====================
 
-- Better support for Python 3.5+ BUILD_MAP_UNPACK
-- Start 3.6 CALL_FUNCTION_EX support
+- Better support for Python 3.5+ `BUILD_MAP_UNPACK`
+- Start 3.6 `CALL_FUNCTION_EX` support
 - Many decompilation bug fixes. (Many more remain). See ChangeLog
 
 2.9.10 2017-02-25
@@ -466,7 +536,7 @@ Very minor changes
 
 - Python grammar rule fixes
 - Add ability to get grammar coverage on runs
-- Handle Python 3.6 opcode BUILD_CONST_KEYMAP
+- Handle Python 3.6 opcode `BUILD_CONST_KEYMAP`
 
 2.9.9 2016-12-16
 
@@ -482,13 +552,13 @@ Very minor changes
 ====================
 
 - Better control-flow detection
-- pseudo instruction THEN in 2.x
+- pseudo instruction `THEN` in 2.x
   to disambiguate if from and
-- fix bug in --verify option
+- fix bug in `--verify` option
 - DRY (a little) control-flow detection
 - fix syntax in tuples with one element
 - if AST rule inheritance in Python 2.5
-- NAME_MODULE removal for Python <= 2.4
+- `NAME_MODULE` removal for Python <= 2.4
 - verify call fixes for Python <= 2.4
 - more Python lint
 
@@ -499,9 +569,9 @@ Very minor changes
 - Some Python 3.6 bytecode to wordcode conversion fixes
 - option -g: show start-end range when possible
 - track print_docstring move to help (used in python 3.1)
-- verify: allow RETURN_VALUE to match RETURN_END_IF
+- verify: allow `RETURN_VALUE` to match `RETURN_END_IF`
 - some 3.2 compatibility
-- Better Python 3 control flow detection by adding Pseudo ELSE opcodes
+- Better Python 3 control flow detection by adding Pseudo `ELSE` opcodes
 
 2.9.6 2016-12-04
 ====================
@@ -517,7 +587,7 @@ Very minor changes
 
 - Correct MANIFEST.in
 - More AST grammar checking
-- --linemapping option or linenumbers.line_number_mapping()
+- `--linemapping` option or _linenumbers.line_number_mapping()_
   Shows correspondence of lines between source
   and decompiled source
 - Some control flow adjustments in code for 2.x.
@@ -537,7 +607,7 @@ Very minor changes
   * improper while 1 else
   * docstring indent
   * 3.3 default values in lambda expressions
-  * start 3.0 decompilation (needs newer xdis)
+  * start 3.0 decompilation (needs newer `xdis`)
 - Start grammar misparse checking
 
 
@@ -551,12 +621,12 @@ Very minor changes
 2.9.3 2016-10-26
 ====================
 
-Release forced by incompatibility change in xdis 3.2.0.
+Release forced by incompatibility change in` xdis` 3.2.0.
 
 - Python 3.1 bugs:
-  * handle "with ... as"
-  * handle "with"
-  * Start handling def (...) -> yy (has bugs still)
+  * handle `with`... `as`
+  * handle `with`
+  * Start handling `def` (...) -> _yy_ (has bugs still)
 
 - DRY Python 3.x via inheritance
 - Python 3.6 work (from Daniel Bradburn)
@@ -582,12 +652,12 @@ Release forced by incompatibility change in xdis 3.2.0.
 2.9.0 2016-10-09
 ====================
 
-- Use xdis 3.0.0 protocol load_module.
+- Use `xdis` 3.0.0 protocol `load_module`.
   this Forces change in requirements.txt and _pkg_info_.py
 - Start Python 1.5 decompiling; another round of work is needed to
   remove bugs
 - Simplify python 2.1 grammar
-- Fix bug with -t ...  Wasn't showing source text when -t option was given
+- Fix bug with `-t` ...  Wasn't showing source text when `-t` option was given
 - Fix 2.1-2.6 bug in list comprehension
 
 2.8.4 2016-10-08
@@ -596,8 +666,8 @@ Release forced by incompatibility change in xdis 3.2.0.
 - Python 3 disassembly bug fixes
 - Python 3.6 fstring bug fixes (from moagstar)
 - Python 2.1 disassembly
-- COME_FROM suffixes added in Python3
-- use .py extension in verification disassembly
+- `COME_FROM` suffixes added in Python3
+- use `.py` extension in verification disassembly
 
 2.8.3 2016-09-11 live from NYC!
 =======================================
@@ -642,8 +712,8 @@ control-flow structure detection is done.
 - Add Python 2.2 decompilation
 
 - Fix bugs
- * PyPy LOOKUP_METHOD bug
- * Python 3.6 FORMAT_VALUE handles expressions now
+ * PyPy `LOOKUP_METHOD` bug
+ * Python 3.6 `FORMAT_VALUE` handles expressions now
 
 2.8.0 2016-08-03
 ====================
@@ -693,7 +763,7 @@ control-flow structure detection is done.
 ====================
 
 - Improve Python 2.6 bytecode deparsing:
-  stdlib now will deparse something
+  _stdlib_ now will deparse something
 - Better <2.6 vs. 2.7 grammar separation
 - Fix some 2.7 deparsing bugs
 - Fix bug in installing uncompyle6 script
@@ -756,9 +826,9 @@ control-flow structure detection is done.
 2.3.2 2016-05-1
 ===================
 
-- Add --version option standalone scripts
+- Add `--version` option standalone scripts
 - Correct License information in package
-- expose fns uncompyle_file, load_file, and load_module
+- expose functions `uncompyle_file()`, `load_file()`, and `load_module()`
 - Start to DRY Python2 and Python3 grammars Separate out 3.2, and 3.5+
   specific grammar code
 - Fix bug in 3.5+ constant map parsing
@@ -766,12 +836,12 @@ control-flow structure detection is done.
 2.3.0, 2.3.1 2016-04-30
 =============================
 
-- Require spark_parser  >= 1.1.0
+- Require `spark_parser` >= 1.1.0
 
 2.2.0 2016-04-30
 ====================
 
-- Spark is no longer here but pulled separate package spark_parse
+- Spark is no longer here but pulled separate package [spark_parser](https://pypi.org/project/spark_parser/)
 - Python 3 parsing fixes
 - More tests
 
@@ -781,7 +851,7 @@ control-flow structure detection is done.
 - Support single-mode (in addition to exec-mode) compilation
 - Start to DRY Python 2 and Python 3 grammars
 - Fix bug in if else ternary  construct
-- Fix bug in uncomplye6 -d and -r options (via lelicopter)
+- Fix bug in uncomplye6 `-d` and `-r` options (via lelicopter)
 
 
 2.1.3 2016-01-02
@@ -789,7 +859,7 @@ control-flow structure detection is done.
 
 - Limited support for decompiling Python 3.5
 - Improve Python 3 class deparsing
-- Handle MAKE_CLOSURE opcode
+- Handle `MAKE_CLOSURE` opcode
 - Start to DRY opcode code.
 - increase test coverage
 - fix misc small bugs and some improvements
@@ -831,5 +901,5 @@ Changes from uncompyle2
 
 SPARK:
   add option to show grammar rules applied
-  allow Python-style # comments in grammar
+  allow Python-style `#` comments in grammar
   Runs on Python 3 and Python 2

README.rst

@@ -1,4 +1,6 @@
-|buildstatus| |Latest Version| |Supported Python Versions|
+|buildstatus|  |Pypi Installs| |Latest Version| |Supported Python Versions|
+
+|packagestatus|
 
 uncompyle6
 ==========
@@ -11,9 +13,9 @@ Introduction
 ------------
 
 *uncompyle6* translates Python bytecode back into equivalent Python
-source code. It accepts bytecodes from Python version 1.3 to version
+source code. It accepts bytecodes from Python version 1.0 to version
 3.8, spanning over 24 years of Python releases. We include Dropbox's
-Python 2.5 bytecode and some PyPy bytecode.
+Python 2.5 bytecode and some PyPy bytecodes.
 
 Why this?
 ---------
@@ -44,14 +46,15 @@ not exist and there is just bytecode. Again, my debuggers make use of
 this.
 
 There were (and still are) a number of decompyle, uncompyle,
-uncompyle2, uncompyle3 forks around. Almost all of them come basically
-from the same code base, and (almost?) all of them are no longer
-actively maintained. One was really good at decompiling Python 1.5-2.3
-or so, another really good at Python 2.7, but that only. Another
-handles Python 3.2 only; another patched that and handled only 3.3.
-You get the idea. This code pulls all of these forks together and
-*moves forward*. There is some serious refactoring and cleanup in this
-code base over those old forks.
+uncompyle2, uncompyle3 forks around. Many of them come basically from
+the same code base, and (almost?) all of them are no longer actively
+maintained. One was really good at decompiling Python 1.5-2.3, another
+really good at Python 2.7, but that only. Another handles Python 3.2
+only; another patched that and handled only 3.3.  You get the
+idea. This code pulls all of these forks together and *moves
+forward*. There is some serious refactoring and cleanup in this code
+base over those old forks. Even more experimental refactoring is going
+on in decompile3_.
 
 This demonstrably does the best in decompiling Python across all
 Python versions. And even when there is another project that only
@@ -73,11 +76,11 @@ fixed in the other decompilers.
 Requirements
 ------------
 
-The code here can be run on Python versions 2.6 or later, PyPy 3-2.4,
-or PyPy-5.0.1.  Python versions 2.4-2.7 are supported in the
-python-2.4 branch.  The bytecode files it can read have been tested on
-Python bytecodes from versions 1.4, 2.1-2.7, and 3.0-3.8 and the
-above-mentioned PyPy versions.
+The code here can be run on Python versions 2.6 or later, PyPy 3-2.4
+and later.  Python versions 2.4-2.7 are supported in the python-2.4
+branch.  The bytecode files it can read have been tested on Python
+bytecodes from versions 1.4, 2.1-2.7, and 3.0-3.8 and later PyPy
+versions.
 
 Installation
 ------------
@@ -86,9 +89,9 @@ This uses setup.py, so it follows the standard Python routine:
 
 ::
 
-    pip install -e .  # set up to run from source tree
-                      # Or if you want to install instead
-    python setup.py install # may need sudo
+    $ pip install -e .  # set up to run from source tree
+                        # Or if you want to install instead
+    $ python setup.py install # may need sudo
 
 A GNU makefile is also provided so :code:`make install` (possibly as root or
 sudo) will do the steps above.
@@ -132,7 +135,7 @@ could be compared with the original bytecode. However as Python's code
 generation got better, this no longer was feasible.
 
 If you want Python syntax verification of the correctness of the
-decompilation process, add the `--syntax-verify` option. However since
+decompilation process, add the :code:`--syntax-verify` option. However since
 Python syntax changes, you should use this option if the bytecode is
 the right bytecode for the Python interpreter that will be checking
 the syntax.
@@ -146,7 +149,7 @@ available give stronger verification: those programs that when run
 test themselves. Our test suite includes these.
 
 And Python comes with another a set of programs like this: its test
-suite for the standard library. We have some code in `test/stdlib` to
+suite for the standard library. We have some code in :code:`test/stdlib` to
 facilitate this kind of checking too.
 
 Known Bugs/Restrictions
@@ -177,22 +180,28 @@ In the Python 3 series, Python support is is strongest around 3.4 or
 3.0 is weird in that it in some ways resembles 2.6 more than it does
 3.1 or 2.7. Python 3.6 changes things drastically by using word codes
 rather than byte codes. As a result, the jump offset field in a jump
-instruction argument has been reduced. This makes the `EXTENDED_ARG`
+instruction argument has been reduced. This makes the :code:`EXTENDED_ARG`
 instructions are now more prevalent in jump instruction; previously
 they had been rare.  Perhaps to compensate for the additional
-`EXTENDED_ARG` instructions, additional jump optimization has been
+:code:`EXTENDED_ARG` instructions, additional jump optimization has been
 added. So in sum handling control flow by ad hoc means as is currently
 done is worse.
 
-Between Python 3.5, 3.6 and 3.7 there have been major changes to the
-`MAKE_FUNCTION` and `CALL_FUNCTION` instructions.
+Between Python 3.5, 3.6, 3.7 there have been major changes to the
+:code:`MAKE_FUNCTION` and :code:`CALL_FUNCTION` instructions.  Python
+
+Python 3.8 removes :code:`SETUP_LOOP`, :code:`SETUP_EXCEPT`,
+:code:`BREAK_LOOP`, and :code:`CONTINUE_LOOP`, instructions which may
+make control-flow detection harder, lacking the more sophisticated
+control-flow analysis that is planned. We'll see.
 
 Currently not all Python magic numbers are supported. Specifically in
 some versions of Python, notably Python 3.6, the magic number has
 changes several times within a version.
 
-**We support only released versions, not candidate versions.** Note however
-that the magic of a released version is usually the same as the *last* candidate version prior to release.
+**We support only released versions, not candidate versions.** Note
+however that the magic of a released version is usually the same as
+the *last* candidate version prior to release.
 
 There are also customized Python interpreters, notably Dropbox,
 which use their own magic and encrypt bytcode. With the exception of
@@ -214,15 +223,16 @@ There is lots to do, so please dig in and help.
 See Also
 --------
 
-* https://github.com/zrax/pycdc : purports to support all versions of Python. It is written in C++ and is most accurate for Python versions around 2.7 and 3.3 when the code was more actively developed. Accuracy for more recent versions of Python 3 and early versions of Python are especially lacking. See its `issue tracker <https://github.com/zrax/pycdc/issues>`_ for details. Currently lightly maintained.
+* https://github.com/rocky/python-decompile3 : Much smaller and more modern code, focusing on 3.7+. Changes in that will get migrated back ehre.
 * https://code.google.com/archive/p/unpyc3/ : supports Python 3.2 only. The above projects use a different decompiling technique than what is used here. Currently unmaintained.
 * https://github.com/figment/unpyc3/ : fork of above, but supports Python 3.3 only. Includes some fixes like supporting function annotations. Currently unmaintained.
-* https://github.com/wibiti/uncompyle2 : supports Python 2.7 only, but does that fairly well. There are situations where `uncompyle6` results are incorrect while `uncompyle2` results are not, but more often uncompyle6 is correct when uncompyle2 is not. Because `uncompyle6` adheres to accuracy over idiomatic Python, `uncompyle2` can produce more natural-looking code when it is correct. Currently `uncompyle2` is lightly maintained. See its issue `tracker <https://github.com/wibiti/uncompyle2/issues>`_ for more details
+* https://github.com/wibiti/uncompyle2 : supports Python 2.7 only, but does that fairly well. There are situations where :code:`uncompyle6` results are incorrect while :code:`uncompyle2` results are not, but more often uncompyle6 is correct when uncompyle2 is not. Because :code:`uncompyle6` adheres to accuracy over idiomatic Python, :code:`uncompyle2` can produce more natural-looking code when it is correct. Currently :code:`uncompyle2` is lightly maintained. See its issue `tracker <https://github.com/wibiti/uncompyle2/issues>`_ for more details
 * `How to report a bug <https://github.com/rocky/python-uncompyle6/blob/master/HOW-TO-REPORT-A-BUG.md>`_
 * The HISTORY_ file.
 * https://github.com/rocky/python-xdis : Cross Python version disassembler
 * https://github.com/rocky/python-xasm : Cross Python version assembler
 * https://github.com/rocky/python-uncompyle6/wiki : Wiki Documents which describe the code and aspects of it in more detail
+* https://github.com/zrax/pycdc : The README for this C++ code syas it aims to support all versions of Python. It is best for Python versions around 2.7 and 3.3 when the code was initially developed. Accuracy for current versions of Python3 and early versions of Python is lacking. Without major effort, it is unlikely it can be made to support current Python 3. See its `issue tracker <https://github.com/zrax/pycdc/issues>`_ for details. Currently lightly maintained.
 
 
 .. _trepan: https://pypi.python.org/pypi/trepan2g
@@ -231,12 +241,16 @@ See Also
 .. _debuggers: https://pypi.python.org/pypi/trepan3k
 .. _remake: https://bashdb.sf.net/remake
 .. _pycdc: https://github.com/zrax/pycdc
+.. _decompile3: https://github.com/rocky/python-decompile3
 .. _this: https://github.com/rocky/python-uncompyle6/wiki/Deparsing-technology-and-its-use-in-exact-location-reporting
 .. |buildstatus| image:: https://travis-ci.org/rocky/python-uncompyle6.svg
 		 :target: https://travis-ci.org/rocky/python-uncompyle6
+.. |packagestatus| image:: https://repology.org/badge/vertical-allrepos/python:uncompyle6.svg
+		 :target: https://repology.org/project/python:uncompyle6/versions
 .. _PJOrion: http://www.koreanrandom.com/forum/topic/15280-pjorion-%D1%80%D0%B5%D0%B4%D0%B0%D0%BA%D1%82%D0%B8%D1%80%D0%BE%D0%B2%D0%B0%D0%BD%D0%B8%D0%B5-%D0%BA%D0%BE%D0%BC%D0%BF%D0%B8%D0%BB%D1%8F%D1%86%D0%B8%D1%8F-%D0%B4%D0%B5%D0%BA%D0%BE%D0%BC%D0%BF%D0%B8%D0%BB%D1%8F%D1%86%D0%B8%D1%8F-%D0%BE%D0%B1%D1%84
 .. _Deobfuscator: https://github.com/extremecoders-re/PjOrion-Deobfuscator
 .. _Py2EXE: https://en.wikipedia.org/wiki/Py2exe
 .. |Supported Python Versions| image:: https://img.shields.io/pypi/pyversions/uncompyle6.svg
 .. |Latest Version| image:: https://badge.fury.io/py/uncompyle6.svg
 		 :target: https://badge.fury.io/py/uncompyle6
+.. |Pypi Installs| image:: https://pepy.tech/badge/uncompyle6/month

__pkginfo__.py

@@ -26,46 +26,46 @@ copyright   = """
 Copyright (C) 2015-2019 Rocky Bernstein <rb@dustyfeet.com>.
 """
 
-classifiers =  ['Development Status :: 5 - Production/Stable',
-                'Intended Audience :: Developers',
-                'License :: OSI Approved :: GNU General Public License v3 (GPLv3)',
-                'Operating System :: OS Independent',
-                'Programming Language :: Python',
-                'Programming Language :: Python :: 2.4',
-                'Programming Language :: Python :: 2.5',
-                'Programming Language :: Python :: 2.6',
-                'Programming Language :: Python :: 2.7',
-                'Programming Language :: Python :: 3.0',
-                'Programming Language :: Python :: 3.1',
-                'Programming Language :: Python :: 3.2',
-                'Programming Language :: Python :: 3.3',
-                'Programming Language :: Python :: 3.4',
-                'Programming Language :: Python :: 3.5',
-                'Programming Language :: Python :: 3.6',
-                'Programming Language :: Python :: 3.7',
-                'Programming Language :: Python :: 3.8',
-                'Topic :: Software Development :: Debuggers',
-                'Topic :: Software Development :: Libraries :: Python Modules',
+classifiers =  ["Development Status :: 5 - Production/Stable",
+                "Intended Audience :: Developers",
+                "License :: OSI Approved :: GNU General Public License v3 (GPLv3)",
+                "Operating System :: OS Independent",
+                "Programming Language :: Python",
+                "Programming Language :: Python :: 2.4",
+                "Programming Language :: Python :: 2.5",
+                "Programming Language :: Python :: 2.6",
+                "Programming Language :: Python :: 2.7",
+                "Programming Language :: Python :: 3.0",
+                "Programming Language :: Python :: 3.1",
+                "Programming Language :: Python :: 3.2",
+                "Programming Language :: Python :: 3.3",
+                "Programming Language :: Python :: 3.4",
+                "Programming Language :: Python :: 3.5",
+                "Programming Language :: Python :: 3.6",
+                "Programming Language :: Python :: 3.7",
+                "Programming Language :: Python :: 3.8",
+                "Topic :: Software Development :: Debuggers",
+                "Topic :: Software Development :: Libraries :: Python Modules",
                 ]
 
 # The rest in alphabetic order
 author             = "Rocky Bernstein, Hartmut Goebel, John Aycock, and others"
 author_email       = "rb@dustyfeet.com"
 entry_points       = {
-    'console_scripts': [
-        'uncompyle6=uncompyle6.bin.uncompile:main_bin',
-        'pydisassemble=uncompyle6.bin.pydisassemble:main',
+    "console_scripts": [
+        "uncompyle6=uncompyle6.bin.uncompile:main_bin",
+        "pydisassemble=uncompyle6.bin.pydisassemble:main",
     ]}
 ftp_url            = None
-install_requires   = ['spark-parser >= 1.8.7, < 1.9.0',
-                      'xdis >= 4.0.2, < 4.1.0']
+install_requires   = ["spark-parser >= 1.8.9, < 1.9.0",
+                      "xdis >= 4.2.0, < 4.3.0"]
 
-license            = 'GPL3'
-mailing_list       = 'python-debugger@googlegroups.com'
-modname            = 'uncompyle6'
+license            = "GPL3"
+mailing_list       = "python-debugger@googlegroups.com"
+modname            = "uncompyle6"
 py_modules         = None
-short_desc         = 'Python cross-version byte-code decompiler'
-web                = 'https://github.com/rocky/python-uncompyle6/'
+short_desc         = "Python cross-version byte-code decompiler"
+web                = "https://github.com/rocky/python-uncompyle6/"
 
 # tracebacks in zip files are funky and not debuggable
 zip_safe = True
@@ -82,5 +82,5 @@ def read(*rnames):
     return open(os.path.join(srcdir, *rnames)).read()
 
 # Get info from files; set: long_description and VERSION
-long_description   = ( read("README.rst") + '\n' )
-exec(read('uncompyle6/version.py'))
+long_description   = ( read("README.rst") + "\n" )
+exec(read("uncompyle6/version.py"))

admin-tools/how-to-make-a-release.md

@@ -56,19 +56,21 @@
 
     $ . ./admin-tools/make-dist-older.sh
     $ git tag release-python-2.4-$VERSION
-
+	$ twine check dist/uncompyle6-$VERSION*
     $ . ./admin-tools/make-dist-newer.sh
-
-Goto https://github.com/rocky/python-uncompyle6/releases
+	$ twine check dist/uncompyle6-$VERSION*
 
 # Upload single package and look at Rst Formating
 
+	$ twine check dist/uncompyle6-${VERSION}*
     $ twine upload dist/uncompyle6-${VERSION}-py3.3.egg
 
 # Upload rest of versions
 
     $ twine upload dist/uncompyle6-${VERSION}*
 
+Goto https://github.com/rocky/python-uncompyle6/releases
+
 # Push tags:
 
     $ git push --tags

admin-tools/pyenv-newer-versions

@@ -5,4 +5,4 @@ if [[ $0 == ${BASH_SOURCE[0]} ]] ; then
     echo "This script should be *sourced* rather than run directly through bash"
     exit 1
 fi
-export PYVERSIONS='3.6.8 3.7.3 2.6.9 3.3.7 2.7.16 3.2.6 3.1.5 3.4.8'
+export PYVERSIONS='3.5.9 3.6.9 2.6.9 3.3.7 2.7.17 3.2.6 3.1.5 3.4.10 3.7.5'

admin-tools/setup-master.sh

@@ -1,5 +1,5 @@
 #!/bin/bash
-PYTHON_VERSION=3.6.8
+PYTHON_VERSION=3.7.5
 
 # FIXME put some of the below in a common routine
 function finish {

pytest/test_grammar.py

@@ -1,69 +1,81 @@
 import re
-from uncompyle6 import PYTHON_VERSION, PYTHON3, IS_PYPY # , PYTHON_VERSION
+from uncompyle6 import PYTHON_VERSION, PYTHON3, IS_PYPY  # , PYTHON_VERSION
 from uncompyle6.parser import get_python_parser, python_parser
 from uncompyle6.scanner import get_scanner
 
-def test_grammar():
 
+def test_grammar():
     def check_tokens(tokens, opcode_set):
         remain_tokens = set(tokens) - opcode_set
-        remain_tokens = set([re.sub(r'_\d+$','', t) for t in remain_tokens])
-        remain_tokens = set([re.sub('_CONT$','', t) for t in remain_tokens])
-        remain_tokens = set([re.sub('LOAD_CODE$','', t) for t in remain_tokens])
+        remain_tokens = set([re.sub(r"_\d+$", "", t) for t in remain_tokens])
+        remain_tokens = set([re.sub("_CONT$", "", t) for t in remain_tokens])
+        remain_tokens = set([re.sub("LOAD_CODE$", "", t) for t in remain_tokens])
         remain_tokens = set(remain_tokens) - opcode_set
-        assert remain_tokens == set([]), \
-            "Remaining tokens %s\n====\n%s" % (remain_tokens, p.dump_grammar())
+        assert remain_tokens == set([]), "Remaining tokens %s\n====\n%s" % (
+            remain_tokens,
+            p.dump_grammar(),
+        )
 
     p = get_python_parser(PYTHON_VERSION, is_pypy=IS_PYPY)
-    (lhs, rhs, tokens,
-     right_recursive, dup_rhs) = p.check_sets()
+    (lhs, rhs, tokens, right_recursive, dup_rhs) = p.check_sets()
 
     # We have custom rules that create the below
-    expect_lhs = set(['pos_arg', 'attribute'])
+    expect_lhs = set(["pos_arg"])
+
     if PYTHON_VERSION < 3.8:
-        expect_lhs.add('get_iter')
+        if PYTHON_VERSION < 3.7:
+            expect_lhs.add("attribute")
 
+        expect_lhs.add("get_iter")
+    else:
+        expect_lhs.add("async_with_as_stmt")
+        expect_lhs.add("async_with_stmt")
 
-    unused_rhs = set(['list', 'mkfunc',
-                      'mklambda',
-                      'unpack',])
+    unused_rhs = set(["list", "mkfunc", "mklambda", "unpack"])
 
-    expect_right_recursive = set([('designList',
-                                   ('store', 'DUP_TOP', 'designList'))])
+    expect_right_recursive = set([("designList", ("store", "DUP_TOP", "designList"))])
 
-    if PYTHON_VERSION < 3.7:
-        unused_rhs.add('call')
+    if PYTHON_VERSION <= 3.7:
+        unused_rhs.add("call")
 
     if PYTHON_VERSION > 2.6:
-        expect_lhs.add('kvlist')
-        expect_lhs.add('kv3')
-        unused_rhs.add('dict')
+        expect_lhs.add("kvlist")
+        expect_lhs.add("kv3")
+        unused_rhs.add("dict")
 
     if PYTHON3:
-        expect_lhs.add('load_genexpr')
+        expect_lhs.add("load_genexpr")
 
-        unused_rhs = unused_rhs.union(set("""
+        unused_rhs = unused_rhs.union(
+            set(
+                """
         except_pop_except generator_exp
-        """.split()))
+        """.split()
+            )
+        )
         if PYTHON_VERSION >= 3.0:
-            expect_lhs.add("annotate_arg")
-            expect_lhs.add("annotate_tuple")
-            unused_rhs.add("mkfunc_annotate")
+            if PYTHON_VERSION < 3.7:
+                expect_lhs.add("annotate_arg")
+                expect_lhs.add("annotate_tuple")
+                unused_rhs.add("mkfunc_annotate")
+
             unused_rhs.add("dict_comp")
             unused_rhs.add("classdefdeco1")
             unused_rhs.add("tryelsestmtl")
             if PYTHON_VERSION >= 3.5:
-                expect_right_recursive.add((('l_stmts',
-                                             ('lastl_stmt', 'come_froms', 'l_stmts'))))
+                expect_right_recursive.add(
+                    (("l_stmts", ("lastl_stmt", "come_froms", "l_stmts")))
+                )
                 pass
             elif 3.0 < PYTHON_VERSION < 3.3:
-                expect_right_recursive.add((('l_stmts',
-                                             ('lastl_stmt', 'COME_FROM', 'l_stmts'))))
+                expect_right_recursive.add(
+                    (("l_stmts", ("lastl_stmt", "COME_FROM", "l_stmts")))
+                )
                 pass
             pass
         pass
     else:
-        expect_lhs.add('kwarg')
+        expect_lhs.add("kwarg")
 
     assert expect_lhs == set(lhs)
 
@@ -73,9 +85,16 @@ def test_grammar():
 
     assert expect_right_recursive == right_recursive
 
-    expect_dup_rhs = frozenset([('COME_FROM',), ('CONTINUE',), ('JUMP_ABSOLUTE',),
-                                ('LOAD_CONST',),
-                                ('JUMP_BACK',), ('JUMP_FORWARD',)])
+    expect_dup_rhs = frozenset(
+        [
+            ("COME_FROM",),
+            ("CONTINUE",),
+            ("JUMP_ABSOLUTE",),
+            ("LOAD_CONST",),
+            ("JUMP_BACK",),
+            ("JUMP_FORWARD",),
+        ]
+    )
     reduced_dup_rhs = dict((k, dup_rhs[k]) for k in dup_rhs if k not in expect_dup_rhs)
     for k in reduced_dup_rhs:
         print(k, reduced_dup_rhs[k])
@@ -83,7 +102,7 @@ def test_grammar():
 
     s = get_scanner(PYTHON_VERSION, IS_PYPY)
     ignore_set = set(
-            """
+        """
             JUMP_BACK CONTINUE
             COME_FROM COME_FROM_EXCEPT
             COME_FROM_EXCEPT_CLAUSE
@@ -92,22 +111,33 @@ def test_grammar():
             LOAD_GENEXPR LOAD_ASSERT LOAD_SETCOMP LOAD_DICTCOMP LOAD_STR LOAD_CODE
             LAMBDA_MARKER
             RETURN_END_IF RETURN_END_IF_LAMBDA RETURN_VALUE_LAMBDA RETURN_LAST
-            """.split())
+            """.split()
+    )
     if 2.6 <= PYTHON_VERSION <= 2.7:
         opcode_set = set(s.opc.opname).union(ignore_set)
         if PYTHON_VERSION == 2.6:
             opcode_set.add("THEN")
         check_tokens(tokens, opcode_set)
     elif PYTHON_VERSION == 3.4:
-        ignore_set.add('LOAD_CLASSNAME')
-        ignore_set.add('STORE_LOCALS')
+        ignore_set.add("LOAD_CLASSNAME")
+        ignore_set.add("STORE_LOCALS")
         opcode_set = set(s.opc.opname).union(ignore_set)
         check_tokens(tokens, opcode_set)
 
+
 def test_dup_rule():
     import inspect
-    python_parser(PYTHON_VERSION, inspect.currentframe().f_code,
-                  is_pypy=IS_PYPY,
-                  parser_debug={
-                      'dups': True, 'transition': False, 'reduce': False,
-                      'rules': False, 'errorstack': None, 'context': True})
+
+    python_parser(
+        PYTHON_VERSION,
+        inspect.currentframe().f_code,
+        is_pypy=IS_PYPY,
+        parser_debug={
+            "dups": True,
+            "transition": False,
+            "reduce": False,
+            "rules": False,
+            "errorstack": None,
+            "context": True,
+        },
+    )

pytest/test_pysource.py

@@ -15,7 +15,7 @@ else:
     def iteritems(d):
         return d.iteritems()
 
-from uncompyle6.semantics.pysource import SourceWalker as SourceWalker
+from uncompyle6.semantics.pysource import (SourceWalker, deparse_code2str)
 
 def test_template_engine():
     s = StringIO()
@@ -185,3 +185,11 @@ def test_tables():
             assert arg == len(entry), (
                 "%s[%s] arg %d should be length of entry %d. Full entry: %s" %
                             (name, k, arg, len(entry), entry))
+
+def test_deparse_code2str():
+    def deparse_test(co):
+        "This is a docstring"
+        s = deparse_code2str(co, debug_opts={"asm": "after", "tree": True})
+        assert s
+        return
+    deparse_test(deparse_test.__code__)

pytest/validate.py

@@ -1,16 +1,20 @@
 # future
 from __future__ import print_function
+
 # std
 import os
 import difflib
 import subprocess
 import tempfile
 import functools
+
 # uncompyle6 / xdis
 from uncompyle6 import PYTHON_VERSION, PYTHON3, IS_PYPY, code_deparse
+
 # TODO : I think we can get xdis to support the dis api (python 3 version) by doing something like this there
 from xdis.bytecode import Bytecode
 from xdis.main import get_opcode
+
 opc = get_opcode(PYTHON_VERSION, IS_PYPY)
 Bytecode = functools.partial(Bytecode, opc=opc)
 import six
@@ -20,6 +24,7 @@ if PYTHON3:
 else:
     from StringIO import StringIO
 
+
 def _dis_to_text(co):
     return Bytecode(co).dis()
 
@@ -33,36 +38,32 @@ def print_diff(original, uncompyled):
     :param original: Text describing the original code object.
     :param uncompyled: Text describing the uncompyled code object.
     """
-    original_lines = original.split('\n')
-    uncompyled_lines = uncompyled.split('\n')
-    args = original_lines, uncompyled_lines, 'original', 'uncompyled'
+    original_lines = original.split("\n")
+    uncompyled_lines = uncompyled.split("\n")
+    args = original_lines, uncompyled_lines, "original", "uncompyled"
     try:
         from bs4 import BeautifulSoup
+
         diff = difflib.HtmlDiff().make_file(*args)
         diff = BeautifulSoup(diff, "html.parser")
         diff.select_one('table[summary="Legends"]').extract()
     except ImportError:
-        print('\nTo display diff highlighting run:\n    pip install BeautifulSoup4')
+        print("\nTo display diff highlighting run:\n    pip install BeautifulSoup4")
         diff = difflib.HtmlDiff().make_table(*args)
 
     with tempfile.NamedTemporaryFile(delete=False) as f:
-        f.write(str(diff).encode('utf-8'))
+        f.write(str(diff).encode("utf-8"))
 
     try:
         print()
-        html = subprocess.check_output([
-            'elinks',
-            '-dump',
-            '-no-references',
-            '-dump-color-mode',
-            '1',
-            f.name,
-        ]).decode('utf-8')
+        html = subprocess.check_output(
+            ["elinks", "-dump", "-no-references", "-dump-color-mode", "1", f.name]
+        ).decode("utf-8")
         print(html)
     except:
-        print('\nFor side by side diff install elinks')
+        print("\nFor side by side diff install elinks")
         diff = difflib.Differ().compare(original_lines, uncompyled_lines)
-        print('\n'.join(diff))
+        print("\n".join(diff))
     finally:
         os.unlink(f.name)
 
@@ -80,18 +81,19 @@ def are_instructions_equal(i1, i2):
 
     :return: True if the two instructions are approximately equal, otherwise False.
     """
-    result = (1 == 1
+    result = (
+        1 == 1
         and i1.opname == i2.opname
         and i1.opcode == i2.opcode
         and i1.arg == i2.arg
         # ignore differences due to code objects
         # TODO : Better way of ignoring address
-        and (i1.argval == i2.argval or '<code object' in str(i1.argval))
+        and (i1.argval == i2.argval or "<code object" in str(i1.argval))
         # TODO : Should probably recurse to check code objects
-        and (i1.argrepr == i2.argrepr or '<code object' in i1.argrepr)
+        and (i1.argrepr == i2.argrepr or "<code object" in i1.argrepr)
         and i1.offset == i2.offset
         # ignore differences in line numbers
-        #and i1.starts_line
+        # and i1.starts_line
         and i1.is_jump_target == i2.is_jump_target
     )
     return result
@@ -115,22 +117,21 @@ def are_code_objects_equal(co1, co2):
     return True
 
 
-def validate_uncompyle(text, mode='exec'):
+def validate_uncompyle(text, mode="exec"):
     """
     Validate decompilation of the given source code.
 
     :param text: Source to validate decompilation of.
     """
-    original_code = compile(text, '<string>', mode)
+    original_code = compile(text, "<string>", mode)
     original_dis = _dis_to_text(original_code)
     original_text = text
 
-    deparsed = code_deparse(original_code,
-                            out=six.StringIO(),
-                            version=PYTHON_VERSION,
-                            compile_mode=mode)
+    deparsed = code_deparse(
+        original_code, out=six.StringIO(), version=PYTHON_VERSION, compile_mode=mode
+    )
     uncompyled_text = deparsed.text
-    uncompyled_code = compile(uncompyled_text, '<string>', 'exec')
+    uncompyled_code = compile(uncompyled_text, "<string>", "exec")
 
     if not are_code_objects_equal(uncompyled_code, original_code):
 
@@ -138,15 +139,17 @@ def validate_uncompyle(text, mode='exec'):
 
         def output(text, dis):
             width = 60
-            return '\n\n'.join([
-                ' SOURCE CODE '.center(width, '#'),
-                text.strip(),
-                ' BYTECODE '.center(width, '#'),
-                dis
-            ])
+            return "\n\n".join(
+                [
+                    " SOURCE CODE ".center(width, "#"),
+                    text.strip(),
+                    " BYTECODE ".center(width, "#"),
+                    dis,
+                ]
+            )
 
         original = output(original_text, original_dis)
         uncompyled = output(uncompyled_text, uncompyled_dis)
         print_diff(original, uncompyled)
 
-        assert 'original' == 'uncompyled'
+        assert "original" == "uncompyled"

requirements-dev.txt

@@ -1,2 +1,4 @@
 flake8
 hypothesis<=3.0.0
+six
+pytest==3.2.5

setup.cfg

@@ -1,6 +1,6 @@
 [bdist_rpm]
 release = 1
-packager = Mysterie <kajusska@gmail.com>
+packager = rocky <rb@dustyfeet.com>
 doc_files = README
 #		CHANGES.txt
 #		USAGE.txt
@@ -8,4 +8,4 @@ doc_files = README
 #		examples/
 
 [bdist_wheel]
-universal=1
+# universal=1

setup.py

@@ -4,8 +4,8 @@ import sys
 """Setup script for the 'uncompyle6' distribution."""
 
 SYS_VERSION = sys.version_info[0:2]
-if not ((2, 6) <= SYS_VERSION  <= (3, 8)):
-    mess = "Python Release 2.6 .. 3.8 are supported in this code branch."
+if not ((2, 6) <= SYS_VERSION  <= (3, 9)):
+    mess = "Python Release 2.6 .. 3.9 are supported in this code branch."
     if ((2, 4) <= SYS_VERSION <= (2, 7)):
         mess += ("\nFor your Python, version %s, use the python-2.4 code/branch." %
                  sys.version[0:3])
@@ -32,6 +32,7 @@ setup(
        install_requires   = install_requires,
        license            = license,
        long_description   = long_description,
+       long_description_content_type = "text/x-rst",
        name               = modname,
        packages           = find_packages(),
        py_modules         = py_modules,

test/Makefile

@@ -1,5 +1,6 @@
 PHONY=check clean dist distclean test test-unit test-functional rmChangeLog clean_pyc nosetests \
-      check-bytecode-1  check-bytecode-1.3 check-bytecode-1.4 check-bytecode-1.5 \
+      check-bytecode-1.0 check-bytecode-1.1 check-bytecode-1.2 check-bytecode-1.3 \
+      check-bytecode-1 check-bytecode-1.4 check-bytecode-1.5 check-bytecode-1.6 \
       check-bytecode-2 check-bytecode-3 check-bytecode-3-short \
       check-bytecode-2.2 check-byteocde-2.3 check-bytecode-2.4 \
       check-short check-2.6 check-2.7 check-3.0 check-3.1 check-3.2 check-3.3 \
@@ -85,7 +86,7 @@ check-disasm:
 	$(PYTHON) dis-compare.py
 
 #: Check deparsing bytecode 1.x only
-check-bytecode-1: check-bytecode-1.4 check-bytecode-1.5
+check-bytecode-1: check-bytecode-1.0 check-bytecode-1.1 check-bytecode-1.2 check-bytecode-1.3 check-bytecode-1.4 check-bytecode-1.5 check-bytecode-1.6
 
 #: Check deparsing bytecode 2.x only
 check-bytecode-2:
@@ -98,8 +99,8 @@ check-bytecode-3:
 	$(PYTHON) test_pythonlib.py --bytecode-3.0 \
 	--bytecode-3.1 --bytecode-3.2 --bytecode-3.3 \
 	--bytecode-3.4 --bytecode-3.5 --bytecode-3.6 \
-        --bytecode-3.7 --bytecode-3.8 \
-	--bytecode-pypy3.2
+        --bytecode-3.7  \
+	--bytecode-pypy3.2 --bytecode-pypy3.6 --bytecode-3.8
 
 #: Check deparsing on selected bytecode 3.x
 check-bytecode-3-short:
@@ -109,6 +110,7 @@ check-bytecode-3-short:
 #: Check deparsing bytecode on all Python 2 and Python 3 versions
 check-bytecode: check-bytecode-3
 	$(PYTHON) test_pythonlib.py \
+	--bytecode-1.0 --bytecode-1.1 --bytecode-1.2 \
 	--bytecode-1.3 --bytecode-1.4 --bytecode-1.5 \
         --bytecode-2.2 --bytecode-2.3 --bytecode-2.4 \
 	--bytecode-2.1 --bytecode-2.2 --bytecode-2.3 --bytecode-2.4 \
@@ -122,6 +124,18 @@ check-bytecode-short: check-bytecode-3-short
         --bytecode-2.6 --bytecode-2.7 --bytecode-pypy2.7
 
 
+#: Check deparsing bytecode 1.0 only
+check-bytecode-1.0:
+	$(PYTHON) test_pythonlib.py --bytecode-1.0
+
+#: Check deparsing bytecode 1.1 only
+check-bytecode-1.1:
+	$(PYTHON) test_pythonlib.py --bytecode-1.1
+
+#: Check deparsing bytecode 1.2 only
+check-bytecode-1.2:
+	$(PYTHON) test_pythonlib.py --bytecode-1.2
+
 #: Check deparsing bytecode 1.3 only
 check-bytecode-1.3:
 	$(PYTHON) test_pythonlib.py --bytecode-1.3
@@ -134,6 +148,10 @@ check-bytecode-1.4:
 check-bytecode-1.5:
 	$(PYTHON) test_pythonlib.py --bytecode-1.5
 
+#: Check deparsing bytecode 1.6 only
+check-bytecode-1.6:
+	$(PYTHON) test_pythonlib.py --bytecode-1.6
+
 #: Check deparsing Python 2.1
 check-bytecode-2.1:
 	$(PYTHON) test_pythonlib.py --bytecode-2.1
@@ -314,8 +332,16 @@ pypy-2.7 5.0 5.3 6.0:
 pypy-3.2 2.4:
 	$(PYTHON) test_pythonlib.py --bytecode-pypy3.2 --verify
 
-#: PyPy 5.0.x with Python 3.6 ...
+#: PyPy 5.0.x with Python 3.6.1 ...
+check-bytecode-pypy3.6: 7.1
 7.1:
+	$(PYTHON) test_pythonlib.py --bytecode-pypy3.6-run --verify-run
+	$(PYTHON) test_pythonlib.py --bytecode-pypy3.6 --verify
+
+#: PyPy 5.0.x with Python 3.6.9
+check-bytecode-pypy3.6: 7.2
+7.2:
+	$(PYTHON) test_pythonlib.py --bytecode-pypy3.6-run --verify-run
 	$(PYTHON) test_pythonlib.py --bytecode-pypy3.6 --verify
 
 

test/add-test.py

@@ -2,22 +2,23 @@
 """ Trivial helper program to bytecompile and run an uncompile
 """
 import os, sys, py_compile
+
 assert len(sys.argv) >= 2
 version = sys.version[0:3]
-if sys.argv[1] == '--run':
-    suffix = '_run'
+if sys.argv[1] in ("--run", "-r"):
+    suffix = "_run"
     py_source = sys.argv[2:]
 else:
-    suffix = ''
+    suffix = ""
     py_source = sys.argv[1:]
 
 for path in py_source:
     short = os.path.basename(path)
-    if hasattr(sys, 'pypy_version_info'):
-        cfile =  "bytecode_pypy%s%s/%s" % (version, suffix, short) + 'c'
+    if hasattr(sys, "pypy_version_info"):
+        cfile = "bytecode_pypy%s%s/%s" % (version, suffix, short) + "c"
     else:
-        cfile =  "bytecode_%s%s/%s" % (version, suffix, short) + 'c'
+        cfile = "bytecode_%s%s/%s" % (version, suffix, short) + "c"
     print("byte-compiling %s to %s" % (path, cfile))
     py_compile.compile(path, cfile)
     if isinstance(version, str) or version >= (2, 6, 0):
-        os.system("../bin/uncompyle6 -a -t %s" % cfile)
+        os.system("../bin/uncompyle6 -a -T %s" % cfile)

test/simple_source/bug26/01_if_while_return.py

@@ -0,0 +1,19 @@
+# Issue #284 in Python 2.6
+# See https://github.com/rocky/python-uncompyle6/issues/284
+# Decompilation failed when return was the last statetement
+# in the while loop inside the if block
+
+# This code is RUNNABLE!
+
+def f1():
+    if True:
+        while True:
+            return 5
+
+def f2():
+    if True:
+        while 1:
+            return 6
+
+
+assert f1() == 5 and f2() == 6

test/simple_source/bug26/03_complex_and.py

@@ -14,4 +14,4 @@ for y in (1, 2, 10):
         expected = 3
     result.append(expected)
 
-assert result == [10, 2, 3]
+assert result == [3, 2, 3]

test/simple_source/bug30/02_try_except_except.py

@@ -7,3 +7,13 @@ def start_new_thread(function, args, kwargs={}):
         pass
     except:
         args()
+
+# Adapted from 3.0.1 code.py
+# Bug is again JUMP_FORWARD elimination compared
+# to earlier and later Pythons.
+def interact():
+    while 1:
+        try:
+            more = 1
+        except KeyboardInterrupt:
+            more = 0

test/simple_source/bug30/02_while1_if_while1.py

@@ -7,3 +7,8 @@ while 1:
             raise RuntimeError
     else:
         raise RuntimeError
+
+# Adapted from 3.0.1 cgi.py
+def _parseparam(s, end):
+    while end > 0 and s.count(''):
+        end = s.find(';')

test/simple_source/bug30/03_ifelse.py

@@ -0,0 +1,50 @@
+# Adapted from 3.0 base64
+# Problem was handling if/else which
+# needs to be like Python 2.6 (and not like 2.7 or 3.1)
+def main(args, f, func, sys):
+    """Small main program"""
+    if args and args[0] != '-':
+        func(f, sys.stdout.buffer)
+    else:
+        func(sys.stdin.buffer, sys.stdout.buffer)
+
+# From Python 3.0 _markupbase.py.
+#
+# The Problem was in the way "if"s are generated in 3.0 which is sort
+# of like a more optimized Python 2.6, with reduced extraneous jumps,
+# but still 2.6-ish and not 2.7- or 3.1-ish.
+def parse_marked_section(fn, i, rawdata, report=1):
+    if report:
+        j = 1
+        fn(rawdata[i: j])
+    return 10
+
+# From 3.0.1 _abcoll.py
+# Bug was in genexpr_func which doesn't have a JUMP_BACK but
+# in its gen_comp_body, we can use COME_FROM in its place.
+# As above omission of JUMPs is a feature of 3.0 that doesn't
+# seem to be in later versions (or earlier like 2.6).
+def __and__(self, other, Iterable):
+    if not isinstance(other, Iterable):
+        return NotImplemented
+    return self._from_iterable(value for value in other if value in self)
+
+# Adapted from 3.0.1 abc.py
+# Bug was in handling multiple COME_FROMs in return_if_stmt
+def __instancecheck__(subtype, subclass, cls):
+    if subtype:
+        if (cls and subclass):
+            return False
+
+
+# Adapted from 3.0.1 abc.py
+# Bug was rule in "jump_absolute_else" and disallowing
+# "else" to the wrong place.
+
+def _strptime(locale_time, found_zone, time):
+    for tz_values in locale_time:
+        if found_zone:
+            if (time and found_zone):
+                break
+            else:
+                break

test/simple_source/bug30/04_and_del.py

@@ -18,3 +18,13 @@ assert normpath(['.']) == []
 assert normpath(['a', 'b', '..']) == ['a']
 assert normpath(['a', 'b', '', 'c']) == ['a', 'b', 'c']
 assert normpath(['a', 'b', '.', '', 'c', '..']) == ['a', 'b']
+
+# Adapted from 3.0.1 cgi.py
+# Bug was in detecting "or" and "and" due to lack of PUSH/POP_IF instructions.
+def handle(format, html, text):
+    formatter = (format and html) or text
+    return formatter
+
+assert handle(False, False, True)
+assert not handle(True, False, False)
+assert handle(True, True, False)

test/simple_source/bug35/02_build_list_unpack.py

@@ -0,0 +1,17 @@
+a = 5
+x = [1, 2, 3]
+i = [(a,), x]
+j = [a, *x]
+
+def f1(a):
+    return a[0], a[1]
+
+def f2(b):
+    return len(b), b[0]+5, b[2]
+
+def f3(x, y):
+    return [1, *x, y]
+
+assert f1(i) == ((5,), x)
+assert f2(j) == (4, 10, 2)
+assert f3(x, a) == [1, 1, 2, 3, 5]

test/simple_source/bug37/01_assert2.py

@@ -0,0 +1,8 @@
+# Self-checking test.
+# Bug was in if transform not inverting expression
+# This file is RUNNABLE!
+def test_assert2(c):
+    if c < 2:
+        raise SyntaxError('Oops')
+
+test_assert2(5)

test/simple_source/bug37/03_ifelse_chained_for.py

@@ -0,0 +1,24 @@
+# From decompyle3/semantics/customize3.py
+# The bug is handling "for" loop inside the
+# chained compare ifelse
+def n_classdef3(a, b, c, l):
+    r = 1
+    if 3.0 <= a <= 3.2:
+        for n in l:
+            if b:
+                break
+            elif c:
+                r = 2
+            pass
+        pass
+    else:
+        r = 3
+        pass
+    return r
+
+assert n_classdef3(10, True, True, []) == 3
+assert n_classdef3(0, False, True, []) == 3
+assert n_classdef3(3.1, True, True, []) == 1
+assert n_classdef3(3.1, True, False, [1]) == 1
+assert n_classdef3(3.1, True, True, [2]) == 1
+assert n_classdef3(3.1, False, True, [3]) == 2

test/simple_source/bug38/01_extra_iter.py

@@ -0,0 +1,7 @@
+# Adapted from From 3.3 urllib/parse.py
+qs = "https://travis-ci.org/rocky/python-uncompyle6/builds/605260823?utm_medium=notification&utm_source=email"
+expect = ['https://travis-ci.org/rocky/python-uncompyle6/builds/605260823?utm_medium=notification', 'utm_source=email']
+
+# Should visually see that we don't add an extra iter() which is not technically wrong, just
+# unnecessary.
+assert expect == [s2 for s1 in qs.split('&') for s2 in s1.split(';')]

test/simple_source/bug38/01_for_continue.py

@@ -0,0 +1,5 @@
+# Bug is turning a JUMP_BACK for a CONTINUE so for has no JUMP_BACK.
+# Also there is no POP_BLOCK since there isn't anything in the loop.
+# In the future when we have better control flow, we might redo all of this.
+for i in range(2):
+    pass

test/simple_source/bug38/01_named_expr.py

@@ -0,0 +1,5 @@
+(x, y) = "foo", 0
+if x := __name__:
+    y = 1
+assert x == "__main__", "Walrus operator changes value"
+assert y == 1, "Walrus operator branch taken"

test/simple_source/bug38/04_for_no_jump_back.py

@@ -0,0 +1,17 @@
+# from mult_by_const/instruction.py
+# Bug in 3.8 was handling no JUMP_BACK in "for" loop. It is
+# in the "if" instead
+def instruction_sequence_value(instrs, a, b):
+    for instr in instrs:
+        if a:
+            a = 6
+        elif b:
+            return 0
+        pass
+
+    return a
+
+assert instruction_sequence_value([], True, True) == 1
+assert instruction_sequence_value([1], True, True) == 6
+assert instruction_sequence_value([1], False, True) == 0
+assert instruction_sequence_value([1], False, False) == False

test/simple_source/expression/02_true_divide.py

@@ -1,4 +1,4 @@
 # Tests BINARY_TRUE_DIVIDE and INPLACE_TRUE_DIVIDE
-from __future__ import division  # needed on 2.6 and 2.7
+from __future__ import division  # needed on 2.2 .. 2.7
 x = len(__file__) / 1
 x /= 1

test/simple_source/expression/14_mixed_expressions.py

@@ -0,0 +1,53 @@
+# Covers a large number of operators
+#
+# This code is RUNNABLE!
+
+import sys
+PYTHON_VERSION = sys.version_info[0] + (sys.version_info[1] / 10.0)
+
+assert PYTHON_VERSION >= 3.7
+
+# some floats (from 01_float.py)
+
+x = 1e300
+assert 0.0 == x * 0
+assert x * 1e300 == float("inf")
+assert str(float("inf") * 0.0) == "nan"
+assert str(float("-inf") * 0.0) == "nan"
+assert -1e300 * 1e300 == float("-inf")
+
+# Complex (adapted from 02_complex.py)
+y = 5j
+assert y ** 2 == -25
+y **= 3
+assert y == (-0-125j)
+
+
+# Tests BINARY_TRUE_DIVIDE and INPLACE_TRUE_DIVIDE (from 02_try_divide.py)
+x = 2
+assert 4 / x == 2
+
+x = 5
+assert x / 2 == 2.5
+x = 3
+x /= 2
+assert x == 1.5
+
+x = 2
+assert 4 // x == 2
+x = 7
+x //= 2
+assert x == 3
+
+x = 3
+assert x % 2 == 1
+x %= 2
+assert x == 1
+
+assert x << 2 == 4
+x <<= 3
+assert x == 8
+
+assert x >> 1 == 4
+x >>= 1
+assert x == 4

test/simple_source/expression/15_mixed_expressions.py

@@ -0,0 +1,55 @@
+# Covers a large number of operators
+#
+# This code is RUNNABLE!
+from __future__ import division  # needed on 2.2 .. 2.7
+
+import sys
+PYTHON_VERSION = sys.version_info[0] + (sys.version_info[1] / 10.0)
+
+# some floats (from 01_float.py)
+
+x = 1e300
+assert 0.0 == x * 0
+assert x * 1e300 == float("inf")
+if PYTHON_VERSION > 2.41:
+    assert str(float("inf") * 0.0) == "nan"
+else:
+    assert str(float("inf") * 0.0) == "-nan"
+assert -1e300 * 1e300 == float("-inf")
+
+# Complex (adapted from 02_complex.py)
+y = 5j
+assert y ** 2 == -25
+y **= 3
+assert y == (-0-125j)
+
+
+# Tests BINARY_TRUE_DIVIDE and INPLACE_TRUE_DIVIDE (from 02_try_divide.py)
+x = 2
+assert 4 / x == 2
+
+if PYTHON_VERSION >= 2.19:
+    x = 5
+    assert x / 2 == 2.5
+    x = 3
+    x /= 2
+    assert x == 1.5
+
+x = 2
+assert 4 // x == 2
+x = 7
+x //= 2
+assert x == 3
+
+x = 3
+assert x % 2 == 1
+x %= 2
+assert x == 1
+
+assert x << 2 == 4
+x <<= 3
+assert x == 8
+
+assert x >> 1 == 4
+x >>= 1
+assert x == 4

test/stdlib/runtests.sh

@@ -34,40 +34,57 @@ case $PYVERSION in
 	SKIP_TESTS=(
 	    [test_dis.py]=1   # We change line numbers - duh!
 	    [test_grp.py]=1      # Long test - might work Control flow?
-	    [test_pwd.py]=1 # Long test - might work? Control flow?
 	    [test_pep247.py]=1 # Long test - might work? Control flow?
+	    [test_pwd.py]=1 # Long test - might work? Control flow?
+	    [test_pyclbr.py]=1 # Investigate
+	    [test_pyexpat.py]=1 # Investigate
 	    [test_queue.py]=1 # Control flow?
-	    # [test_threading.py]=1 # Long test - works
+	    [test_re.py]=1 # try confused with try-else again
+	    [test_socketserver.py]=1 # -- test takes too long to run: 40 seconds
+	    [test_threading.py]=1 # Line numbers are expected to be different
+	    [test_thread.py]=1 # test takes too long to run: 36 seconds
+	    [test_trace.py]=1 # Long test - works
+	    [test_zipfile64.py]=1  # Runs ok but takes 204 seconds
 	)
 	;;
     2.5)
 	SKIP_TESTS=(
 	    [test_contextlib.py]=1 # Syntax error - look at
-	    [test_dis.py]=1   # We change line numbers - duh!
-	    [test_grammar.py]=1  # Too many stmts. Handle large stmts
-	    [test_grp.py]=1      # Long test - might work Control flow?
-	    [test_pdb.py]=1      # Line-number specific
+	    [test_dis.py]=1        # We change line numbers - duh!
+	    [test_grammar.py]=1    # Too many stmts. Handle large stmts
+	    [test_grp.py]=1        # Long test - might work Control flow?
+	    [test_pdb.py]=1        # Line-number specific
+	    [test_pep247.py]=1     # "assert xxx or .." not detected properly in check_hash_module()
+	    [test_pep352.py]=1     # Investigate
 	    [test_pwd.py]=1 # Long test - might work? Control flow?
+	    [test_pyclbr.py]=1 # Investigate
 	    [test_queue.py]=1 # Control flow?
-	    [test_re.py]=1 # Probably Control flow?
+	    [test_re.py]=1 # Possibly try confused with try-else again
+	    [test_struct.py]=1 # "if and" confused for if .. assert and
+	    [test_sys.py]=1 # try confused with try-else again; in test_current_frames()
+	    [test_tarfile.py]=1  # try confused with try-else again; top-level import
+	    [test_threading.py]=1 # Line numbers are expected to be different
+	    [test_thread.py]=1 # test takes too long to run: 36 seconds
 	    [test_trace.py]=1  # Line numbers are expected to be different
 	    [test_zipfile64.py]=1  # Runs ok but takes 204 seconds
 	)
 	;;
     2.6)
 	SKIP_TESTS=(
+	    [test_aepack.py]=1 # Fails on its own
+	    [test_codeccallbacks.py]=1  # Fails on its own
 	    [test_compile.py]=1  # Intermittent - sometimes works and sometimes doesn't
-	    [test_grammar.py]=1  # Need real flow control. "and" in side "or"
-	                         # "and" inside ifelse need to simulatenously work
+	    [test_exceptions.py]=1
+	    [test_generators.py]=1 # Investigate
 	    [test_grp.py]=1      # Long test - might work Control flow?
-	    [test_opcodes.py]=1
+	    [test_pep352.py]=1     # Investigate
+	    [test_pprint.py]=1
+	    [test_pyclbr.py]=1 # Investigate
 	    [test_pwd.py]=1 # Long test - might work? Control flow?
-	    [test_re.py]=1 # Probably Control flow?
-	    [test_queue.py]=1 # Control flow?
-	    [test_strftime.py]=1
 	    [test_trace.py]=1  # Line numbers are expected to be different
+	    [test_urllib2net.py]=1 # Fails on its own. May need interactive input
 	    [test_zipfile64.py]=1  # Skip Long test
-	    [test_zlib.py]=1  # Look at
+	    [test_zlib.py]=1  # Takes too long to run (more than 3 minutes 39 seconds)
 	    # .pyenv/versions/2.6.9/lib/python2.6/lib2to3/refactor.pyc
 	    # .pyenv/versions/2.6.9/lib/python2.6/pyclbr.pyc
 	    # .pyenv/versions/2.6.9/lib/python2.6/quopri.pyc -- look at ishex, is short
@@ -77,8 +94,6 @@ case $PYVERSION in
 	    # .pyenv/versions/2.6.9/lib/python2.6/tabnanny.pyc
 	    # .pyenv/versions/2.6.9/lib/python2.6/tarfile.pyc
 
-	    # Not getting set by bach below?
-	    [test_pprint.py]=1
 
 	)
 	if (( batch )) ; then
@@ -103,36 +118,38 @@ case $PYVERSION in
 	    [test_httplib.py]=1  # Ok, but POWER has problems with this
 	    [test_pdb.py]=1 # Ok, but POWER has problems with this
 
+	    [test_capi.py]=1
 	    [test_curses.py]=1  # Possibly fails on its own but not detected
+	    [test test_cmd_line.py]=1 # Takes too long, maybe hangs, or looking for interactive input?
 	    [test_dis.py]=1   # We change line numbers - duh!
 	    [test_doctest.py]=1 # Fails on its own
+	    [test_exceptions.py]=1
 	    [test_format.py]=1  # control flow. uncompyle2 does not have problems here
 	    [test_generators.py]=1  # control flow. uncompyle2 has problem here too
 	    [test_grammar.py]=1     # Too many stmts. Handle large stmts
+	    [test_grp.py]=1     # test takes to long, works interactively though
+	    [test_hashlib.py]=1 # Investigate
 	    [test_io.py]=1 # Test takes too long to run
 	    [test_ioctl.py]=1 # Test takes too long to run
-	    [test_itertools.py]=1 # Fix erroneous reduction to "conditional_true".
-	                          # See test/simple_source/bug27+/05_not_unconditional.py
+	    [test_long.py]=1
+	    [test_long_future.py]=1
+	    [test_math.py]=1
 	    [test_memoryio.py]=1 # FIX
-	    [test_multiprocessing.py]=1 # On uncompyle2, taks 24 secs
-	    [test_pep352.py]=1  # ?
-	    [test_posix.py]=1   # Bug in try-else detection inside test_initgroups()
-	                        # Deal with when we have better flow-control detection
+	    [test_modulefinder.py]=1 # FIX
+	    [test_multiprocessing.py]=1 # On uncompyle2, takes 24 secs
 	    [test_pwd.py]=1     # Takes too long
 	    [test_pty.py]=1
-	    [test_queue.py]=1   # Control flow?
-	    [test_re.py]=1      # Probably Control flow?
+	    [test_runpy.py]=1   # Long and fails on its own
 	    [test_select.py]=1  # Runs okay but takes 11 seconds
 	    [test_socket.py]=1  # Runs ok but takes 22 seconds
 	    [test_subprocess.py]=1 # Runs ok but takes 22 seconds
+	    [test_sys_setprofile.py]=1
 	    [test_sys_settrace.py]=1 # Line numbers are expected to be different
-	    [test_strtod.py]=1 # FIX
 	    [test_traceback.py]=1 # Line numbers change - duh.
-	    [test_types.py]=1     # try/else confusions
 	    [test_unicode.py]=1  # Too long to run 11 seconds
 	    [test_xpickle.py]=1 # Runs ok but takes 72 seconds
 	    [test_zipfile64.py]=1  # Runs ok but takes 204 seconds
-	    [test_zipimport.py]=1  # We can't distinguish try from try/else yet
+	    [test_zipimport.py]=1  # FIXME: improper try from try/else ?
         )
 	if (( batch )) ; then
 	    # Fails in crontab environment?
@@ -188,6 +205,7 @@ fi
 mkdir $TESTDIR || exit $?
 cp -r ${PYENV_ROOT}/versions/${PYVERSION}.${MINOR}/lib/python${PYVERSION}/test $TESTDIR
 cd $TESTDIR/test
+pyenv local $FULLVERSION
 export PYTHONPATH=$TESTDIR
 
 # Run tests
@@ -204,10 +222,14 @@ else
 fi
 
 typeset -i ALL_FILES_STARTTIME=$(date +%s)
+typeset -i skipped=0
 
 for file in $files; do
     # AIX bash doesn't grok [[ -v SKIP... ]]
-    [[ ${SKIP_TESTS[$file]} == 1 ]] && continue
+    if [[ ${SKIP_TESTS[$file]} == 1 ]] ; then
+	((skipped++))
+	continue
+    fi
 
     # If the fails *before* decompiling, skip it!
     typeset -i STARTTIME=$(date +%s)
@@ -241,7 +263,7 @@ for file in $files; do
     fi
     (( rc != 0 && allerrs++ ))
     if (( STOP_ONERROR && rc )) ; then
-	echo "** Ran $i tests before failure **"
+	echo "** Ran $i tests before failure. Skipped $skipped test for known failures. **"
 	exit $allerrs
     fi
 done
@@ -251,5 +273,5 @@ typeset -i ALL_FILES_ENDTIME=$(date +%s)
 
 printf "Ran $i unit-test files in "
 displaytime $time_diff
-
+echo "Skipped $skipped test for known failures."
 exit $allerrs

test/test_pyenvlib.py

@@ -28,64 +28,83 @@ from fnmatch import fnmatch
 from uncompyle6 import main, PYTHON3
 import xdis.magics as magics
 
-#----- configure this for your needs
+# ----- configure this for your needs
 
-python_versions = [v for v in magics.python_versions if
-                       re.match('^[0-9.]+$', v)]
+python_versions = [v for v in magics.python_versions if re.match("^[0-9.]+$", v)]
 
 # FIXME: we should remove Python versions that we don't support.
 # These include Jython, and Python bytecode changes pre release.
 
 TEST_VERSIONS = (
-               'pypy3-2.4.0', 'pypy-2.6.1',
-               'pypy-5.0.1', 'pypy-5.3.1', 'pypy3.5-5.7.1-beta',
-               'pypy3.5-5.9.0', 'pypy3.5-6.0.0',
-               'native') + tuple(python_versions)
+    "pypy3-2.4.0",
+    "pypy-2.6.1",
+    "pypy-5.0.1",
+    "pypy-5.3.1",
+    "pypy3.5-5.7.1-beta",
+    "pypy3.5-5.9.0",
+    "pypy3.5-6.0.0",
+    "pypy3.6-7.1.0",
+    "pypy3.6-7.1.1",
+    "pypy3.6-7.2.0",
+    "native",
+) + tuple(python_versions)
 
 
-target_base = '/tmp/py-dis/'
-lib_prefix = os.path.join(os.environ['HOME'], '.pyenv/versions')
+target_base = "/tmp/py-dis/"
+lib_prefix = os.path.join(os.environ["HOME"], ".pyenv/versions")
 
-PYC = ('*.pyc', )
-PYO = ('*.pyo', )
-PYOC = ('*.pyc', '*.pyo')
+PYC = ("*.pyc",)
+PYO = ("*.pyo",)
+PYOC = ("*.pyc", "*.pyo")
 
-#-----
+# -----
 
 test_options = {
     # name: (src_basedir, pattern, output_base_suffix)
-    'test': ('./test', PYOC, 'test'),
-    'max=': 200,
-    }
+    "test": ("./test", PYOC, "test"),
+    "max=": 200,
+}
 
 for vers in TEST_VERSIONS:
-    if vers.startswith('pypy'):
-        if vers.startswith('pypy3.'):
+    if vers.startswith("pypy"):
+        if vers.startswith("pypy3."):
             short_vers = vers[4:6]
         else:
             short_vers = vers[0:-2]
 
-        test_options[vers] = (os.path.join(lib_prefix, vers, 'lib_pypy'),
-                              PYC, 'python-lib'+short_vers)
+        test_options[vers] = (
+            os.path.join(lib_prefix, vers, "lib_pypy"),
+            PYC,
+            "python-lib" + short_vers,
+        )
     else:
-        if vers == 'native':
+        if vers == "native":
             short_vers = os.path.basename(sys.path[-1])
-            test_options[vers] = (sys.path[-1],
-                                  PYC, short_vers)
+            from xdis import PYTHON_VERSION
+            if PYTHON_VERSION > 3.0:
+                PYC = "*.cpython-%d.pyc" % int(PYTHON_VERSION * 10)
+            test_options[vers] = (sys.path[-1], PYC, short_vers)
         else:
             short_vers = vers[:3]
-            test_options[vers] = (os.path.join(lib_prefix, vers, 'lib', 'python'+short_vers),
-                                  PYC, 'python-lib'+short_vers)
+            test_options[vers] = (
+                os.path.join(lib_prefix, vers, "lib", "python" + short_vers),
+                PYC,
+                "python-lib" + short_vers,
+            )
 
-def do_tests(src_dir, patterns, target_dir, start_with=None,
-             do_verify=False, max_files=200):
 
+def do_tests(
+    src_dir, patterns, target_dir, start_with=None, do_verify=False, max_files=200
+):
     def visitor(files, dirname, names):
         files.extend(
-            [os.path.normpath(os.path.join(dirname, n))
-                 for n in names
-                    for pat in patterns
-                        if fnmatch(n, pat)])
+            [
+                os.path.normpath(os.path.join(dirname, n))
+                for n in names
+                for pat in patterns
+                if fnmatch(n, pat)
+            ]
+        )
 
     files = []
     cwd = os.getcwd()
@@ -93,10 +112,13 @@ def do_tests(src_dir, patterns, target_dir, start_with=None,
     if PYTHON3:
         for root, dirname, names in os.walk(os.curdir):
             files.extend(
-                [os.path.normpath(os.path.join(root, n))
-                     for n in names
-                        for pat in patterns
-                            if fnmatch(n, pat)])
+                [
+                    os.path.normpath(os.path.join(root, n))
+                    for n in names
+                    for pat in patterns
+                    if fnmatch(n, pat)
+                ]
+            )
             pass
         pass
     else:
@@ -108,26 +130,29 @@ def do_tests(src_dir, patterns, target_dir, start_with=None,
         try:
             start_with = files.index(start_with)
             files = files[start_with:]
-            print('>>> starting with file', files[0])
+            print(">>> starting with file", files[0])
         except ValueError:
             pass
 
     if len(files) > max_files:
-        files = [file for file in files if not 'site-packages' in file]
-        files = [file for file in files if not 'test' in file]
+        files = [file for file in files if not "site-packages" in file]
+        files = [file for file in files if not "test" in file]
         if len(files) > max_files:
             # print("Number of files %d - truncating to last 200" % len(files))
-            print("Number of files %d - truncating to first %s" %
-                  (len(files), max_files))
+            print(
+                "Number of files %d - truncating to first %s" % (len(files), max_files)
+            )
             files = files[:max_files]
 
     print(time.ctime())
-    (tot_files, okay_files, failed_files,
-     verify_failed_files) = main.main(src_dir, target_dir, files, [], do_verify=do_verify)
+    (tot_files, okay_files, failed_files, verify_failed_files) = main.main(
+        src_dir, target_dir, files, [], do_verify=do_verify
+    )
     print(time.ctime())
     return verify_failed_files + failed_files
 
-if __name__ == '__main__':
+
+if __name__ == "__main__":
     import getopt, sys
 
     do_coverage = do_verify = False
@@ -136,38 +161,46 @@ if __name__ == '__main__':
 
     test_options_keys = list(test_options.keys())
     test_options_keys.sort()
-    opts, args = getopt.getopt(sys.argv[1:], '',
-                               ['start-with=', 'verify', 'verify-run', 'syntax-verify',
-                                'max=', 'coverage', 'all', ] \
-                               + test_options_keys )
-    vers = ''
+    opts, args = getopt.getopt(
+        sys.argv[1:],
+        "",
+        [
+            "start-with=",
+            "verify",
+            "verify-run",
+            "syntax-verify",
+            "max=",
+            "coverage",
+            "all",
+        ]
+        + test_options_keys,
+    )
+    vers = ""
 
     for opt, val in opts:
-        if opt == '--verify':
-            do_verify = 'strong'
-        elif opt == '--syntax-verify':
-            do_verify = 'weak'
-        elif opt == '--verify-run':
-            do_verify = 'verify-run'
-        elif opt == '--coverage':
+        if opt == "--verify":
+            do_verify = "strong"
+        elif opt == "--syntax-verify":
+            do_verify = "weak"
+        elif opt == "--verify-run":
+            do_verify = "verify-run"
+        elif opt == "--coverage":
             do_coverage = True
-        elif opt == '--start-with':
+        elif opt == "--start-with":
             start_with = val
         elif opt[2:] in test_options_keys:
             triple = test_options[opt[2:]]
             vers = triple[-1]
             test_dirs.append(triple)
-        elif opt == '--max':
-            test_options['max='] = int(val)
-        elif opt == '--all':
-            vers = 'all'
+        elif opt == "--max":
+            test_options["max="] = int(val)
+        elif opt == "--all":
+            vers = "all"
             for val in test_options_keys:
                 test_dirs.append(test_options[val])
 
     if do_coverage:
-        os.environ['SPARK_PARSER_COVERAGE'] = (
-            '/tmp/spark-grammar-%s.cover' % vers
-            )
+        os.environ["SPARK_PARSER_COVERAGE"] = "/tmp/spark-grammar-%s.cover" % vers
 
     failed = 0
     for src_dir, pattern, target_dir in test_dirs:
@@ -175,8 +208,14 @@ if __name__ == '__main__':
             target_dir = os.path.join(target_base, target_dir)
             if os.path.exists(target_dir):
                 shutil.rmtree(target_dir, ignore_errors=1)
-            failed += do_tests(src_dir, pattern, target_dir, start_with,
-                               do_verify, test_options['max='])
+            failed += do_tests(
+                src_dir,
+                pattern,
+                target_dir,
+                start_with,
+                do_verify,
+                test_options["max="],
+            )
         else:
             print("### Path %s doesn't exist; skipping" % src_dir)
             pass

test/test_pythonlib.py

@@ -35,69 +35,93 @@ from uncompyle6 import PYTHON_VERSION
 from uncompyle6.main import main
 from fnmatch import fnmatch
 
+
 def get_srcdir():
     filename = os.path.normcase(os.path.dirname(__file__))
     return os.path.realpath(filename)
 
+
 src_dir = get_srcdir()
 
 
-#----- configure this for your needs
+# ----- configure this for your needs
 
-lib_prefix = '/usr/lib'
-#lib_prefix = [src_dir, '/usr/lib/', '/usr/local/lib/']
+lib_prefix = "/usr/lib"
+# lib_prefix = [src_dir, '/usr/lib/', '/usr/local/lib/']
 
-target_base = tempfile.mkdtemp(prefix='py-dis-')
+target_base = tempfile.mkdtemp(prefix="py-dis-")
 
-PY = ('*.py', )
-PYC = ('*.pyc', )
-PYO = ('*.pyo', )
-PYOC = ('*.pyc', '*.pyo')
+PY = ("*.py",)
+PYC = ("*.pyc",)
+PYO = ("*.pyo",)
+PYOC = ("*.pyc", "*.pyo")
 
 test_options = {
     # name:   (src_basedir, pattern, output_base_suffix, python_version)
-    'test':
-        ('test', PYC, 'test'),
-
-    'ok-2.6':    (os.path.join(src_dir, 'ok_lib2.6'),
-                 PYOC, 'ok-2.6', 2.6),
-
-    'ok-2.7':    (os.path.join(src_dir, 'ok_lib2.7'),
-                 PYOC, 'ok-2.7', 2.7),
-
-    'ok-3.2':    (os.path.join(src_dir, 'ok_lib3.2'),
-                 PYOC, 'ok-3.2', 3.2),
-
-    'base-2.7':  (os.path.join(src_dir, 'base_tests', 'python2.7'),
-                  PYOC, 'base_2.7', 2.7),
+    "test": ("test", PYC, "test"),
+    "ok-2.6": (os.path.join(src_dir, "ok_lib2.6"), PYOC, "ok-2.6", 2.6),
+    "ok-2.7": (os.path.join(src_dir, "ok_lib2.7"), PYOC, "ok-2.7", 2.7),
+    "ok-3.2": (os.path.join(src_dir, "ok_lib3.2"), PYOC, "ok-3.2", 3.2),
+    "base-2.7": (
+        os.path.join(src_dir, "base_tests", "python2.7"),
+        PYOC,
+        "base_2.7",
+        2.7,
+    ),
 }
 
-for  vers in (2.7, 3.4, 3.5, 3.6):
+for vers in (2.7, 3.4, 3.5, 3.6):
     pythonlib = "ok_lib%s" % vers
     key = "ok-%s" % vers
     test_options[key] = (os.path.join(src_dir, pythonlib), PYOC, key, vers)
     pass
 
-for  vers in (1.3, 1.4, 1.5,
-              2.1, 2.2, 2.3, 2.4, 2.5, 2.6, 2.7,
-              3.0, 3.1, 3.2, 3.3,
-              3.4, 3.5, 3.6, 3.7, 3.8, 'pypy3.2', 'pypy2.7', 'pypy3.6'):
+for vers in (
+    1.0,
+    1.1,
+    1.2,
+    1.3,
+    1.4,
+    1.5,
+    1.6,
+    2.1,
+    2.2,
+    2.3,
+    2.4,
+    2.5,
+    2.6,
+    2.7,
+    3.0,
+    3.1,
+    3.2,
+    3.3,
+    3.4,
+    3.5,
+    3.6,
+    3.7,
+    3.8,
+    "pypy3.2",
+    "pypy2.7",
+    "pypy3.6",
+):
     bytecode = "bytecode_%s" % vers
     key = "bytecode-%s" % vers
-    test_options[key] =  (bytecode, PYC, bytecode, vers)
+    test_options[key] = (bytecode, PYC, bytecode, vers)
     bytecode = "bytecode_%s_run" % vers
     key = "bytecode-%s-run" % vers
-    test_options[key] =  (bytecode, PYC, bytecode, vers)
+    test_options[key] = (bytecode, PYC, bytecode, vers)
     key = "%s" % vers
     pythonlib = "python%s" % vers
     if isinstance(vers, float) and vers >= 3.0:
-        pythonlib = os.path.join(pythonlib, '__pycache__')
-    test_options[key] =  (os.path.join(lib_prefix, pythonlib), PYOC, pythonlib, vers)
+        pythonlib = os.path.join(pythonlib, "__pycache__")
+    test_options[key] = (os.path.join(lib_prefix, pythonlib), PYOC, pythonlib, vers)
+
+# -----
 
-#-----
 
 def help():
-    print("""Usage-Examples:
+    print(
+        """Usage-Examples:
 
   # compile, decompyle and verify short tests for Python 2.7:
   test_pythonlib.py --bytecode-2.7 --verify --compile
@@ -107,18 +131,21 @@ def help():
 
   # decompile and verify known good python 2.7
   test_pythonlib.py --ok-2.7 --verify
-""")
+"""
+    )
     sys.exit(1)
 
 
 def do_tests(src_dir, obj_patterns, target_dir, opts):
-
     def file_matches(files, root, basenames, patterns):
         files.extend(
-            [os.path.normpath(os.path.join(root, n))
-                 for n in basenames
-                    for pat in patterns
-                        if fnmatch(n, pat)])
+            [
+                os.path.normpath(os.path.join(root, n))
+                for n in basenames
+                for pat in patterns
+                if fnmatch(n, pat)
+            ]
+        )
 
     files = []
     # Change directories so use relative rather than
@@ -127,11 +154,14 @@ def do_tests(src_dir, obj_patterns, target_dir, opts):
     cwd = os.getcwd()
     os.chdir(src_dir)
 
-    if opts['do_compile']:
-        compiled_version = opts['compiled_version']
+    if opts["do_compile"]:
+        compiled_version = opts["compiled_version"]
         if compiled_version and PYTHON_VERSION != compiled_version:
-            print("Not compiling: desired Python version is %s but we are running %s" %
-                  (compiled_version, PYTHON_VERSION), file=sys.stderr)
+            print(
+                "Not compiling: desired Python version is %s but we are running %s"
+                % (compiled_version, PYTHON_VERSION),
+                file=sys.stderr,
+            )
         else:
             for root, dirs, basenames in os.walk(src_dir):
                 file_matches(files, root, basenames, PY)
@@ -143,34 +173,36 @@ def do_tests(src_dir, obj_patterns, target_dir, opts):
             pass
         pass
 
-    for root, dirs, basenames in os.walk('.'):
+    for root, dirs, basenames in os.walk("."):
         # Turn root into a relative path
         dirname = root[2:]  # 2 = len('.') + 1
         file_matches(files, dirname, basenames, obj_patterns)
 
     if not files:
-        print("Didn't come up with any files to test! Try with --compile?",
-              file=sys.stderr)
+        print(
+            "Didn't come up with any files to test! Try with --compile?",
+            file=sys.stderr,
+        )
         exit(1)
 
     os.chdir(cwd)
     files.sort()
 
-    if opts['start_with']:
+    if opts["start_with"]:
         try:
-            start_with = files.index(opts['start_with'])
+            start_with = files.index(opts["start_with"])
             files = files[start_with:]
-            print('>>> starting with file', files[0])
+            print(">>> starting with file", files[0])
         except ValueError:
             pass
 
     print(time.ctime())
-    print('Source directory: ', src_dir)
-    print('Output directory: ', target_dir)
+    print("Source directory: ", src_dir)
+    print("Output directory: ", target_dir)
     try:
-        _, _, failed_files, failed_verify = \
-          main(src_dir, target_dir, files, [],
-               do_verify=opts['do_verify'])
+        _, _, failed_files, failed_verify = main(
+            src_dir, target_dir, files, [], do_verify=opts["do_verify"]
+        )
         if failed_files != 0:
             sys.exit(2)
         elif failed_verify != 0:
@@ -179,71 +211,81 @@ def do_tests(src_dir, obj_patterns, target_dir, opts):
     except (KeyboardInterrupt, OSError):
         print()
         sys.exit(1)
-    if test_opts['rmtree']:
+    if test_opts["rmtree"]:
         parent_dir = os.path.dirname(target_dir)
         print("Everything good, removing %s" % parent_dir)
         shutil.rmtree(parent_dir)
 
-if __name__ == '__main__':
+
+if __name__ == "__main__":
     test_dirs = []
     checked_dirs = []
     start_with = None
 
     test_options_keys = list(test_options.keys())
     test_options_keys.sort()
-    opts, args = getopt.getopt(sys.argv[1:], '',
-                               ['start-with=', 'verify', 'verify-run',
-                                'syntax-verify', 'all',
-                                'compile', 'coverage',
-                                'no-rm'] \
-                               + test_options_keys )
-    if not opts: help()
+    opts, args = getopt.getopt(
+        sys.argv[1:],
+        "",
+        [
+            "start-with=",
+            "verify",
+            "verify-run",
+            "syntax-verify",
+            "all",
+            "compile",
+            "coverage",
+            "no-rm",
+        ]
+        + test_options_keys,
+    )
+    if not opts:
+        help()
 
     test_opts = {
-        'do_compile': False,
-        'do_verify': False,
-        'start_with': None,
-        'rmtree' : True,
-        'coverage' : False
-        }
+        "do_compile": False,
+        "do_verify": False,
+        "start_with": None,
+        "rmtree": True,
+        "coverage": False,
+    }
 
     for opt, val in opts:
-        if opt == '--verify':
-            test_opts['do_verify'] = 'strong'
-        elif opt == '--syntax-verify':
-            test_opts['do_verify'] = 'weak'
-        elif opt == '--verify-run':
-            test_opts['do_verify'] = 'verify-run'
-        elif opt == '--compile':
-            test_opts['do_compile'] = True
-        elif opt == '--start-with':
-            test_opts['start_with'] = val
-        elif opt == '--no-rm':
-            test_opts['rmtree'] = False
+        if opt == "--verify":
+            test_opts["do_verify"] = "strong"
+        elif opt == "--syntax-verify":
+            test_opts["do_verify"] = "weak"
+        elif opt == "--verify-run":
+            test_opts["do_verify"] = "verify-run"
+        elif opt == "--compile":
+            test_opts["do_compile"] = True
+        elif opt == "--start-with":
+            test_opts["start_with"] = val
+        elif opt == "--no-rm":
+            test_opts["rmtree"] = False
         elif opt[2:] in test_options_keys:
             test_dirs.append(test_options[opt[2:]])
-        elif opt == '--all':
+        elif opt == "--all":
             for val in test_options_keys:
                 test_dirs.append(test_options[val])
-        elif opt == '--coverage':
-            test_opts['coverage'] = True
+        elif opt == "--coverage":
+            test_opts["coverage"] = True
         else:
             help()
             pass
         pass
 
-    if test_opts['coverage']:
-        os.environ['SPARK_PARSER_COVERAGE'] = (
-            '/tmp/spark-grammar-python-lib%s.cover' % test_dirs[0][-1]
-            )
+    if test_opts["coverage"]:
+        os.environ["SPARK_PARSER_COVERAGE"] = (
+            "/tmp/spark-grammar-python-lib%s.cover" % test_dirs[0][-1]
+        )
 
     last_compile_version = None
     for src_dir, pattern, target_dir, compiled_version in test_dirs:
         if os.path.isdir(src_dir):
             checked_dirs.append([src_dir, pattern, target_dir])
         else:
-            print("Can't find directory %s. Skipping" % src_dir,
-                  file=sys.stderr)
+            print("Can't find directory %s. Skipping" % src_dir, file=sys.stderr)
             continue
         last_compile_version = compiled_version
         pass
@@ -252,7 +294,7 @@ if __name__ == '__main__':
         print("No directories found to check", file=sys.stderr)
         sys.exit(1)
 
-    test_opts['compiled_version'] = last_compile_version
+    test_opts["compiled_version"] = last_compile_version
 
     for src_dir, pattern, target_dir in checked_dirs:
         target_dir = os.path.join(target_base, target_dir)

uncompyle6/bin/uncompile.py

@@ -46,10 +46,12 @@ Options:
   --help        show this message
 
 Debugging Options:
-  --asm     | -a  include byte-code       (disables --verify)
-  --grammar | -g  show matching grammar
-  --tree    | -t  include syntax tree     (disables --verify)
-  --tree++        add template rules to --tree when possible
+  --asm     | -a        include byte-code       (disables --verify)
+  --grammar | -g        show matching grammar
+  --tree={before|after}
+  -t {before|after}     include syntax before (or after) tree transformation
+                        (disables --verify)
+  --tree++ | -T         add template rules to --tree=before when possible
 
 Extensions of generated files:
   '.pyc_dis' '.pyo_dis'   successfully decompiled (and verified if --verify)
@@ -89,7 +91,7 @@ def main_bin():
     try:
         opts, pyc_paths = getopt.getopt(sys.argv[1:], 'hac:gtTdrVo:p:',
                                     'help asm compile= grammar linemaps recurse '
-                                    'timestamp tree tree+ '
+                                    'timestamp tree= tree+ '
                                     'fragments verify verify-run version '
                                     'syntax-verify '
                                     'showgrammar encoding='.split(' '))
@@ -119,10 +121,19 @@ def main_bin():
             options['showasm'] = 'after'
             options['do_verify'] = None
         elif opt in ('--tree', '-t'):
-            options['showast'] = True
+            if 'showast' not in options:
+                options['showast'] = {}
+            if val == 'before':
+                options['showast'][val] = True
+            elif val == 'after':
+                options['showast'][val] = True
+            else:
+                options['showast']['before'] = True
             options['do_verify'] = None
         elif opt in ('--tree+', '-T'):
-            options['showast'] = 'Full'
+            if 'showast' not in options:
+                options['showast'] = {}
+            options['showast']['Full'] = True
             options['do_verify'] = None
         elif opt in ('--grammar', '-g'):
             options['showgrammar'] = True

uncompyle6/disas.py

@@ -34,12 +34,11 @@ from __future__ import print_function
 import sys
 from collections import deque
 
-import uncompyle6
-
 from xdis.code import iscode
 from xdis.load import check_object_path, load_module
 from uncompyle6.scanner import get_scanner
 
+
 def disco(version, co, out=None, is_pypy=False):
     """
     diassembles and deparses a given code block 'co'
@@ -49,10 +48,9 @@ def disco(version, co, out=None, is_pypy=False):
 
     # store final output stream for case of error
     real_out = out or sys.stdout
-    print('# Python %s' % version, file=real_out)
+    print("# Python %s" % version, file=real_out)
     if co.co_filename:
-        print('# Embedded file name: %s' % co.co_filename,
-              file=real_out)
+        print("# Embedded file name: %s" % co.co_filename, file=real_out)
 
     scanner = get_scanner(version, is_pypy=is_pypy)
 
@@ -63,10 +61,12 @@ def disco(version, co, out=None, is_pypy=False):
 def disco_loop(disasm, queue, real_out):
     while len(queue) > 0:
         co = queue.popleft()
-        if co.co_name != '<module>':
-            print('\n# %s line %d of %s' %
-                      (co.co_name, co.co_firstlineno, co.co_filename),
-                      file=real_out)
+        if co.co_name != "<module>":
+            print(
+                "\n# %s line %d of %s"
+                % (co.co_name, co.co_firstlineno, co.co_filename),
+                file=real_out,
+            )
         tokens, customize = disasm(co)
         for t in tokens:
             if iscode(t.pattr):
@@ -77,6 +77,7 @@ def disco_loop(disasm, queue, real_out):
             pass
         pass
 
+
 # def disassemble_fp(fp, outstream=None):
 #     """
 #     disassemble Python byte-code from an open file
@@ -90,6 +91,7 @@ def disco_loop(disasm, queue, real_out):
 #         disco(version, co, outstream, is_pypy=is_pypy)
 #     co = None
 
+
 def disassemble_file(filename, outstream=None):
     """
     disassemble Python byte-code file (.pyc)
@@ -98,8 +100,7 @@ def disassemble_file(filename, outstream=None):
     try to find the corresponding compiled object.
     """
     filename = check_object_path(filename)
-    (version, timestamp, magic_int, co, is_pypy,
-     source_size) = load_module(filename)
+    (version, timestamp, magic_int, co, is_pypy, source_size) = load_module(filename)
     if type(co) == list:
         for con in co:
             disco(version, con, outstream)
@@ -107,6 +108,7 @@ def disassemble_file(filename, outstream=None):
         disco(version, co, outstream, is_pypy=is_pypy)
     co = None
 
+
 def _test():
     """Simple test program to disassemble a file."""
     argc = len(sys.argv)

uncompyle6/main.py

@@ -45,10 +45,21 @@ def _get_outstream(outfile):
         return open(outfile, mode='w', encoding='utf-8')
 
 def decompile(
-        bytecode_version, co, out=None, showasm=None, showast=False,
-        timestamp=None, showgrammar=False, source_encoding=None, code_objects={},
-        source_size=None, is_pypy=None, magic_int=None,
-        mapstream=None, do_fragments=False):
+    bytecode_version,
+    co,
+    out=None,
+    showasm=None,
+    showast={},
+    timestamp=None,
+    showgrammar=False,
+    source_encoding=None,
+    code_objects={},
+    source_size=None,
+    is_pypy=None,
+    magic_int=None,
+    mapstream=None,
+    do_fragments=False,
+):
     """
     ingests and deparses a given code block 'co'
 
@@ -294,7 +305,7 @@ def main(in_base, out_base, compiled_files, source_files, outfile=None,
         #     failed_files += 1
         #     if current_outfile:
         #         outstream.close()
-        #         os.rename(current_outfile, current_outfile + '_failed')
+        #         os.rename(current_outfile, current_outfile + "_failed")
         #     else:
         #         sys.stderr.write("\n# %s" % sys.exc_info()[1])
         #         sys.stderr.write("\n# Can't uncompile %s\n" % infile)

uncompyle6/parser.py

@@ -628,12 +628,30 @@ def get_python_parser(
 
     if version < 3.0:
         if version < 2.2:
+            if version == 1.0:
+                import uncompyle6.parsers.parse10 as parse10
+                if compile_mode == 'exec':
+                    p = parse10.Python10Parser(debug_parser)
+                else:
+                    p = parse10.Python01ParserSingle(debug_parser)
+            elif version == 1.1:
+                import uncompyle6.parsers.parse11 as parse11
+                if compile_mode == 'exec':
+                    p = parse11.Python11Parser(debug_parser)
+                else:
+                    p = parse11.Python11ParserSingle(debug_parser)
+            if version == 1.2:
+                import uncompyle6.parsers.parse12 as parse12
+                if compile_mode == 'exec':
+                    p = parse12.Python12Parser(debug_parser)
+                else:
+                    p = parse12.Python12ParserSingle(debug_parser)
             if version == 1.3:
                 import uncompyle6.parsers.parse13 as parse13
                 if compile_mode == 'exec':
-                    p = parse13.Python14Parser(debug_parser)
+                    p = parse13.Python13Parser(debug_parser)
                 else:
-                    p = parse13.Python14ParserSingle(debug_parser)
+                    p = parse13.Python13ParserSingle(debug_parser)
             elif version == 1.4:
                 import uncompyle6.parsers.parse14 as parse14
                 if compile_mode == 'exec':
@@ -646,6 +664,12 @@ def get_python_parser(
                     p = parse15.Python15Parser(debug_parser)
                 else:
                     p = parse15.Python15ParserSingle(debug_parser)
+            elif version == 1.6:
+                import uncompyle6.parsers.parse16 as parse16
+                if compile_mode == 'exec':
+                    p = parse16.Python16Parser(debug_parser)
+                else:
+                    p = parse16.Python16ParserSingle(debug_parser)
             elif version == 2.1:
                 import uncompyle6.parsers.parse21 as parse21
                 if compile_mode == 'exec':

uncompyle6/parsers/parse10.py

@@ -0,0 +1,25 @@
+#  Copyright (c) 2019 Rocky Bernstein
+
+from spark_parser import DEFAULT_DEBUG as PARSER_DEFAULT_DEBUG
+from uncompyle6.parser import PythonParserSingle
+from uncompyle6.parsers.parse11 import Python11Parser
+
+
+class Python10Parser(Python11Parser):
+    def __init__(self, debug_parser=PARSER_DEFAULT_DEBUG):
+        super(Python11Parser, self).__init__(debug_parser)
+        self.customized = {}
+
+
+class Python10ParserSingle(Python10Parser, PythonParserSingle):
+    pass
+
+
+if __name__ == "__main__":
+    # Check grammar
+    p = Python10Parser()
+    p.check_grammar()
+    p.dump_grammar()
+
+# local variables:
+# tab-width: 4

uncompyle6/parsers/parse11.py

@@ -0,0 +1,25 @@
+#  Copyright (c) 2019 Rocky Bernstein
+
+from spark_parser import DEFAULT_DEBUG as PARSER_DEFAULT_DEBUG
+from uncompyle6.parser import PythonParserSingle
+from uncompyle6.parsers.parse12 import Python12Parser
+
+
+class Python11Parser(Python12Parser):
+    def __init__(self, debug_parser=PARSER_DEFAULT_DEBUG):
+        super(Python12Parser, self).__init__(debug_parser)
+        self.customized = {}
+
+
+class Python11ParserSingle(Python11Parser, PythonParserSingle):
+    pass
+
+
+if __name__ == "__main__":
+    # Check grammar
+    p = Python12Parser()
+    p.check_grammar()
+    p.dump_grammar()
+
+# local variables:
+# tab-width: 4

uncompyle6/parsers/parse12.py

@@ -0,0 +1,25 @@
+#  Copyright (c) 2019 Rocky Bernstein
+
+from spark_parser import DEFAULT_DEBUG as PARSER_DEFAULT_DEBUG
+from uncompyle6.parser import PythonParserSingle
+from uncompyle6.parsers.parse13 import Python13Parser
+
+
+class Python12Parser(Python13Parser):
+    def __init__(self, debug_parser=PARSER_DEFAULT_DEBUG):
+        super(Python12Parser, self).__init__(debug_parser)
+        self.customized = {}
+
+
+class Python12ParserSingle(Python12Parser, PythonParserSingle):
+    pass
+
+
+if __name__ == "__main__":
+    # Check grammar
+    p = Python12Parser()
+    p.check_grammar()
+    p.dump_grammar()
+
+# local variables:
+# tab-width: 4

uncompyle6/parsers/parse16.py

@@ -0,0 +1,46 @@
+#  Copyright (c) 2019 Rocky Bernstein
+
+from spark_parser import DEFAULT_DEBUG as PARSER_DEFAULT_DEBUG
+from uncompyle6.parser import PythonParserSingle, nop_func
+from uncompyle6.parsers.parse21 import Python21Parser
+
+class Python16Parser(Python21Parser):
+
+    def __init__(self, debug_parser=PARSER_DEFAULT_DEBUG):
+        super(Python16Parser, self).__init__(debug_parser)
+        self.customized = {}
+
+    def p_import16(self, args):
+        """
+        import      ::= filler IMPORT_NAME STORE_FAST
+        import      ::= filler IMPORT_NAME STORE_NAME
+
+        import_from ::= filler IMPORT_NAME importlist
+        import_from ::= filler filler IMPORT_NAME importlist POP_TOP
+
+        importlist  ::= importlist IMPORT_FROM
+        importlist  ::= IMPORT_FROM
+        """
+
+    def customize_grammar_rules(self, tokens, customize):
+        super(Python16Parser, self).customize_grammar_rules(tokens, customize)
+        for i, token in enumerate(tokens):
+            opname = token.kind
+            opname_base = opname[:opname.rfind('_')]
+
+            if opname_base == 'UNPACK_LIST':
+                self.addRule("store ::= unpack_list", nop_func)
+
+
+
+class Python16ParserSingle(Python16Parser, PythonParserSingle):
+    pass
+
+if __name__ == '__main__':
+    # Check grammar
+    p = Python15Parser()
+    p.check_grammar()
+    p.dump_grammar()
+
+# local variables:
+# tab-width: 4