Get ready for release 3.2.6

Add test for last change

.circleci/config.yml

@@ -42,7 +42,7 @@ jobs:
     # This is based on your 1.0 configuration file or project settings
     - run:
         working_directory: ~/rocky/python-uncompyle6
-        command: pyenv install 2.4.6 && pyenv local 2.4.6 && pyenv rehash && easy_install nose && pyenv rehash
+        command: pyenv local 2.7.11 && pyenv rehash && pip install virtualenv && pip install nose && pip install pep8 && pip install six && pyenv rehash
     # Dependencies
     #   This would typically go in either a build or a build-and-test job when using workflows
     # Restore the dependency cache
@@ -55,7 +55,9 @@ jobs:
         # Any branch if there are none on the default branch - this should be unnecessary if you have your default branch configured correctly
         - v1-dep-
     # This is based on your 1.0 configuration file or project settings
-    - run: easy_install spark_parser==1.8.5 && easy_install xdis==3.8.4
+    - run: pip install --upgrade setuptools
+    - run: pip install -e .
+    - run: pip install pytest==3.2.5 hypothesis==3.0.0
     # Save dependency cache
     - save_cache:
         key: v1-dep-{{ .Branch }}-{{ epoch }}
@@ -73,8 +75,8 @@ jobs:
     # Test
     #   This would typically be a build job when using workflows, possibly combined with build
     # This is based on your 1.0 configuration file or project settings
-    - run: python ./setup.py develop && make check-2.4
-    - run: cd ./test/stdlib && pyenv local 2.4.6 && bash ./runtests.sh 'test_[p-z]*.py'
+    - run: python ./setup.py develop && make check-2.7
+    - run: cd ./test/stdlib && pyenv local 2.7.11 && bash ./runtests.sh 'test_[p-z]*.py'
     # Teardown
     #   If you break your build into multiple jobs with workflows, you will probably want to do the parts of this that are relevant in each
     # Save test results

.travis.yml

@@ -1,9 +1,16 @@
 language: python
 
-sudo: false
-
 python:
-  - '2.7' # this is a cheat here because travis doesn't do 2.4-2.6
+  - '3.5'
+  - '2.7'
+  - '2.6'
+  - '3.4'
+  - '3.6'
+
+matrix:
+  include:
+    - python: '3.7'
+      dist: xenial  # required for Python >= 3.7 (travis-ci/travis-ci#9069)
 
 install:
 - pip install -e .

HOW-TO-REPORT-A-BUG.md

@@ -1,6 +1,21 @@
-# How to report a Bug
+<!-- markdown-toc start - Don't edit this section. Run M-x markdown-toc-refresh-toc -->
+**Table of Contents**
 
-## The difficulty of the problem
+- [The difficulty of the problem](#the-difficulty-of-the-problem)
+- [Is it really a bug?](#is-it-really-a-bug)
+    - [Do you have valid bytecode?](#do-you-have-valid-bytecode)
+    - [Semantic equivalence vs. exact source code](#semantic-equivalence-vs-exact-source-code)
+- [What to send (minimum requirements)](#what-to-send-minimum-requirements)
+- [What to send (additional helpful information)](#what-to-send-additional-helpful-information)
+    - [But I don't *have* the source code!](#but-i-dont-have-the-source-code)
+    - [But I don't *have* the source code and am incapable of figuring how how to do a hand disassembly!](#but-i-dont-have-the-source-code-and-am-incapable-of-figuring-how-how-to-do-a-hand-disassembly)
+- [Narrowing the problem](#narrowing-the-problem)
+- [Karma](#karma)
+- [Confidentiality of Bug Reports](#confidentiality-of-bug-reports)
+- [Ethics](#ethics)
+
+<!-- markdown-toc end -->
+# The difficulty of the problem
 
 This decompiler is a constant work in progress: Python keeps
 changing, and so does its code generation.
@@ -41,10 +56,10 @@ bugs you may have an interest in. If you require decompiling bytecode
 immediately, consider using a decompilation service, listed further
 down in this document.
 
-## Is it really a bug?
+# Is it really a bug?
 
 
-### Do you have valid bytecode?
+## Do you have valid bytecode?
 
 As mentioned in README.rst, this project doesn't handle obfuscated
 code. See README.rst for suggestions for how to remove some kinds of
@@ -55,11 +70,11 @@ Python comes with a disassembly module called `dis`. A prerequisite
 module for this package, `xdis` has a cross-python version
 disassembler called `pydisasm`.
 
-### Semantic equivalence vs. exact source code
+## Semantic equivalence vs. exact source code
 
 Consider how Python compiles something like "(x*y) + 5". Early on
 Python creates an "abstract syntax tree" (AST) for this. And this is
-"abstract" in the sense that unimportant, redundant or unnecceary
+"abstract" in the sense that unimportant, redundant or unnecessary
 items have been removed. Here, this means that any notion that you
 wrote "x+y" in parenthesis is lost, since in this context they are
 unneeded. Also lost is the fact that the multiplication didn't have
@@ -132,7 +147,7 @@ Python will eliminate the entire "if" statement.
 So just because the text isn't the same, does not
 necessarily mean there's a bug.
 
-## What to send (minimum requirements)
+# What to send (minimum requirements)
 
 The basic requirement is pretty simple:
 
@@ -146,7 +161,7 @@ sending is too large.
 
 Also try to narrow the bug. See below.
 
-## What to send (additional helpful information)
+# What to send (additional helpful information)
 
 Some kind folks also give the invocation they used and the output
 which usually includes an error message produced. This is
@@ -159,7 +174,7 @@ provide the input command and the output from that, please give:
 * Python interpreter version used
 
 
-### But I don't *have* the source code!
+## But I don't *have* the source code!
 
 Sure, I get it. No problem. There is Python assembly code on parse
 errors, so simply by hand decompile that. To get a full disassembly,
@@ -167,7 +182,7 @@ use `pydisasm` from the [xdis](https://pypi.python.org/pypi/xdis)
 package. Opcodes are described in the documentation for
 the [dis](https://docs.python.org/3.6/library/dis.html) module.
 
-### But I don't *have* the source code and am incapable of figuring how how to do a hand disassembly!
+### But I don't *have* the source code and am incapable of figuring how to do a hand disassembly!
 
 Well, you could learn. No one is born into this world knowing how to
 disassemble Python bytecode. And as Richard Feynman once said, "What
@@ -179,7 +194,7 @@ Compilers](http://www.crazy-compilers.com/decompyle/) offers a
 byte-code decompiler service for versions of Python up to 2.6. (If
 there are others around let me know and I'll list them here.)
 
-## Narrowing the problem
+# Narrowing the problem
 
 I don't need or want the entire source code base for the file(s) or
 module(s) can't be decompiled. I just need those file(s) or module(s).
@@ -197,22 +212,53 @@ what doesn't. That is useful. Or maybe the same file will decompile
 properly on a neighboring version of Python. That is helpful too.
 
 In sum, the more you can isolate or narrow the problem, the more
-likley the problem will be fixed and fixed sooner.
+likely the problem will be fixed and fixed sooner.
 
-## Confidentiality of Bug Reports
+# Karma
+
+I realize that following the instructions given herein puts a bit of
+burden on the bug reporter. In my opinion, this is justified as
+attempts to balance somewhat the burden and effort needed to fix the
+bug and the attempts to balance number of would-be bug reporters with
+the number of bug fixers. Better bug reporters are more likely to move
+in the category of bug fixers.
+
+The barrier to reporting a big is pretty small: all you really need is
+a github account, and the ability to type something after clicking
+some buttons. So the reality is that many people just don't bother to
+read these instructions, let alone follow it to any simulacrum.
+
+And the reality is also that bugs sometimes get fixed even though
+these instructions are not followed.
+
+So one factors I may take into consideration is the bug reporter's karma.
+
+* Have you demonstrably contributed to open source? I may look at your
+  github profile to see what contributions you have made, how popular
+  those contributions are, or how popular you are.
+* How appreciative are you? Have you starred this project that you are
+  seeking help from? Have you starred _any_ github project? And the above
+  two kind of feed into ...
+* Attitude. Some people feel that they are doing me and the world a
+  great favor by just pointing out that there is a problem whose solution
+  would greatly benefit them. Perhaps this is why they feel that
+  instructions are not to be followed by them, nor any need for
+  showing evidence gratitude when help is offered them.
+
+# Confidentiality of Bug Reports
 
 When you report a bug, you are giving up confidentiality to the source
 code and the byte code. However, I would imagine that if you have
 narrowed the problem sufficiently, confidentiality of the little that
 remains would not be an issue.
 
-However feel free to remove any commments, and modify variable names
+However feel free to remove any comments, and modify variable names
 or constants in the source code.
 
-## Ethics
+# Ethics
 
 I do not condone using this program for unethical or illegal purposes.
-More detestful, at least to me, is asking for help to assist you in
+More detestable, at least to me, is asking for help to assist you in
 something that might not legitimate.
 
 Don't use the issue tracker for such solicitations. To try to stave

Makefile

@@ -40,10 +40,6 @@ check-3.0 check-3.1 check-3.2 check-3.6:
 check-3.7: pytest
 	$(MAKE) -C test check
 
-#:Tests for Python 2.4-2.5 (don't have pytest)
-check-2.4 check-2.5:
-	$(MAKE) -C test $@
-
 #:PyPy 2.6.1 PyPy 5.0.1, or PyPy 5.8.0-beta0
 # Skip for now
 2.6 5.0 5.3 5.6 5.8:

NEWS.md

@@ -1,36 +1,74 @@
-uncompyle6 3.2.4 2018-06-04 7x9 release
+3.2.6 2019-03-23 Mueller Report
+=======================================
+
+Mostly more of the same: bug fixes and pull requests.
+
+Bug Fixes
+-----------
+
+* [#155: Python 3.x bytecode confusing "try/else" with "try" in a loop](https://github.com/rocky/python-uncompyle6/issues/155),
+* [#200: Python 3 bug in not detecting end bounds of an "if" ... "elif"](https://github.com/rocky/python-uncompyle6/issues/200),
+* [#208: Comma placement in 3.6 and 3.7 **kwargs](https://github.com/rocky/python-uncompyle6/issues/208),
+* [#209: Fix "if" return boundary in 3.6+](https://github.com/rocky/python-uncompyle6/issues/209),
+* [#215: 2.7 can have two JUMP_BACKs at the end of a while loop](https://github.com/rocky/python-uncompyle6/issues/215)
+
+Pull Requests
+----------------
+
+* [#202: Better "assert" statement detemination in Python 2.7](https://github.com/rocky/python-uncompyle6/pull/211)
+* [#204: Python 3.7 testing](https://github.com/rocky/python-uncompyle6/pull/204)
+* [#205: Run more f-string tests on Python 3.7](https://github.com/rocky/python-uncompyle6/pull/205)
+* [#211: support utf-8 chars in Python 3 sourcecode](https://github.com/rocky/python-uncompyle6/pull/202)
+
+
+
+3.2.5 2018-12-30 Clearout sale
+======================================
+
+- 3.7.2 Remove deprecation warning on regexp string that isn't raw
+- main.main() parameter `codes` is not used - note that
+- Improve Python 3.6+ control flow detection
+- More complete fragment instruction annotation for `imports`
+
+3.2.4 2018-10-27 7x9 release
+===================================
 
 - Bug fixes #180, #182, #187, #192
 - Enhancements #189
 - Internal improvements
 
-uncompyle6 3.2.3 2018-06-04 Michael Cohen flips and Fleetwood Redux
-
+3.2.3 2018-06-04 Michael Cohen flips and Fleetwood Redux
+======================================================================
 - Python 1.3 support 3.0 bug and
 - fix botched parameter ordering of 3.x in last release
 
-uncompyle6 3.2.2 2018-06-04 When I'm 64
+3.2.2 2018-06-04 When I'm 64
+===================================
 
 - Python 3.0 support and bug fixes
 
-uncompyle6 3.2.1 2018-06-04 MF
+3.2.1 2018-06-04 MF
+=======================
 
 - Python 1.4 and 1.5 bug fixes
 
-uncompyle6 3.2.0 2018-05-19 Rocket Scientist
+3.2.0 2018-05-19 Rocket Scientist
+=========================================
 
 - Add rudimentary 1.4 support (still a bit buggy)
 - add --tree+ option to show formatting rule, when it is constant
 - Python 2.7.15candidate1 support (via xdis)
 - bug fixes, especially for 3.7 (but 2.7 and 3.6 and others as well)
 
-uncompyle6 3.1.3 2018-04-16
+3.1.3 2018-04-16
+====================
 
 - Add some Python 3.7 rules, such as for handling LOAD_METHOD (not complete)
 - Fix some fragment bugs
 - small doc changes
 
-uncompyle6 3.1.2 2018-04-08 Eastern Orthodox Easter
+3.1.2 2018-04-08 Eastern Orthodox Easter
+==================================================
 
 - Python 3.x subclass and call parsing fixes
 - Allow/note running on Python 3.1
@@ -38,7 +76,8 @@ uncompyle6 3.1.2 2018-04-08 Eastern Orthodox Easter
 - DRY instruction building code between 2.x and 3.x
 - expand testing
 
-uncompyle6 3.1.1 2018-04-01 Easter April Fool's
+3.1.1 2018-04-01 Easter April Fool's
+=============================================
 
 Jesus on Friday's New York Times puzzle: "I'm stuck on 2A"
 
@@ -51,7 +90,8 @@ Jesus on Friday's New York Times puzzle: "I'm stuck on 2A"
 - more runtime testing of decompiled code
 - more removal of parenthesis around calls via setting precidence
 
-uncompyle6 3.1.0 2018-03-21 Equinox
+3.1.0 2018-03-21 Equinox
+==============================
 
 - Add code_deparse_with_offset() fragment function.
 - Correct paramenter call fragment deparse_code()
@@ -59,7 +99,8 @@ uncompyle6 3.1.0 2018-03-21 Equinox
   About 5% of 3.6 fail parsing now. But
   semantics still needs much to be desired.
 
-uncompyle6 3.0.1 2018-02-17
+3.0.1 2018-02-17
+====================
 
 - All Python 2.6.9 standard library files weakly verify
 - Many 3.6 fixes. 84% of the first 200 standard library files weakly compile.
@@ -69,7 +110,8 @@ uncompyle6 3.0.1 2018-02-17
 - And more add tests target previous existing bugs more completely
 - sync recent license changes in metadata
 
-uncompyle6 3.0.0 2018-02-17
+3.0.0 2018-02-17
+====================
 
 - deparse_code() and lookalikes from the various semantic actions are
   now deprecated. Instead use new API code_deparse() which makes the
@@ -91,7 +133,8 @@ A bit more work is still needed for 3.6 especially in the area of
 function calls and definitions.
 
 
-uncompyle6 2.16.0 2018-02-17
+2.16.0 2018-02-17
+=====================
 
 - API additions:
   - add fragments.op_at_code_loc() and
@@ -103,18 +146,21 @@ uncompyle6 2.16.0 2018-02-17
 - Fix Python 3.5+ CALL_FUNCTION_VAR and BUILD_LIST_UNPACK in call; with this
   we can can handle 3.5+ f(a, b, *c, *d, *e) now
 
-uncompyle6 2.15.1 2018-02-05
+2.15.1 2018-02-05
+=====================
 
 - More bug fixes and revert an improper bug fix in 2.15.0
 
-uncompyle6 2.15.0 2018-02-05 pycon2018.co
+2.15.0 2018-02-05 pycon2018.co
+=====================================
 
 - Bug fixes
 - Code fragment improvements
 - Code cleanups
 - Expand testing
 
-uncompyle6 2.15.1 2018-01-27
+2.15.1 2018-01-27
+=====================
 
 - Add --linemap option to give line correspondences
   between original source lines and reconstructed line sources.
@@ -126,7 +172,8 @@ uncompyle6 2.15.1 2018-01-27
 - Correct 3.6+ calls with kwargs
 - Describe the difficulty of 3.6 in README
 
-uncompyle6 2.14.3 2018-01-19
+2.14.3 2018-01-19
+=====================
 
 - Fix bug in 3.5+ await stmt
 - Better version to magic handling; handle 3.5.2 .. 3.5.4 versions
@@ -138,7 +185,8 @@ uncompyle6 2.14.3 2018-01-19
 - better tests in setup.py for running the right version of Python
 - Fix 2.6- parsing of "for .. try/else" ...  with "continue"  inside
 
-uncompyle6 2.14.2 2018-01-09 Samish
+2.14.2 2018-01-09 Samish
+==============================
 
 Decompilation bug fixes, mostly 3.6 and pre 2.7
 
@@ -156,7 +204,8 @@ Decompilation bug fixes, mostly 3.6 and pre 2.7
   Python versions
 - Match Python AST names more closely when possible
 
-uncompyle6 2.14.1 2017-12-10 Dr. Gecko
+2.14.1 2017-12-10 Dr. Gecko
+===================================
 
 - Many decompilation bugfixes
 - Grammar rule reduction and version isolation
@@ -164,7 +213,8 @@ uncompyle6 2.14.1 2017-12-10 Dr. Gecko
   with Python AST
 - Start automated Python stdlib testing - full round trip
 
-uncompyle6 2.14.0 2017-11-26 johnnybamazing
+2.14.0 2017-11-26 johnnybamazing
+=========================================
 
 - Start to isolate grammar rules between versions
   and remove used grammar rules
@@ -172,7 +222,8 @@ uncompyle6 2.14.0 2017-11-26 johnnybamazing
   (many more remain)
 - Add stdlib/runtests.sh for even more rigorous testing
 
-uncompyle6 2.13.3 2017-11-13
+2.13.3 2017-11-13
+=====================
 
 Overall: better 3.6 decompiling and some much needed code refactoring and cleanup
 
@@ -198,22 +249,26 @@ Overall: better 3.6 decompiling and some much needed code refactoring and cleanu
 - reinstate some bytecode tests since decompiling has gotten better
 - Revise how to report a bug
 
-uncompyle6 2.13.2 2017-10-12
+2.13.2 2017-10-12
+=====================
 
 - Re-release using a more automated approach
 
-uncompyle6 2.13.1 2017-10-11
+2.13.1 2017-10-11
+=====================
 
 - Re-release because Python 2.4 source uploaded rather than 2.6-3.6
 
-uncompyle6 2.13.0 2017-10-10
+2.13.0 2017-10-10
+=====================
 
 - Fixes in deparsing lambda expressions
 - Improve table-semantics descriptions
 - Document hacky customize arg count better (until we can remove it)
 - Update to use xdis 3.7.0 or greater
 
-uncompyle6 2.12.0 2017-09-26
+2.12.0 2017-09-26
+=====================
 
 - Use xdis 3.6.0 or greater now
 - Small semantic table cleanups
@@ -221,11 +276,13 @@ uncompyle6 2.12.0 2017-09-26
 - Slightly more Python 3.7, but still failing a lot
 - Cross Python 2/3 compatibility with annotation arguments
 
-uncompyle6 2.11.5 2017-08-31
+2.11.5 2017-08-31
+=====================
 
 - Skeletal support for Python 3.7
 
-uncompyle6 2.11.4 2017-08-15
+2.11.4 2017-08-15
+=====================
 
 * scanner and parser now allow 3-part version string lookups,
   e.g. 2.7.1 We allow a float here, but if passed a string like '2.7'. or
@@ -237,7 +294,8 @@ uncompyle6 2.11.4 2017-08-15
 * Some PyPy tolerance in validate testing.
 * Some pyston tolerance
 
-uncompyle6 2.11.3 2017-08-09
+2.11.3 2017-08-09
+=====================
 
 Very minor changes
 
@@ -246,20 +304,24 @@ Very minor changes
 - use xdis opcode sets
 - xdis "exception match" is now "exception-match"
 
-uncompyle6 2.11.2 2017-07-09
+2.11.2 2017-07-09
+=====================
 
 - Start supporting Pypy 3.5 (5.7.1-beta)
 - use xdis 3.5.0's opcode sets and require xdis 3.5.0
 - Correct some Python 2.4-2.6 loop detection
 - guard against badly formatted bytecode
 
-uncompyle6 2.11.1 2017-06-25
+2.11.1 2017-06-25
+=====================
 
 - Python 3.x annotation and function signature fixes
 - Bump xdis version
 - Small pysource bug fixes
 
-uncompyle6 2.11.0 2017-06-18 Fleetwood
+2.11.0 2017-06-18 Fleetwood
+==================================
+
 - Major improvements in fragment tracking
   * Add nonterminal node in extractInfo
   * tag more offsets in expressions
@@ -269,14 +331,16 @@ uncompyle6 2.11.0 2017-06-18 Fleetwood
 - Fixes yet again for make_function node handling; document what's up here
 - Fix bug in snowflake Python 3.5 *args kwargs
 
-uncompyle6 2.10.1 2017-06-3 Marylin Frankel
+2.10.1 2017-06-3 Marylin Frankel
+========================================
 
 - fix some fragments parsing bugs
   - was returning the wrong type sometimes in deparse_code_around_offset()
   - capture function name in offsets
   - track changes to ifelstrmtr node from pysource into fragments
 
-uncompyle6 2.10.0 2017-05-30 Elaine Gordon
+2.10.0 2017-05-30 Elaine Gordon
+=======================================
 
 - Add fuzzy offset deparse look up
 - 3.6 bug fixes
@@ -296,19 +360,21 @@ uncompyle6 2.10.0 2017-05-30 Elaine Gordon
 - 2.3, 2.4 "if 1 .." fixes
 - 3.x annotation fixes
 
-uncompyle6 2.9.11 2017-04-06
+2.9.11 2017-04-06
+=====================
 
 - Better support for Python 3.5+ BUILD_MAP_UNPACK
 - Start 3.6 CALL_FUNCTION_EX support
 - Many decompilation bug fixes. (Many more remain). See ChangeLog
 
-uncompyle6 2.9.10 2017-02-25
+2.9.10 2017-02-25
+=====================
 
 - Python grammar rule fixes
 - Add ability to get grammar coverage on runs
 - Handle Python 3.6 opcode BUILD_CONST_KEYMAP
 
-uncompyle6 2.9.9 2016-12-16
+2.9.9 2016-12-16
 
 - Remaining Python 3.5 ops handled
   (this also means more Python 3.6 ops are handled)
@@ -318,7 +384,8 @@ uncompyle6 2.9.9 2016-12-16
 - Better control-flow detection
 - Code cleanups and misc bug fixes
 
-uncompyle6 2.9.8 2016-12-16
+2.9.8 2016-12-16
+====================
 
 - Better control-flow detection
 - pseudo instruction THEN in 2.x
@@ -331,7 +398,8 @@ uncompyle6 2.9.8 2016-12-16
 - verify call fixes for Python <= 2.4
 - more Python lint
 
-uncompyle6 2.9.7 2016-12-16
+2.9.7 2016-12-16
+====================
 
 - Start to handle 3.5/3.6 build_map_unpack_with_call
 - Some Python 3.6 bytecode to wordcode conversion fixes
@@ -341,7 +409,8 @@ uncompyle6 2.9.7 2016-12-16
 - some 3.2 compatibility
 - Better Python 3 control flow detection by adding Pseudo ELSE opcodes
 
-uncompyle6 2.9.6 2016-12-04
+2.9.6 2016-12-04
+====================
 
 - Shorten Python3 grammars with + and *
   this requires spark parser 1.5.1
@@ -349,7 +418,8 @@ uncompyle6 2.9.6 2016-12-04
   decompile accuracy. This too requires
   spark parser 1.5.1
 
-uncompyle6 2.9.6 2016-11-20
+2.9.6 2016-11-20
+====================
 
 - Correct MANIFEST.in
 - More AST grammar checking
@@ -366,7 +436,8 @@ uncompyle6 2.9.6 2016-11-20
 - Python 2 and 3 detect structure code is more similar
 - Handle Docstrings with embedded triple quotes (""")
 
-uncompyle6 2.9.5 2016-11-13
+2.9.5 2016-11-13
+====================
 
 - Fix Python 3 bugs:
   * improper while 1 else
@@ -376,13 +447,15 @@ uncompyle6 2.9.5 2016-11-13
 - Start grammar misparse checking
 
 
-uncompyle6 2.9.4 2016-11-02
+2.9.4 2016-11-02
+====================
 
 - Handle Python 3.x function annotations
 - track def keyword-parameter line-splitting in source code better
 - bump min xdis version to mask previous xdis bug
 
-uncompyle6 2.9.3 2016-10-26
+2.9.3 2016-10-26
+====================
 
 Release forced by incompatibility change in xdis 3.2.0.
 
@@ -397,7 +470,8 @@ Release forced by incompatibility change in xdis 3.2.0.
   * Handle 3.6 handle single and multiple fstring better
 
 
-uncompyle6 2.9.2 2016-10-15
+2.9.2 2016-10-15
+====================
 
 - use source-code line breaks to assist in where to break
   in tuples and maps
@@ -405,12 +479,14 @@ uncompyle6 2.9.2 2016-10-15
 - Fix some Python 2.6 and below bugs
 - DRY fragments.py code a little
 
-uncompyle6 2.9.1 2016-10-09
+2.9.1 2016-10-09
+====================
 
 - Improved Python 1.5 decompiling
 - Handle old-style pre Python 2.2 classes
 
-uncompyle6 2.9.0 2016-10-09
+2.9.0 2016-10-09
+====================
 
 - Use xdis 3.0.0 protocol load_module.
   this Forces change in requirements.txt and _pkg_info_.py
@@ -420,7 +496,8 @@ uncompyle6 2.9.0 2016-10-09
 - Fix bug with -t ...  Wasn't showing source text when -t option was given
 - Fix 2.1-2.6 bug in list comprehension
 
-uncompyle6 2.8.4 2016-10-08
+2.8.4 2016-10-08
+====================
 
 - Python 3 disassembly bug fixes
 - Python 3.6 fstring bug fixes (from moagstar)
@@ -428,7 +505,8 @@ uncompyle6 2.8.4 2016-10-08
 - COME_FROM suffixes added in Python3
 - use .py extension in verification disassembly
 
-uncompyle6 2.8.3 2016-09-11 live from NYC!
+2.8.3 2016-09-11 live from NYC!
+=======================================
 
 NOTE: this is possibly the last release before a major reworking of
 control-flow structure detection is done.
@@ -456,14 +534,16 @@ control-flow structure detection is done.
 - bump xdis requirement so we can deparse dropbox 2.5 code
 - Added H. Goebel's changes before 2.4 in DECOMPYLE-2.4-CHANGELOG.txt
 
-uncompyle6 2.8.2 2016-08-29
+2.8.2 2016-08-29
+====================
 
 - Handle Python 3.6 format string conversions !r, !s, !a
 - Start to handle 3.1 bytecode
 - Fix some PyPy translation bugs
 - We now only handle 3.6.0a3+ since that is incompatible with 3.6 before that
 
-uncompyle6 2.8.1 2016-08-20
+2.8.1 2016-08-20
+====================
 
 - Add Python 2.2 decompilation
 
@@ -471,7 +551,8 @@ uncompyle6 2.8.1 2016-08-20
  * PyPy LOOKUP_METHOD bug
  * Python 3.6 FORMAT_VALUE handles expressions now
 
-uncompyle6 2.8.0 2016-08-03
+2.8.0 2016-08-03
+====================
 
 - Start Python 3.6 support (moagstar)
   more work on PEP 498 needed
@@ -482,20 +563,23 @@ uncompyle6 2.8.0 2016-08-03
 - better grammar and semantic action segregation based
   on python bytecode version
 
-uncompyle6 2.7.1 2016-07-26
+2.7.1 2016-07-26
+====================
 
 - PyPy bytecodes for 2.7 and 3.2 added
 - Instruction formatting improved slightly
 - 2.7 bytecode "continue" bug fixed
 
-uncompyle6 2.7.0 2016-07-15
+2.7.0 2016-07-15
+====================
 
 - Many Syntax and verification bugs removed
   tested on standard libraries from 2.3.7 to 3.5.1
   and they all decompile and verify fine.
   I'm sure there are more bugs though.
 
-uncompyle6 2.6.2 2016-07-11 Manhattenhenge
+2.6.2 2016-07-11 Manhattenhenge
+=======================================
 
 - Extend bytecodes back to 2.3
 - Fix bugs:
@@ -504,13 +588,15 @@ uncompyle6 2.6.2 2016-07-11 Manhattenhenge
   * continue statements
 - DRY and segregate grammar more
 
-uncompyle6 2.6.1 2016-07-08
+2.6.1 2016-07-08
+====================
 
 - Go over Python 2.5 bytecode deparsing
   all library programs now deparse
 - Fix a couple bugs in 2.6 deparsing
 
-uncompyle6 2.6.0 2016-07-07
+2.6.0 2016-07-07
+====================
 
 - Improve Python 2.6 bytecode deparsing:
   stdlib now will deparse something
@@ -519,7 +605,8 @@ uncompyle6 2.6.0 2016-07-07
 - Fix bug in installing uncompyle6 script
 - Doc improvements
 
-uncompyle6 2.5.0 2016-06-22  Summer Solstice
+2.5.0 2016-06-22 Summer Solstice
+========================================
 
 - Much better Python 3.2-3.5 coverage.
   3.4.6 is probably the best;3.2 and 3.5 are weaker
@@ -528,7 +615,8 @@ uncompyle6 2.5.0 2016-06-22  Summer Solstice
 - Better fragment offset tracking
 - Some (much-needed) code refactoring
 
-uncompyle6 2.4.0 2016-05-18 (in memory of Lewis Bernstein)
+2.4.0 2016-05-18 (in memory of Lewis Bernstein)
+===========================================================
 
 - Many Python 3 bugs fixed:
   * Python 3.2 to 3.5 libraries largely
@@ -543,7 +631,8 @@ uncompyle6 2.4.0 2016-05-18 (in memory of Lewis Bernstein)
   * handle complex number unmarshaling
   * Running on Python 2 to works on Python 3.5 bytecodes now
 
-uncompyle6 2.3.5 and 2.3.6 2016-05-14
+2.3.5 and 2.3.6 2016-05-14
+=================================
 
 - Python 2 class decorator fix (thanks to Tey)
 - Fix fragment parsing bugs
@@ -555,20 +644,23 @@ uncompyle6 2.3.5 and 2.3.6 2016-05-14
 - Correct history based on info from Dan Pascu
 - Fix up pip packaging, ugh.
 
-uncompyle6 2.3.4 2016-05-5
+2.3.4 2016-05-5
+===================
 
 - More Python 3.5 parsing bugs addressed
 - decompiling Python 3.5 from other Python versions works
 - test from Python 3.2
 - remove "__module__ = __name__" in 3.0 <= Python 3.2
 
-uncompyle6 2.3.3 2016-05-3
+2.3.3 2016-05-3
+===================
 
 - Fix bug in running uncompyle6 script on Python 3
 - Speed up performance on deparsing long lists by grouping in chunks of 32 and 256 items
 - DRY Python expressions between Python 2 and 3
 
-uncompyle6 2.3.2 2016-05-1
+2.3.2 2016-05-1
+===================
 
 - Add --version option standalone scripts
 - Correct License information in package
@@ -577,17 +669,20 @@ uncompyle6 2.3.2 2016-05-1
   specific grammar code
 - Fix bug in 3.5+ constant map parsing
 
-uncompyle6 2.3.0, 2.3.1 2016-04-30
+2.3.0, 2.3.1 2016-04-30
+=============================
 
 - Require spark_parser  >= 1.1.0
 
-uncompyle6 2.2.0 2016-04-30
+2.2.0 2016-04-30
+====================
 
 - Spark is no longer here but pulled separate package spark_parse
 - Python 3 parsing fixes
 - More tests
 
-uncompyle6 2.2.0 2016-04-02
+2.2.0 2016-04-02
+====================
 
 - Support single-mode (in addition to exec-mode) compilation
 - Start to DRY Python 2 and Python 3 grammars
@@ -595,7 +690,8 @@ uncompyle6 2.2.0 2016-04-02
 - Fix bug in uncomplye6 -d and -r options (via lelicopter)
 
 
-uncompyle6 2.1.3 2016-01-02
+2.1.3 2016-01-02
+====================
 
 - Limited support for decompiling Python 3.5
 - Improve Python 3 class deparsing
@@ -604,18 +700,21 @@ uncompyle6 2.1.3 2016-01-02
 - increase test coverage
 - fix misc small bugs and some improvements
 
-uncompyle6 2.1.2 2015-12-31
+2.1.2 2015-12-31
+====================
 
 - Fix cross-version Marshal loading
 - Handle Python 3.3 . dotted class names
 - Limited 3.5 support: allows deparsing other versions
 - Refactor code more, misc bug fixes
 
-uncompyle6 2.1.1 2015-12-27
+2.1.1 2015-12-27
+====================
 
 - packaging issues
 
-uncompyle6 2.1.0 2015-12-27
+2.1.0 2015-12-27
+====================
 
 - Python 3.x deparsing much more solid
 - Better cross-version deparsing
@@ -624,7 +723,8 @@ Some bugs squashed while other run rampant. Some code cleanup while
 much more is yet needed. More tests added, but many more are needed.
 
 
-uncompyle6 2.0.0 2015-12-11
+2.0.0 2015-12-11
+====================
 
 Changes from uncompyle2
 

__pkginfo__.py

@@ -57,7 +57,7 @@ entry_points       = {
     ]}
 ftp_url            = None
 install_requires   = ['spark-parser >= 1.8.7, < 1.9.0',
-                      'xdis >= 3.8.8, < 3.9.0']
+                      'xdis >= 3.9.0, < 3.10.0']
 
 license            = 'GPL3'
 mailing_list       = 'python-debugger@googlegroups.com'

admin-tools/how-to-make-a-release.md

@@ -30,9 +30,9 @@
 
     $ make ChangeLog
 
-#  Update NEWS from ChangeLog:
+#  Update NEWS.md from ChangeLog:
 
-    $ emacs NEWS
+    $ emacs NEWS.md
     $ make check
     $ git commit --amend .
     $ git push   # get CI testing going early

admin-tools/how-to-make-a-release.txt

@@ -1,46 +0,0 @@
-git pull
-
-Change version in uncompyle6/version.py
-source uncompyle6/version.py
-echo $VERSION
-git commit -m"Get ready for release $VERSION" .
-
-Update ChangeLog:
-  make ChangeLog
-
-Update NEWS from ChangeLog
-make check
-
-git commit --amend .
-
-git push
-
-Make sure pyenv is running
-# Pyenv
-
-source admin-tools/check-newer-versions.sh
-
-
-# Switch to python-2.4 and build that first...
-source admin-tools/setup-python-2.4
-
-rm ChangeLog
-git merge master
-
-Update NEWS from master branch
-
-git commit -m"Get ready for release $VERSION" .
-
-source admin-tools/check-older-versions.sh
-source admin-tools/check-newer-versions.sh
-
-make-dist-older.sh
-
-git tag release-python-2.4-$VERSION
-
-./make-dist-newer.sh
-
-git tag release-$VERSION
-
-
-twine upload dist/uncompyle6-${VERSION}*

admin-tools/pyenv-newer-versions

@@ -5,4 +5,4 @@ if [[ $0 == ${BASH_SOURCE[0]} ]] ; then
     echo "This script should be *sourced* rather than run directly through bash"
     exit 1
 fi
-export PYVERSIONS='3.5.5 3.6.6 3.7.1 2.6.9 3.3.7 2.7.14 3.2.6 3.1.5 3.4.8'
+export PYVERSIONS='3.2.6 3.6.8 3.7.2 2.6.9 3.3.7 2.7.15 3.2.6 3.1.5 3.4.8'

admin-tools/update-sources.sh

@@ -1,3 +0,0 @@
-#!/bin/bash
-cd $(dirname ${BASH_SOURCE[0]})/..
-git pull

pytest/test_deparse.py

@@ -30,7 +30,7 @@ def list_comp():
     [y for y in range(3)]
 
 def get_parsed_for_fn(fn):
-    code = fn.func_code
+    code = fn.__code__ if PYTHON3 else fn.func_code
     return deparse(code, version=PYTHON_VERSION)
 
 def check_expect(expect, parsed, fn_name):

pytest/test_docstring.py

@@ -10,7 +10,7 @@ else:
     maxint = sys.maxint
 from uncompyle6.semantics.helper import print_docstring
 
-class PrintFake:
+class PrintFake():
     def __init__(self):
         self.pending_newlines = 0
         self.f = StringIO()

pytest/test_fjt.py

@@ -20,7 +20,7 @@ def bug_loop(disassemble, tb=None):
     disassemble(tb)
 
 def test_if_in_for():
-    code = bug.func_code
+    code = bug.__code__
     scan = get_scanner(PYTHON_VERSION)
     if 2.7 <= PYTHON_VERSION <= 3.0 and not IS_PYPY:
         scan.build_instructions(code)

pytest/test_fstring.py

@@ -0,0 +1,154 @@
+# std
+# test
+from uncompyle6 import PYTHON_VERSION, deparse_code
+import pytest
+pytestmark = pytest.mark.skipif(PYTHON_VERSION <= 2.6,
+                                reason='hypothesis needs 2.7 or later')
+if PYTHON_VERSION > 2.6:
+
+    import hypothesis
+    from hypothesis import strategies as st
+
+    # uncompyle6
+
+
+    @st.composite
+    def expressions(draw):
+        # todo : would be nice to generate expressions using hypothesis however
+        # this is pretty involved so for now just use a corpus of expressions
+        # from which to select.
+        return draw(st.sampled_from((
+            'abc',
+            'len(items)',
+            'x + 1',
+            'lineno',
+            'container',
+            'self.attribute',
+            'self.method()',
+            # These expressions are failing, I think these are control
+            # flow problems rather than problems with FORMAT_VALUE,
+            # however I need to confirm this...
+            #'sorted(items, key=lambda x: x.name)',
+            #'func(*args, **kwargs)',
+            #'text or default',
+            #'43 if life_the_universe and everything else None'
+        )))
+
+
+    @st.composite
+    def format_specifiers(draw):
+        """
+        Generate a valid format specifier using the rules:
+
+        format_spec ::=  [[fill]align][sign][#][0][width][,][.precision][type]
+        fill        ::=  <any character>
+        align       ::=  "<" | ">" | "=" | "^"
+        sign        ::=  "+" | "-" | " "
+        width       ::=  integer
+        precision   ::=  integer
+        type        ::=  "b" | "c" | "d" | "e" | "E" | "f" | "F" | "g" | "G" | "n" | "o" | "s" | "x" | "X" | "%"
+
+        See https://docs.python.org/2/library/string.html
+
+        :param draw: Let hypothesis draw from other strategies.
+
+        :return: An example format_specifier.
+        """
+        alphabet_strategy = st.characters(min_codepoint=ord('a'), max_codepoint=ord('z'))
+        fill = draw(st.one_of(alphabet_strategy, st.none()))
+        align = draw(st.sampled_from(list('<>=^')))
+        fill_align = (fill + align or '') if fill else ''
+
+        type_ = draw(st.sampled_from('bcdeEfFgGnosxX%'))
+        can_have_sign = type_ in 'deEfFgGnoxX%'
+        can_have_comma = type_ in 'deEfFgG%'
+        can_have_precision = type_ in 'fFgG'
+        can_have_pound = type_ in 'boxX%'
+        can_have_zero = type_ in 'oxX'
+
+        sign = draw(st.sampled_from(list('+- ') + [''])) if can_have_sign else ''
+        pound = draw(st.sampled_from(('#', '',))) if can_have_pound else ''
+        zero = draw(st.sampled_from(('0', '',))) if can_have_zero else ''
+
+        int_strategy = st.integers(min_value=1, max_value=1000)
+
+        width = draw(st.one_of(int_strategy, st.none()))
+        width = str(width) if width is not None else ''
+
+        comma = draw(st.sampled_from((',', '',))) if can_have_comma else ''
+        if can_have_precision:
+            precision = draw(st.one_of(int_strategy, st.none()))
+            precision = '.' + str(precision) if precision else ''
+        else:
+            precision = ''
+
+        return ''.join((fill_align, sign, pound, zero, width, comma, precision, type_,))
+
+
+    @st.composite
+    def fstrings(draw):
+        """
+        Generate a valid f-string.
+        See https://www.python.org/dev/peps/pep-0498/#specification
+
+        :param draw: Let hypothsis draw from other strategies.
+
+        :return: A valid f-string.
+        """
+        character_strategy = st.characters(
+            blacklist_characters='\r\n\'\\s{}',
+            min_codepoint=1,
+            max_codepoint=1000,
+        )
+        is_raw = draw(st.booleans())
+        integer_strategy = st.integers(min_value=0, max_value=3)
+        expression_count = draw(integer_strategy)
+        content = []
+        for _ in range(expression_count):
+            expression = draw(expressions())
+            conversion = draw(st.sampled_from(('', '!s', '!r', '!a',)))
+            has_specifier = draw(st.booleans())
+            specifier = ':' + draw(format_specifiers()) if has_specifier else ''
+            content.append('{{{}{}}}'.format(expression, conversion, specifier))
+            content.append(draw(st.text(character_strategy)))
+        content = ''.join(content)
+        return "f{}'{}'".format('r' if is_raw else '', content)
+
+
+    @pytest.mark.skipif(PYTHON_VERSION != 3.6, reason='need Python 3.6')
+    @hypothesis.given(format_specifiers())
+    def test_format_specifiers(format_specifier):
+        """Verify that format_specifiers generates valid specifiers"""
+        try:
+            exec('"{:' + format_specifier + '}".format(0)')
+        except ValueError as e:
+            if 'Unknown format code' not in str(e):
+                raise
+
+
+    def run_test(text):
+        hypothesis.assume(len(text))
+        hypothesis.assume("f'{" in text)
+        expr = text + '\n'
+        code = compile(expr, '<string>', 'single')
+        deparsed = deparse_code(PYTHON_VERSION, code, compile_mode='single')
+        recompiled = compile(deparsed.text, '<string>', 'single')
+        if recompiled != code:
+            assert 'dis(' + deparsed.text.strip('\n') + ')' == 'dis(' + expr.strip('\n') + ')'
+
+
+    @pytest.mark.skipif(PYTHON_VERSION != 3.6, reason='need Python 3.6')
+    @hypothesis.given(fstrings())
+    def test_uncompyle_fstring(fstring):
+        """Verify uncompyling fstring bytecode"""
+        run_test(fstring)
+
+
+    @pytest.mark.skipif(PYTHON_VERSION < 3.6, reason='need Python 3.6+')
+    @pytest.mark.parametrize('fstring', [
+        "f'{abc}{abc!s}'",
+        "f'{abc}0'",
+    ])
+    def test_uncompyle_direct(fstring):
+        """useful for debugging"""
+        run_test(fstring)

pytest/test_function_call.py

@@ -0,0 +1,185 @@
+import string
+from uncompyle6 import PYTHON_VERSION
+import pytest
+pytestmark = pytest.mark.skip(PYTHON_VERSION < 2.7,
+                              reason="need at least Python 2.7")
+
+if PYTHON_VERSION > 2.6:
+    from hypothesis import given, assume, example, settings, strategies as st
+    from validate import validate_uncompyle
+    from test_fstring import expressions
+
+    alpha = st.sampled_from(string.ascii_lowercase)
+    numbers = st.sampled_from(string.digits)
+    alphanum = st.sampled_from(string.ascii_lowercase + string.digits)
+
+
+    @st.composite
+    def function_calls(draw,
+                       min_keyword_args=0, max_keyword_args=5,
+                       min_positional_args=0, max_positional_args=5,
+                       min_star_args=0, max_star_args=1,
+                       min_double_star_args=0, max_double_star_args=1):
+        """
+        Strategy factory for generating function calls.
+
+        :param draw: Callable which draws examples from other strategies.
+
+        :return: The function call text.
+        """
+        st_positional_args = st.lists(
+            alpha,
+            min_size=min_positional_args,
+            max_size=max_positional_args
+        )
+        st_keyword_args = st.lists(
+            alpha,
+            min_size=min_keyword_args,
+            max_size=max_keyword_args
+        )
+        st_star_args = st.lists(
+            alpha,
+            min_size=min_star_args,
+            max_size=max_star_args
+        )
+        st_double_star_args = st.lists(
+            alpha,
+            min_size=min_double_star_args,
+            max_size=max_double_star_args
+        )
+
+        positional_args = draw(st_positional_args)
+        keyword_args = draw(st_keyword_args)
+        st_values = st.lists(
+            expressions(),
+            min_size=len(keyword_args),
+            max_size=len(keyword_args)
+        )
+        keyword_args = [
+            x + '=' + e
+            for x, e in
+            zip(keyword_args, draw(st_values))
+        ]
+        star_args = ['*' + x for x in draw(st_star_args)]
+        double_star_args = ['**' + x for x in draw(st_double_star_args)]
+
+        arguments = positional_args + keyword_args + star_args + double_star_args
+        draw(st.randoms()).shuffle(arguments)
+        arguments = ','.join(arguments)
+
+        function_call = 'fn({arguments})'.format(arguments=arguments)
+        try:
+            # TODO: Figure out the exact rules for ordering of positional, keyword,
+            # star args, double star args and in which versions the various
+            # types of arguments are supported so we don't need to check that the
+            # expression compiles like this.
+            compile(function_call, '<string>', 'single')
+        except:
+            assume(False)
+        return function_call
+
+
+    def test_function_no_args():
+        validate_uncompyle("fn()")
+
+    @pytest.mark.skipif(PYTHON_VERSION < 2.7,
+                        reason="need at least Python 2.7")
+    def isolated_function_calls(which):
+        """
+        Returns a strategy for generating function calls, but isolated to
+        particular types of arguments, for example only positional arguments.
+
+        This can help reason about debugging errors in specific types of function
+        calls.
+
+        :param which: One of 'keyword', 'positional', 'star', 'double_star'
+
+        :return: Strategy for generating an function call isolated to specific
+                 argument types.
+        """
+        kwargs = dict(
+            max_keyword_args=0,
+            max_positional_args=0,
+            max_star_args=0,
+            max_double_star_args=0,
+        )
+        kwargs['_'.join(('min', which, 'args'))] = 1
+        kwargs['_'.join(('max', which, 'args'))] = 5 if 'star' not in which else 1
+        return function_calls(**kwargs)
+
+
+    with settings(max_examples=25):
+
+        @pytest.mark.skipif(PYTHON_VERSION < 2.7,
+                            reason="need at least Python 2.7")
+        @given(isolated_function_calls('positional'))
+        @example("fn(0)")
+        def test_function_positional_only(expr):
+            validate_uncompyle(expr)
+
+        @pytest.mark.skipif(PYTHON_VERSION < 2.7,
+                            reason="need at least Python 2.7")
+        @given(isolated_function_calls('keyword'))
+        @example("fn(a=0)")
+        def test_function_call_keyword_only(expr):
+            validate_uncompyle(expr)
+
+        @pytest.mark.skipif(PYTHON_VERSION < 2.7,
+                            reason="need at least Python 2.7")
+        @given(isolated_function_calls('star'))
+        @example("fn(*items)")
+        def test_function_call_star_only(expr):
+            validate_uncompyle(expr)
+
+        @pytest.mark.skipif(PYTHON_VERSION < 2.7,
+                            reason="need at least Python 2.7")
+        @given(isolated_function_calls('double_star'))
+        @example("fn(**{})")
+        def test_function_call_double_star_only(expr):
+            validate_uncompyle(expr)
+
+
+    @pytest.mark.xfail()
+    def test_BUILD_CONST_KEY_MAP_BUILD_MAP_UNPACK_WITH_CALL_BUILD_TUPLE_CALL_FUNCTION_EX():
+        validate_uncompyle("fn(w=0,m=0,**v)")
+
+
+    @pytest.mark.xfail()
+    def test_BUILD_MAP_BUILD_MAP_UNPACK_WITH_CALL_BUILD_TUPLE_CALL_FUNCTION_EX():
+        validate_uncompyle("fn(a=0,**g)")
+
+
+    @pytest.mark.xfail()
+    def test_CALL_FUNCTION_EX():
+        validate_uncompyle("fn(*g,**j)")
+
+
+    @pytest.mark.xfail()
+    def test_BUILD_MAP_CALL_FUNCTION_EX():
+        validate_uncompyle("fn(*z,u=0)")
+
+
+    @pytest.mark.xfail()
+    def test_BUILD_TUPLE_CALL_FUNCTION_EX():
+        validate_uncompyle("fn(**a)")
+
+
+    @pytest.mark.xfail()
+    def test_BUILD_MAP_BUILD_TUPLE_BUILD_TUPLE_UNPACK_WITH_CALL_CALL_FUNCTION_EX():
+        validate_uncompyle("fn(b,b,b=0,*a)")
+
+
+    @pytest.mark.xfail()
+    def test_BUILD_TUPLE_BUILD_TUPLE_UNPACK_WITH_CALL_CALL_FUNCTION_EX():
+        validate_uncompyle("fn(*c,v)")
+
+
+    @pytest.mark.xfail()
+    def test_BUILD_CONST_KEY_MAP_CALL_FUNCTION_EX():
+        validate_uncompyle("fn(i=0,y=0,*p)")
+
+
+    @pytest.mark.skip(reason='skipping property based test until all individual tests are passing')
+    @given(function_calls())
+    def test_function_call(function_call):
+        validate_uncompyle(function_call)

pytest/test_grammar.py

@@ -7,7 +7,7 @@ def test_grammar():
 
     def check_tokens(tokens, opcode_set):
         remain_tokens = set(tokens) - opcode_set
-        remain_tokens = set([re.sub('_\d+$','', t) for t in remain_tokens])
+        remain_tokens = set([re.sub(r'_\d+$','', t) for t in remain_tokens])
         remain_tokens = set([re.sub('_CONT$','', t) for t in remain_tokens])
         remain_tokens = set(remain_tokens) - opcode_set
         assert remain_tokens == set([]), \
@@ -46,6 +46,7 @@ def test_grammar():
             unused_rhs.add("mkfunc_annotate")
             unused_rhs.add("dict_comp")
             unused_rhs.add("classdefdeco1")
+            unused_rhs.add("tryelsestmtl")
             if PYTHON_VERSION >= 3.5:
                 expect_right_recursive.add((('l_stmts',
                                              ('lastl_stmt', 'come_froms', 'l_stmts'))))

pytest/test_pysource.py

@@ -8,8 +8,12 @@ from uncompyle6.semantics.consts import (
 
 if PYTHON3:
     from io import StringIO
+    def iteritems(d):
+        return d.items()
 else:
     from StringIO import StringIO
+    def iteritems(d):
+        return d.iteritems()
 
 from uncompyle6.semantics.pysource import SourceWalker as SourceWalker
 
@@ -26,7 +30,7 @@ def test_template_engine():
     # FIXME: and so on...
 
 from uncompyle6.semantics.consts import (
-    TABLE_R, TABLE_DIRECT,
+    TABLE_DIRECT, TABLE_R,
     )
 
 from uncompyle6.semantics.fragments import (
@@ -40,7 +44,7 @@ def test_tables():
             (TABLE_DIRECT, 'TABLE_DIRECT', False),
             (TABLE_R, 'TABLE_R', False),
             (TABLE_DIRECT_FRAGMENT, 'TABLE_DIRECT_FRAGMENT', True)):
-        for k, entry in t.iteritems():
+        for k, entry in iteritems(t):
             if k in skip_for_now:
                 continue
             fmt = entry[0]

pytest/test_single_compile.py

@@ -1,4 +1,7 @@
+import pytest
 from uncompyle6 import PYTHON_VERSION, code_deparse
+pytestmark = pytest.mark.skip(PYTHON_VERSION < 2.7,
+                              reason="need at least Python 2.7")
 
 if PYTHON_VERSION > 2.6:
     def test_single_mode():

pytest/testdata/if-2.7.right

@@ -9,4 +9,4 @@
           12  JUMP_FORWARD          0  'to 15'
         15_0  COME_FROM            12  '12'
           15  LOAD_CONST            0  None
-          18  RETURN_VALUE
+          18  RETURN_VALUE     

pytest/testdata/ifelse-2.7.right

@@ -12,4 +12,4 @@
           18  STORE_NAME            2  'd'
         21_0  COME_FROM            12  '12'
           21  LOAD_CONST            2  None
-          24  RETURN_VALUE
+          24  RETURN_VALUE     

pytest/validate.py

@@ -1,25 +1,28 @@
+# future
+from __future__ import print_function
 # std
 import os
 import difflib
 import subprocess
 import tempfile
-from StringIO import StringIO
-
+import functools
 # uncompyle6 / xdis
-from uncompyle6 import PYTHON_VERSION, IS_PYPY, deparse_code
+from uncompyle6 import PYTHON_VERSION, PYTHON3, IS_PYPY, deparse_code
 # TODO : I think we can get xdis to support the dis api (python 3 version) by doing something like this there
 from xdis.bytecode import Bytecode
 from xdis.main import get_opcode
 opc = get_opcode(PYTHON_VERSION, IS_PYPY)
-from StringIO import StringIO
+Bytecode = functools.partial(Bytecode, opc=opc)
+import six
+
+if PYTHON3:
+    from io import StringIO
+else:
+    from StringIO import StringIO
+
+def _dis_to_text(co):
+    return Bytecode(co).dis()
 
-try:
-    import functools
-    Bytecode = functools.partial(Bytecode, opc=opc)
-    def _dis_to_text(co):
-        return Bytecode(co).dis()
-except:
-    pass
 
 def print_diff(original, uncompyled):
     """
@@ -42,11 +45,8 @@ def print_diff(original, uncompyled):
         print('\nTo display diff highlighting run:\n    pip install BeautifulSoup4')
         diff = difflib.HtmlDiff().make_table(*args)
 
-    f = tempfile.NamedTemporaryFile(delete=False)
-    try:
+    with tempfile.NamedTemporaryFile(delete=False) as f:
         f.write(str(diff).encode('utf-8'))
-    finally:
-        f.close()
 
     try:
         print()
@@ -63,7 +63,8 @@ def print_diff(original, uncompyled):
         print('\nFor side by side diff install elinks')
         diff = difflib.Differ().compare(original_lines, uncompyled_lines)
         print('\n'.join(diff))
-    os.unlink(f.name)
+    finally:
+        os.unlink(f.name)
 
 
 def are_instructions_equal(i1, i2):
@@ -125,9 +126,8 @@ def validate_uncompyle(text, mode='exec'):
     original_text = text
 
     deparsed = deparse_code(PYTHON_VERSION, original_code,
-
                             compile_mode=mode,
-                            out=StringIO(),
+                            out=six.StringIO(),
                             is_pypy=IS_PYPY)
     uncompyled_text = deparsed.text
     uncompyled_code = compile(uncompyled_text, '<string>', 'exec')

setup.py

@@ -1,16 +1,16 @@
 #!/usr/bin/env python
-"""Setup script for the 'uncompyle6' distribution."""
-
 import sys
 
+"""Setup script for the 'uncompyle6' distribution."""
+
 SYS_VERSION = sys.version_info[0:2]
-if not ((2, 4) <= SYS_VERSION  <= (2, 7)):
-    mess = "Python Release 2.4 .. 2.7 are supported in this code branch."
-    if ((3, 2) <= SYS_VERSION  <= (3, 7)):
-        mess += ("\nFor your Python, version %s, use the master code/branch." %
+if not ((2, 6) <= SYS_VERSION  <= (3, 7)):
+    mess = "Python Release 2.6 .. 3.7 are supported in this code branch."
+    if ((2, 4) <= SYS_VERSION <= (2, 7)):
+        mess += ("\nFor your Python, version %s, use the python-2.4 code/branch." %
                  sys.version[0:3])
-    else:
-        mess += ("\nThis package is not supported before Python 2.4. Your Python version is %s."
+    elif SYS_VERSION < (2, 4):
+        mess += ("\nThis package is not supported for Python version %s."
                  % sys.version[0:3])
     print(mess)
     raise Exception(mess)

test-unit/test_grammar.py

@@ -1,55 +0,0 @@
-import re
-import unittest
-from uncompyle6 import PYTHON_VERSION, IS_PYPY # , PYTHON_VERSION
-from uncompyle6.parser import get_python_parser, python_parser
-
-class TestGrammar(unittest.TestCase):
-    def test_grammar(self):
-
-        def check_tokens(tokens, opcode_set):
-            remain_tokens = set(tokens) - opcode_set
-            remain_tokens = set([re.sub('_\d+$','', t) for t in remain_tokens])
-            remain_tokens = set([re.sub('_CONT$','', t) for t in remain_tokens])
-            remain_tokens = set(remain_tokens) - opcode_set
-            self.assertEqual(remain_tokens, set([]),
-                    "Remaining tokens %s\n====\n%s" % (remain_tokens, p.dump_grammar()))
-
-        p = get_python_parser(PYTHON_VERSION, is_pypy=IS_PYPY)
-        (lhs, rhs, tokens,
-         right_recursive, dup_rhs) = p.check_sets()
-        expect_lhs = set(['expr1024', 'pos_arg'])
-        unused_rhs = set(['list', 'call', 'mkfunc',
-                          'mklambda',
-                          'unpack',])
-
-        expect_right_recursive = frozenset([('designList',
-                                             ('store', 'DUP_TOP', 'designList'))])
-        expect_lhs.add('kwarg')
-
-        self.assertEqual(expect_lhs, set(lhs))
-        self.assertEqual(unused_rhs, set(rhs))
-        self.assertEqual(expect_right_recursive, right_recursive)
-
-        expect_dup_rhs = frozenset([('COME_FROM',), ('CONTINUE',), ('JUMP_ABSOLUTE',),
-                                    ('LOAD_CONST',),
-                                    ('JUMP_BACK',), ('JUMP_FORWARD',)])
-
-        reduced_dup_rhs = {}
-        for k in dup_rhs:
-            if k not in expect_dup_rhs:
-                reduced_dup_rhs[k] = dup_rhs[k]
-                pass
-            pass
-        for k in reduced_dup_rhs:
-            print(k, reduced_dup_rhs[k])
-        # assert not reduced_dup_rhs, reduced_dup_rhs
-
-    def test_dup_rule(self):
-        import inspect
-        python_parser(PYTHON_VERSION, inspect.currentframe().f_code,
-                      is_pypy=IS_PYPY,
-                      parser_debug={
-                          'dups': True, 'transition': False, 'reduce': False,
-                          'rules': False, 'errorstack': None, 'context': True})
-if __name__ == '__main__':
-    unittest.main()

test/Makefile

@@ -29,7 +29,7 @@ check:
 	$(MAKE) check-$(PYTHON_VERSION)
 
 #: Run working tests from Python 2.6 or 2.7
-check-2.4 check-2.5 check-2.6 check-2.7: check-bytecode-2 check-bytecode-3 check-bytecode-1  check-native-short
+check-2.6 check-2.7: check-bytecode-2 check-bytecode-3 check-bytecode-1 check-native-short
 
 #: Run working tests from Python 3.0
 check-3.0: check-bytecode
@@ -64,10 +64,12 @@ check-3.5: check-bytecode
 #: Run working tests from Python 3.6
 check-3.6: check-bytecode
 	$(PYTHON) test_pythonlib.py --bytecode-3.6 --weak-verify $(COMPILE)
+	$(PYTHON) test_pythonlib.py --bytecode-3.6-run --verify-run
 
 #: Run working tests from Python 3.7
 check-3.7: check-bytecode
 	$(PYTHON) test_pythonlib.py --bytecode-3.7 --weak-verify $(COMPILE)
+	$(PYTHON) test_pythonlib.py --bytecode-3.7-run --verify-run
 
 # FIXME
 #: this is called when running under pypy3.5-5.8.0 or pypy2-5.6.0
@@ -89,8 +91,9 @@ check-bytecode-2:
 #: Check deparsing bytecode 3.x only
 check-bytecode-3:
 	$(PYTHON) test_pythonlib.py --bytecode-3.0 \
-       --bytecode-3.1 --bytecode-3.2 --bytecode-3.3 \
-       --bytecode-3.4 --bytecode-3.5 --bytecode-3.6 --bytecode-pypy3.2
+	--bytecode-3.1 --bytecode-3.2 --bytecode-3.3 \
+	--bytecode-3.4 --bytecode-3.5 --bytecode-3.6 --bytecode-3.7 \
+	--bytecode-pypy3.2
 
 #: Check deparsing on selected bytecode 3.x
 check-bytecode-3-short:
@@ -266,10 +269,6 @@ check-native-short:
 	$(PYTHON) test_pythonlib.py --bytecode-$(PYTHON_VERSION) --weak-verify $(COMPILE)
 	$(PYTHON) test_pythonlib.py --bytecode-$(PYTHON_VERSION)-run --verify-run $(COMPILE)
 
-#: Run longer Python 2.6's lib files known to be okay
-check-2.4-ok:
-	$(PYTHON) test_pythonlib.py --ok-2.4 --verify $(COMPILE)
-
 #: Run longer Python 2.6's lib files known to be okay
 check-2.6-ok:
 	$(PYTHON) test_pythonlib.py --ok-2.6 --weak-verify $(COMPILE)

test/dis-compare.py

@@ -1,8 +1,9 @@
 #!/usr/bin/env python
 # Mode: -*- python -*-
 #
-# Copyright (c) 2015, 2017 by Rocky Bernstein <rb@dustyfeet.com>
+# Copyright (c) 2015 by Rocky Bernstein <rb@dustyfeet.com>
 #
+from __future__ import print_function
 
 
 import dis, os.path

test/simple-uncompyle-code-test.py

@@ -1,6 +1,7 @@
 #!/usr/bin/env python
 
-from uncompyle6 import uncompyle
+from __future__ import print_function
+
 from uncompyle6.main import decompile
 from xdis.magics import sysinfo2float
 import sys, inspect

test/simple_source/bug24/03_iftrue.py

@@ -0,0 +1,10 @@
+# Python 2.4 (and before?) bug in handling unconditional "else if true"
+# Doesn't occur in Python > 2.4
+# From Issue #187
+def unconditional_if_true_24(foo):
+    if not foo:
+        pass
+    elif 1:
+        pass
+    else:
+        return None

test/simple_source/bug25/02_yield_bug.py

@@ -1,7 +0,0 @@
-# From 2.4 test_array.py
-# In Python 2.4 and earlier "yield" is not valid and instead
-# we must use "yield None". Bug was not adding "None"
-
-def yield_bug():
-    yield None
-    return

test/simple_source/bug25/04_try_else_confuse.py

@@ -1,19 +0,0 @@
-# From 2.4 test_sax.py
-# Bug was distinguishing try from try/else
-
-def verify_empty_attrs():
-    gvqk = 3
-    try:
-        gvk = 1/0
-    except ZeroDivisionError:
-        gvk = 1
-
-    try:
-        gvqk = 0
-    except KeyError:
-        gvqk = 1
-
-    # If try/else was used above the return will be 4
-    return gvk + gvqk
-
-assert 1 == verify_empty_attrs()

test/simple_source/bug27+/05_while_elif.py

@@ -0,0 +1,16 @@
+# Bug in Python 2.7. Bug is bytecode for while loop having
+# two consecutive JUMP_BACKS at the end of 'elif' and 'while'
+# to the same place
+def PreprocessConditionalStatement(self, IfList, ReplacedLine):
+    while self:
+        if self.__Token:
+            x = 1
+        elif not IfList:
+            if self <= 2:
+                continue
+            RegionSizeGuid = 3
+            if not RegionSizeGuid:
+                RegionLayoutLine = 5
+                continue
+            RegionLayoutLine = self.CurrentLineNumber
+    return 1

test/simple_source/bug27+/06_raise.py

@@ -0,0 +1,22 @@
+# Bug in Python 2.7 is code creating a (useless) JUMP_ABSOLUTE to the instruction right after
+# the "raise" which causes the
+
+# RUNNABLE!
+def testit(a, b):
+    if a:
+        if not b:
+            raise AssertionError("test JUMP_ABSOLUTE to next instruction")
+
+def testit2(a, b):
+    if a:
+        if not b:
+            raise AssertionError("test with dead code after raise")
+            x = 10
+
+testit(False, True)
+testit(False, False)
+testit(True, True)
+
+testit2(False, True)
+testit2(False, False)
+testit2(True, True)

test/simple_source/bug31/01_try_else.py

@@ -0,0 +1,24 @@
+# Bug based on 2.7 test_itertools.py but mis-decompiled in Python 3.x bytecode
+# The bug is confusing "try/else" with "try" as a result of the loop which causes
+# the end of the except to jump back to the beginning of the loop, outside of the
+# try. In 3.x we not distinguising this jump out of the loop with a jump to the
+# end of the "try".
+
+# RUNNABLE!
+def testit(stmts):
+
+    # Bug was confusing When the except jumps back to the beginning of the block
+    # to the beginning of this for loop
+    x = 1
+    results = []
+    for stmt in stmts:
+        try:
+            x = eval(stmt)
+        except SyntaxError:
+            results.append(1)
+        else:
+            results.append(x)
+    return results
+
+results = testit(["1 + 2", "1 +"])
+assert results == [3, 1], "try with else failed"

test/simple_source/bug36/00_if_elif.py

@@ -0,0 +1,15 @@
+# Python 3 bug in not detecting the end bounds of if elif.
+def testit(b):
+    if b == 1:
+        a = 1
+    elif b == 2:
+        a = 2
+    else:
+        a = 4
+
+    return a
+
+for x in (1, 2, 4):
+    x = testit(x)
+    assert x is not None, "Should have returned a value, not None"
+    assert x == x

test/simple_source/bug36/00_return_return_bug.py

@@ -0,0 +1,10 @@
+# Bug in 3.6 and above.
+#Not detecting 2nd return is outside of
+# if/then. Fix was to ensure COME_FROM
+def return_return_bug(foo):
+    if foo =='say_hello':
+        return "hello"
+    return "world"
+
+assert return_return_bug('say_hello') == 'hello'
+assert return_return_bug('world') == 'world'

test/simple_source/bug36/01_if_and_if_bug.py

@@ -0,0 +1,16 @@
+# Bug in 3.6 was not taking "else" branch after compond "if"
+# In earlier versions we had else detection needed here.
+
+def f(a, b, c):
+    if a and b:
+        x = 1
+    else:
+        x = 2
+    if c:
+        x = 3
+    return(x)
+
+assert f(True, True, True) == 3
+assert f(True, True, False) == 1
+assert f(True, False, True) == 3
+assert f(True, False, False) == 2

test/simple_source/bug36/03_if_try.py

@@ -0,0 +1,8 @@
+# The bug in python 3.6+ was in parsing that we
+# add END_IF_THEN and using that inside "return results"
+def whcms_license_info(md5hash, datahash, results):
+    if md5hash == datahash:
+        try:
+            return md5hash
+        except:
+            return results

test/simple_source/bug36/07_kwargs.py

@@ -0,0 +1,9 @@
+# Bug in Python 3.6 and 3.7 was getting comma before **kw
+
+def fn(arg, *, kwarg='test', **kw):
+    assert arg == 1
+    assert kwarg == 'testing'
+    assert kw['foo'] == 'bar'
+
+
+fn(1, kwarg='testing', foo='bar')

test/simple_source/bug37/01_chained_compare.py

@@ -0,0 +1,19 @@
+# From Python 3.7 pickle.py
+# Bug was different code generation for chained comparisons than prior Python versions
+
+
+def chained_compare_a(protocol):
+    if not 0 <= protocol <= 7:
+        raise ValueError("pickle protocol must be <= %d" % 7)
+
+def chained_compare_b(a, obj):
+    if a:
+        if -0x80000000 <= obj <= 0x7fffffff:
+            return 5
+
+chained_compare_a(3)
+try:
+    chained_compare_a(8)
+except ValueError:
+    pass
+chained_compare_b(True, 0x0)

test/stdlib/runtests.sh

@@ -43,10 +43,10 @@ case $PYVERSION in
     2.5)
 	SKIP_TESTS=(
 	    [test_contextlib.py]=1 # Syntax error - look at
-	    [test_dis.py]=1        # We change line numbers - duh!
-	    [test_grammar.py]=1    # Too many stmts. Handle large stmts
-	    [test_grp.py]=1        # Long test - might work Control flow?
-	    [test_pdb.py]=1        # Line-number specific
+	    [test_dis.py]=1   # We change line numbers - duh!
+	    [test_grammar.py]=1  # Too many stmts. Handle large stmts
+	    [test_grp.py]=1      # Long test - might work Control flow?
+	    [test_pdb.py]=1      # Line-number specific
 	    [test_pwd.py]=1 # Long test - might work? Control flow?
 	    [test_queue.py]=1 # Control flow?
 	    [test_re.py]=1 # Probably Control flow?
@@ -115,13 +115,20 @@ case $PYVERSION in
 	                          # See test/simple_source/bug27+/05_not_unconditional.py
 	    [test_memoryio.py]=1 # FIX
 	    [test_multiprocessing.py]=1 # On uncompyle2, taks 24 secs
-	    [test_pep352.py]=1 # ?
+	    [test_pep352.py]=1  # ?
+	    [test_posix.py]=1   # Bug in try-else detection inside test_initgroups()
+	                        # Deal with when we have better flow-control detection
+	    [test_pwd.py]=1     # Takes too long
+	    [test_pty.py]=1
+	    [test_queue.py]=1   # Control flow?
+	    [test_re.py]=1      # Probably Control flow?
 	    [test_select.py]=1  # Runs okay but takes 11 seconds
 	    [test_socket.py]=1  # Runs ok but takes 22 seconds
 	    [test_subprocess.py]=1 # Runs ok but takes 22 seconds
 	    [test_sys_settrace.py]=1 # Line numbers are expected to be different
 	    [test_strtod.py]=1 # FIX
 	    [test_traceback.py]=1 # Line numbers change - duh.
+	    [test_types.py]=1     # try/else confusions
 	    [test_unicode.py]=1  # Too long to run 11 seconds
 	    [test_xpickle.py]=1 # Runs ok but takes 72 seconds
 	    [test_zipfile64.py]=1  # Runs ok but takes 204 seconds

test/test_pyenvlib.py

@@ -20,7 +20,8 @@ Step 2: Run the test:
 	  test_pyenvlib --mylib --verify # decompile verify 'mylib'
 """
 
-from uncompyle6 import main, PYTHON3
+from __future__ import print_function
+
 import os, time, re, shutil, sys
 from fnmatch import fnmatch
 

test/test_pythonlib.py

@@ -27,6 +27,8 @@ Step 2: Run the test:
   test_pythonlib.py --mylib --verify # decompile verify 'mylib'
 """
 
+from __future__ import print_function
+
 import getopt, os, py_compile, sys, shutil, tempfile, time
 
 from uncompyle6 import PYTHON_VERSION
@@ -128,10 +130,8 @@ def do_tests(src_dir, obj_patterns, target_dir, opts):
     if opts['do_compile']:
         compiled_version = opts['compiled_version']
         if compiled_version and PYTHON_VERSION != compiled_version:
-            sys.stderr.write("Not compiling: "
-                             "desired Python version is %s "
-                             "but we are running %s" %
-                             (compiled_version, PYTHON_VERSION))
+            print("Not compiling: desired Python version is %s but we are running %s" %
+                  (compiled_version, PYTHON_VERSION), file=sys.stderr)
         else:
             for root, dirs, basenames in os.walk(src_dir):
                 file_matches(files, root, basenames, PY)
@@ -149,8 +149,8 @@ def do_tests(src_dir, obj_patterns, target_dir, opts):
         file_matches(files, dirname, basenames, obj_patterns)
 
     if not files:
-        sys.stderr.write("Didn't come up with any files to test! "
-                         "Try with --compile?")
+        print("Didn't come up with any files to test! Try with --compile?",
+              file=sys.stderr)
         exit(1)
 
     os.chdir(cwd)
@@ -164,9 +164,9 @@ def do_tests(src_dir, obj_patterns, target_dir, opts):
         except ValueError:
             pass
 
-    print time.ctime()
-    print 'Source directory: ', src_dir
-    print 'Output directory: ', target_dir
+    print(time.ctime())
+    print('Source directory: ', src_dir)
+    print('Output directory: ', target_dir)
     try:
         _, _, failed_files, failed_verify = \
           main(src_dir, target_dir, files, [],
@@ -242,13 +242,14 @@ if __name__ == '__main__':
         if os.path.isdir(src_dir):
             checked_dirs.append([src_dir, pattern, target_dir])
         else:
-            sys.stderr.write("Can't find directory %s. Skipping" % src_dir)
+            print("Can't find directory %s. Skipping" % src_dir,
+                  file=sys.stderr)
             continue
         last_compile_version = compiled_version
         pass
 
     if not checked_dirs:
-        sys.stderr.write("No directories found to check\n")
+        print("No directories found to check", file=sys.stderr)
         sys.exit(1)
 
     test_opts['compiled_version'] = last_compile_version

tox.ini

@@ -3,7 +3,7 @@
 [flake8]
 exclude = .tox,./build,./trepan/processor/command/tmp
 filename = *.py
-ignore = C901,E113,E121,E122,E123,E124,E125,E126,E127,E128,E129,E201,E202,E203,E221,E222,E225,E226,E241,E242,E251,E261,E271,E272,E302,E401,E501,F401,E701,E702
+ignore = C901,E113,E121,E122,E123,E124,E125,E126,E127,E128,E129,E201,E202,E203,E221,E222,E225,E226,E241,E242,E251,E261,E271,E272,E302,E401,E402,E501,F401,E701,E702
 
 [tox]
 envlist = py27, py34, pypy

uncompyle6/bin/pydisassemble.py

@@ -3,6 +3,7 @@
 #
 # Copyright (c) 2015-2016, 2018 by Rocky Bernstein <rb@dustyfeet.com>
 #
+from __future__ import print_function
 import sys, os, getopt
 
 from uncompyle6.disas import disassemble_file
@@ -12,8 +13,8 @@ program, ext = os.path.splitext(os.path.basename(__file__))
 
 __doc__ = """
 Usage:
-  %s [OPTIONS]... FILE
-  %s [--help | -h | -V | --version]
+  {0} [OPTIONS]... FILE
+  {0} [--help | -h | -V | --version]
 
 Disassemble FILE with the instruction mangling that is done to
 assist uncompyle6 in parsing the instruction stream. For example
@@ -22,9 +23,9 @@ BUILD_LIST have arguement counts appended to the instruction name, and
 COME_FROM instructions are inserted into the instruction stream.
 
 Examples:
-  %s foo.pyc
-  %s foo.py    # same thing as above but find the file
-  %s foo.pyc bar.pyc  # disassemble foo.pyc and bar.pyc
+  {0} foo.pyc
+  {0} foo.py    # same thing as above but find the file
+  {0} foo.pyc bar.pyc  # disassemble foo.pyc and bar.pyc
 
 See also `pydisasm' from the `xdis' package.
 
@@ -32,7 +33,7 @@ Options:
   -V | --version     show version and stop
   -h | --help        show this message
 
-""" % ((program,) * 5)
+""".format(program)
 
 PATTERNS = ('*.pyc', '*.pyo')
 
@@ -41,15 +42,15 @@ def main():
 Type -h for for full help.""" % program
 
     if len(sys.argv) == 1:
-        sys.stderr.write("No file(s) given\n")
-        sys.stderr.write(Usage_short)
+        print("No file(s) given", file=sys.stderr)
+        print(Usage_short, file=sys.stderr)
         sys.exit(1)
 
     try:
         opts, files = getopt.getopt(sys.argv[1:], 'hVU',
                                     ['help', 'version', 'uncompyle6'])
-    except getopt.GetoptError(e):
-        sys.stderr.write('%s: %s' % (os.path.basename(sys.argv[0]), e))
+    except getopt.GetoptError as e:
+        print('%s: %s' % (os.path.basename(sys.argv[0]), e),  file=sys.stderr)
         sys.exit(-1)
 
     for opt, val in opts:
@@ -61,14 +62,15 @@ Type -h for for full help.""" % program
             sys.exit(0)
         else:
             print(opt)
-            sys.stderr.write(Usage_short)
+            print(Usage_short, file=sys.stderr)
             sys.exit(1)
 
     for file in files:
         if os.path.exists(files[0]):
             disassemble_file(file, sys.stdout)
         else:
-            sys.stderr.write("Can't read %s - skipping\n" % files[0])
+            print("Can't read %s - skipping" % files[0],
+                  file=sys.stderr)
             pass
         pass
     return

uncompyle6/bin/uncompile.py

@@ -4,6 +4,7 @@
 # Copyright (c) 2015-2017 by Rocky Bernstein
 # Copyright (c) 2000-2002 by hartmut Goebel <h.goebel@crazy-compilers.com>
 #
+from __future__ import print_function
 import sys, os, getopt, time
 
 program = 'uncompyle6'
@@ -68,9 +69,13 @@ def usage():
 
 
 def main_bin():
-    if not (sys.version_info[0:2] in ((2, 4), (2, 5), (2, 6), (2, 7))):
-        sys.stderr.write('Error: this branch of %s requires Python 2.4, 2.5, 2.6 or 2.7'
-                         % program)
+    if not (sys.version_info[0:2] in ((2, 6), (2, 7), (3, 0),
+                                      (3, 1), (3, 2), (3, 3),
+                                      (3, 4), (3, 5), (3, 6),
+                                      (3, 7)
+        )):
+        print('Error: %s requires Python 2.6-3.7' % program,
+              file=sys.stderr)
         sys.exit(-1)
 
     do_verify = recurse_dirs = False
@@ -88,8 +93,8 @@ def main_bin():
                                     'fragments verify verify-run version '
                                     'weak-verify '
                                     'showgrammar'.split(' '))
-    except getopt.GetoptError(e):
-        sys.stderr.write('%s: %s\n' % (os.path.basename(sys.argv[0]), e))
+    except getopt.GetoptError as e:
+        print('%s: %s' % (os.path.basename(sys.argv[0]), e),  file=sys.stderr)
         sys.exit(-1)
 
     options = {}
@@ -132,7 +137,7 @@ def main_bin():
         elif opt in ('--recurse', '-r'):
             recurse_dirs = True
         else:
-            sys.stderr.write(opt)
+            print(opt, file=sys.stderr)
             usage()
 
     # expand directory if specified
@@ -157,7 +162,7 @@ def main_bin():
         files = [f[sb_len:] for f in files]
 
     if not files:
-        sys.stderr.write("No files given\n")
+        print("No files given", file=sys.stderr)
         usage()
 
     if outfile == '-':
@@ -172,7 +177,7 @@ def main_bin():
 
     if numproc <= 1:
         try:
-            result = main(src_base, out_base, files, codes, outfile,
+            result = main(src_base, out_base, files, None, outfile,
                           **options)
             result = list(result) + [options.get('do_verify', None)]
             if len(files) > 1:
@@ -207,7 +212,7 @@ def main_bin():
                     if f is None:
                         break
                     (t, o, f, v) = \
-                      main(src_base, out_base, [f], codes, outfile, **options)
+                      main(src_base, out_base, [f], None, outfile, **options)
                     tot_files += t
                     okay_files += o
                     failed_files += f

uncompyle6/disas.py

@@ -1,4 +1,4 @@
-#  Copyright (c) 2015-2016, 2818 by Rocky Bernstein
+#  Copyright (c) 2015-2016, 2818-2019 by Rocky Bernstein
 #  Copyright (c) 2005 by Dan Pascu <dan@windowmaker.org>
 #  Copyright (c) 2000-2002 by hartmut Goebel <h.goebel@crazy-compilers.com>
 #  Copyright (c) 1999 John Aycock
@@ -29,6 +29,8 @@ Second, we need structured instruction information for the
 want to run on earlier Python versions.
 """
 
+from __future__ import print_function
+
 import sys
 from collections import deque
 
@@ -47,9 +49,10 @@ def disco(version, co, out=None, is_pypy=False):
 
     # store final output stream for case of error
     real_out = out or sys.stdout
-    real_out.write('# Python %s\n' % version)
+    print('# Python %s' % version, file=real_out)
     if co.co_filename:
-        real_out.write('# Embedded file name: %s\n' % co.co_filename)
+        print('# Embedded file name: %s' % co.co_filename,
+              file=real_out)
 
     scanner = get_scanner(version, is_pypy=is_pypy)
 
@@ -61,30 +64,31 @@ def disco_loop(disasm, queue, real_out):
     while len(queue) > 0:
         co = queue.popleft()
         if co.co_name != '<module>':
-            real_out.write('\n# %s line %d of %s\n' %
-                      (co.co_name, co.co_firstlineno, co.co_filename))
+            print('\n# %s line %d of %s' %
+                      (co.co_name, co.co_firstlineno, co.co_filename),
+                      file=real_out)
         tokens, customize = disasm(co)
         for t in tokens:
             if iscode(t.pattr):
                 queue.append(t.pattr)
             elif iscode(t.attr):
                 queue.append(t.attr)
-            real_out.write(t)
+            print(t, file=real_out)
             pass
         pass
 
-def disassemble_fp(fp, outstream=None):
-    """
-    disassemble Python byte-code from an open file
-    """
-    (version, timestamp, magic_int, co, is_pypy,
-     source_size) = load_from_fp(fp)
-    if type(co) == list:
-        for con in co:
-            disco(version, con, outstream)
-    else:
-        disco(version, co, outstream, is_pypy=is_pypy)
-    co = None
+# def disassemble_fp(fp, outstream=None):
+#     """
+#     disassemble Python byte-code from an open file
+#     """
+#     (version, timestamp, magic_int, co, is_pypy,
+#      source_size) = load_from_fp(fp)
+#     if type(co) == list:
+#         for con in co:
+#             disco(version, con, outstream)
+#     else:
+#         disco(version, co, outstream, is_pypy=is_pypy)
+#     co = None
 
 def disassemble_file(filename, outstream=None):
     """
@@ -116,5 +120,6 @@ def _test():
         fn = sys.argv[1]
     disassemble_file(fn)
 
+
 if __name__ == "__main__":
     _test()

uncompyle6/linenumbers.py

@@ -25,7 +25,7 @@ def line_number_mapping(pyc_filename, src_filename):
      source_size) = load_module(pyc_filename)
     try:
         code2 = load_file(src_filename)
-    except SyntaxError, e:
+    except SyntaxError as e:
         return str(e)
 
     queue = deque([code1, code2])

uncompyle6/main.py

@@ -12,10 +12,10 @@
 #
 #  You should have received a copy of the GNU General Public License
 #  along with this program.  If not, see <http://www.gnu.org/licenses/>.
+from __future__ import print_function
+import datetime, os, subprocess, sys, tempfile
 
-import datetime, os, subprocess, sys
-
-from uncompyle6 import verify, IS_PYPY
+from uncompyle6 import verify, IS_PYPY, PYTHON_VERSION
 from xdis.code import iscode
 from xdis.magics import sysinfo2float
 from uncompyle6.disas import check_object_path
@@ -39,7 +39,10 @@ def _get_outstream(outfile):
         os.makedirs(dir)
     except OSError:
         pass
-    return open(outfile, 'wb')
+    if PYTHON_VERSION < 3.0:
+        return open(outfile, mode='wb')
+    else:
+        return open(outfile, mode='w', encoding='utf-8')
 
 def decompile(
         bytecode_version, co, out=None, showasm=None, showast=False,
@@ -66,35 +69,20 @@ def decompile(
 
     assert iscode(co)
 
-    if is_pypy:
-        co_pypy_str = 'PyPy '
-    else:
-        co_pypy_str = ''
-
-    if IS_PYPY:
-        run_pypy_str = 'PyPy '
-    else:
-        run_pypy_str = ''
-
-    if magic_int:
-        m = str(magic_int)
-    else:
-        m = ""
-
+    co_pypy_str = 'PyPy ' if is_pypy else ''
+    run_pypy_str = 'PyPy ' if IS_PYPY else ''
     sys_version_lines = sys.version.split('\n')
     write('# uncompyle6 version %s\n'
           '# %sPython bytecode %s%s\n# Decompiled from: %sPython %s' %
           (VERSION, co_pypy_str, bytecode_version,
-               " (%s)" % m, run_pypy_str,
-           '\n# '.join(sys_version_lines)))
+               " (%s)" % str(magic_int) if magic_int else "",
+          run_pypy_str, '\n# '.join(sys_version_lines)))
     if co.co_filename:
         write('# Embedded file name: %s' % co.co_filename,)
     if timestamp:
-        write('# Compiled at: %s' %
-              datetime.datetime.fromtimestamp(timestamp))
+        write('# Compiled at: %s' % datetime.datetime.fromtimestamp(timestamp))
     if source_size:
-        real_out.write('# Size of source mod 2**32: %d bytes\n' %
-                       source_size)
+        write('# Size of source mod 2**32: %d bytes' % source_size)
 
     debug_opts = {
         'asm': showasm,
@@ -127,7 +115,7 @@ def decompile(
                                   is_pypy=is_pypy)
             pass
         return deparsed
-    except pysource.SourceWalkerError, e:
+    except pysource.SourceWalkerError as e:
         # deparsing failed
         raise pysource.SourceWalkerError(str(e))
 
@@ -169,9 +157,11 @@ def main(in_base, out_base, files, codes, outfile=None,
     """
     in_base	base directory for input files
     out_base	base directory for output files (ignored when
-    files	list of filenames to be uncompyled (relative to src_base)
+    files	list of filenames to be uncompyled (relative to in_base)
     outfile	write output to this filename (overwrites out_base)
 
+    Note: `codes` is not use. Historical compatability?
+
     For redirecting output to
     - <filename>		outfile=<filename> (out_base is ignored)
     - files below out_base	out_base=...
@@ -207,10 +197,17 @@ def main(in_base, out_base, files, codes, outfile=None,
                     prefix = prefix[:-len('.py')]
 
                 # Unbuffer output if possible
-                if sys.stdout.isatty():
-                    buffering = -1
+                buffering = -1 if sys.stdout.isatty() else 0
+                if PYTHON_VERSION >= 3.5:
+                    t = tempfile.NamedTemporaryFile(mode='w+b',
+                                                    buffering=buffering,
+                                                    suffix='.py',
+                                                    prefix=prefix)
                 else:
-                    buffering = 0
+                    t = tempfile.NamedTemporaryFile(mode='w+b',
+                                                    suffix='.py',
+                                                    prefix=prefix)
+                current_outfile = t.name
                 sys.stdout = os.fdopen(sys.stdout.fileno(), 'w', buffering)
                 tee = subprocess.Popen(["tee", current_outfile],
                                        stdin=subprocess.PIPE)
@@ -249,9 +246,9 @@ def main(in_base, out_base, files, codes, outfile=None,
                     pass
                 pass
             tot_files += 1
-        except (ValueError, SyntaxError, ParserError, pysource.SourceWalkerError):
+        except (ValueError, SyntaxError, ParserError, pysource.SourceWalkerError) as e:
             sys.stdout.write("\n")
-            sys.stderr.write("# file %s\n" % (infile))
+            sys.stderr.write("\n# file %s\n# %s\n" % (infile, e))
             failed_files += 1
             tot_files += 1
         except KeyboardInterrupt:
@@ -289,16 +286,13 @@ def main(in_base, out_base, files, codes, outfile=None,
                         else:
                             okay_files += 1
                             pass
-                    except verify.VerifyCmpError, e:
+                    except verify.VerifyCmpError as e:
                         print(e)
                         verify_failed_files += 1
                         os.rename(current_outfile, current_outfile + '_unverified')
                         sys.stderr.write("### Error Verifying %s\n" % filename)
                         sys.stderr.write(str(e) + "\n")
                         if not outfile:
-                            sys.stderr.write("### Error Verifiying %s" %
-                                            filename)
-                            sys.stderr.write(e)
                             if raise_on_error:
                                 raise
                             pass
@@ -308,23 +302,31 @@ def main(in_base, out_base, files, codes, outfile=None,
                     okay_files += 1
                 pass
             elif do_verify:
-                sys.stderr.write("\n### uncompile successful, "
-                                 "but no file to compare against")
+                sys.stderr.write("\n### uncompile successful, but no file to compare against\n")
                 pass
             else:
                 okay_files += 1
                 if not current_outfile:
                     mess = '\n# okay decompiling'
                     # mem_usage = __memUsage()
-                    print mess, infile
+                    print(mess, infile)
         if current_outfile:
             sys.stdout.write("%s\r" %
                              status_msg(do_verify, tot_files, okay_files, failed_files,
                                         verify_failed_files, do_verify))
-            sys.stdout.flush()
+            try:
+                # FIXME: Something is weird with Pypy here
+                sys.stdout.flush()
+            except:
+                pass
     if current_outfile:
         sys.stdout.write("\n")
-        sys.stdout.flush()
+        try:
+            # FIXME: Something is weird with Pypy here
+            sys.stdout.flush()
+        except:
+            pass
+        pass
     return (tot_files, okay_files, failed_files, verify_failed_files)
 
 

uncompyle6/parser.py

@@ -19,6 +19,8 @@
 Common uncompyle6 parser routines.
 """
 
+from __future__ import print_function
+
 import sys
 
 from xdis.code import iscode
@@ -35,7 +37,9 @@ class ParserError(Exception):
         return "Parse error at or near `%r' instruction at offset %s\n" % \
                (self.token, self.offset)
 
-nop_func = lambda self, args: None
+
+def nop_func(self, args):
+    return None
 
 class PythonParser(GenericASTBuilder):
 
@@ -184,7 +188,7 @@ class PythonParser(GenericASTBuilder):
                     indent = '   '
                 else:
                     indent = '-> '
-                print "%s%s" % (indent, instructions[i])
+                print("%s%s" % (indent, instructions[i]))
             raise ParserError(err_token, err_token.offset)
         else:
             raise ParserError(None, -1)
@@ -790,6 +794,7 @@ def python_parser(version, co, out=sys.stdout, showasm=False,
     p = get_python_parser(version, parser_debug)
     return parse(p, tokens, customize)
 
+
 if __name__ == '__main__':
     def parse_test(co):
         from uncompyle6 import PYTHON_VERSION, IS_PYPY
@@ -797,4 +802,4 @@ if __name__ == '__main__':
         ast = python_parser(PYTHON_VERSION, co, showasm=True, is_pypy=IS_PYPY)
         print(ast)
         return
-    # parse_test(parse_test.__code__)
+    parse_test(parse_test.__code__)

uncompyle6/parsers/parse2.py

@@ -25,6 +25,8 @@ If we succeed in creating a parse tree, then we have a Python program
 that a later phase can turn into a sequence of ASCII text.
 """
 
+from __future__ import print_function
+
 from uncompyle6.parser import PythonParser, PythonParserSingle, nop_func
 from uncompyle6.parsers.treenode import SyntaxTree
 from spark_parser import DEFAULT_DEBUG as PARSER_DEFAULT_DEBUG
@@ -156,8 +158,10 @@ class Python2Parser(PythonParser):
         try_except      ::= SETUP_EXCEPT suite_stmts_opt POP_BLOCK
                             except_handler COME_FROM
 
+        # Note: except_stmts may have many jumps after END_FINALLY
         except_handler  ::= JUMP_FORWARD COME_FROM except_stmts
-                            END_FINALLY COME_FROM
+                            END_FINALLY come_froms
+
         except_handler  ::= jmp_abs COME_FROM except_stmts
                              END_FINALLY
 
@@ -529,14 +533,11 @@ class Python2Parser(PythonParser):
         # Dead code testing...
         # if lhs == 'while1elsestmt':
         #     from trepan.api import debug; debug()
-
         if lhs in ('aug_assign1', 'aug_assign2') and ast[0] and ast[0][0] in ('and', 'or'):
             return True
         elif lhs in ('raise_stmt1',):
-            # We will assme 'LOAD_ASSERT' will be handled by an assert grammar rule
-            return (tokens[first] == 'LOAD_ASSERT' and
-                    (last >= len(tokens) or tokens[last] not in
-                     ('COME_FROM', 'JUMP_BACK','JUMP_FORWARD')))
+            # We will assume 'LOAD_ASSERT' will be handled by an assert grammar rule
+            return (tokens[first] == 'LOAD_ASSERT' and (last >= len(tokens)))
         elif rule == ('or', ('expr', 'jmp_true', 'expr', '\\e_come_from_opt')):
             expr2 = ast[2]
             return expr2 == 'expr' and expr2[0] == 'LOAD_ASSERT'

uncompyle6/parsers/parse24.py

@@ -87,6 +87,15 @@ class Python24Parser(Python25Parser):
             l = len(tokens)
             if 0 <= l < len(tokens):
                 return not int(tokens[first].pattr) == tokens[last].offset
+        elif lhs == 'try_except':
+            if last == len(tokens):
+                last -= 1
+            if tokens[last] != 'COME_FROM' and tokens[last-1] == 'COME_FROM':
+                last -= 1
+            return (tokens[last] == 'COME_FROM'
+                    and tokens[last-1] == 'END_FINALLY'
+                    and tokens[last-2] == 'POP_TOP'
+                    and tokens[last-3].kind != 'JUMP_FORWARD')
 
         return False
 

uncompyle6/parsers/parse25.py

@@ -19,9 +19,6 @@ class Python25Parser(Python26Parser):
 
         return_if_stmt ::= ret_expr  RETURN_END_IF JUMP_BACK
 
-        # We have no jumps to jumps, so no "come_froms" but a single "COME_FROM"
-        ifelsestmt ::= testexpr c_stmts_opt jf_cf_pop else_suite COME_FROM
-
         # Python 2.6 uses ROT_TWO instead of the STORE_xxx
         # withas is allowed as a "from future" in 2.5
         # 2.6 and 2.7 do something slightly different
@@ -39,8 +36,6 @@ class Python25Parser(Python26Parser):
         # loop. FIXME: should "come_froms" below be a single COME_FROM?
         tryelsestmt    ::= SETUP_EXCEPT suite_stmts_opt POP_BLOCK
                            except_handler else_suite come_froms
-        tryelsestmtl   ::= SETUP_EXCEPT suite_stmts_opt POP_BLOCK
-                            except_handler else_suitel
 
         # Python 2.6 omits the LOAD_FAST DELETE_FAST below
         # withas is allowed as a "from future" in 2.5
@@ -59,9 +54,6 @@ class Python25Parser(Python26Parser):
     def customize_grammar_rules(self, tokens, customize):
         # Remove grammar rules inherited from Python 2.6 or Python 2
         self.remove_rules("""
-        # No jump to jumps in 2.4 so we have a single "COME_FROM", not "come_froms"
-        ifelsestmt    ::= testexpr c_stmts_opt jf_cf_pop else_suite come_froms
-
         setupwith  ::= DUP_TOP LOAD_ATTR ROT_TWO LOAD_ATTR CALL_FUNCTION_0 POP_TOP
         withstmt   ::= expr setupwith SETUP_FINALLY suite_stmts_opt
                        POP_BLOCK LOAD_CONST COME_FROM WITH_CLEANUP END_FINALLY
@@ -93,15 +85,17 @@ class Python25Parser(Python26Parser):
         super(Python25Parser, self).customize_grammar_rules(tokens, customize)
         if self.version == 2.5:
             self.check_reduce['try_except'] = 'tokens'
+        self.check_reduce['aug_assign1'] = 'AST'
 
-    ## Don't need this for 2.5 yet..
-    # def reduce_is_invalid(self, rule, ast, tokens, first, last):
-    #     invalid = super(Python25Parser,
-    #                     self).reduce_is_invalid(rule, ast,
-    #                                             tokens, first, last)
-    #     if invalid or tokens is None:
-    #         return invalid
-    #     return False
+    def reduce_is_invalid(self, rule, ast, tokens, first, last):
+        invalid = super(Python25Parser,
+                        self).reduce_is_invalid(rule, ast,
+                                                tokens, first, last)
+        if invalid or tokens is None:
+            return invalid
+        if rule == ('aug_assign1', ('expr', 'expr', 'inplace_op', 'store')):
+            return ast[0][0] == 'and'
+        return False
 
 
 class Python25ParserSingle(Python26Parser, PythonParserSingle):

uncompyle6/parsers/parse27.py

@@ -1,9 +1,10 @@
-#  Copyright (c) 2016-2018 Rocky Bernstein
+#  Copyright (c) 2016-2019 Rocky Bernstein
 #  Copyright (c) 2005 by Dan Pascu <dan@windowmaker.org>
 #  Copyright (c) 2000-2002 by hartmut Goebel <hartmut@goebel.noris.de>
 
 from spark_parser import DEFAULT_DEBUG as PARSER_DEFAULT_DEBUG
-from uncompyle6.parser import PythonParserSingle
+from xdis import next_offset
+from uncompyle6.parser import PythonParserSingle, nop_func
 from uncompyle6.parsers.parse2 import Python2Parser
 
 class Python27Parser(Python2Parser):
@@ -155,7 +156,13 @@ class Python27Parser(Python2Parser):
 
         while1stmt        ::= SETUP_LOOP returns bp_come_from
         while1stmt        ::= SETUP_LOOP l_stmts_opt JUMP_BACK POP_BLOCK COME_FROM
+
         whilestmt         ::= SETUP_LOOP testexpr l_stmts_opt JUMP_BACK POP_BLOCK _come_froms
+
+        # Should this be JUMP_BACK+ ?
+        # JUMP_BACK should all be to the same location
+        whilestmt         ::= SETUP_LOOP testexpr l_stmts_opt JUMP_BACK JUMP_BACK POP_BLOCK _come_froms
+
         while1elsestmt    ::= SETUP_LOOP l_stmts JUMP_BACK POP_BLOCK
                               else_suitel COME_FROM
         whileelsestmt     ::= SETUP_LOOP testexpr l_stmts_opt JUMP_BACK POP_BLOCK
@@ -196,6 +203,13 @@ class Python27Parser(Python2Parser):
                            POP_BLOCK LOAD_CONST COME_FROM suite_stmts_opt
                            END_FINALLY
         """)
+        if 'PyPy' in customize:
+            # PyPy-specific customizations
+            self.addRule("""
+                        return_if_stmt ::= ret_expr RETURN_END_IF come_froms
+                        """, nop_func)
+
+
         super(Python27Parser, self).customize_grammar_rules(tokens, customize)
         self.check_reduce['and'] = 'AST'
         # self.check_reduce['or'] = 'AST'
@@ -203,14 +217,17 @@ class Python27Parser(Python2Parser):
         self.check_reduce['list_if_not'] = 'AST'
         self.check_reduce['list_if'] = 'AST'
         self.check_reduce['conditional_true'] = 'AST'
+        self.check_reduce['whilestmt'] = 'tokens'
         return
 
     def reduce_is_invalid(self, rule, ast, tokens, first, last):
         invalid = super(Python27Parser,
                         self).reduce_is_invalid(rule, ast,
                                                 tokens, first, last)
+
         if invalid:
             return invalid
+
         if rule == ('and', ('expr', 'jmp_false', 'expr', '\\e_come_from_opt')):
             # Test that jmp_false jumps to the end of "and"
             # or that it jumps to the same place as the end of "and"
@@ -220,6 +237,12 @@ class Python27Parser(Python2Parser):
                         tokens[last].pattr == jmp_false.pattr)
         elif rule[0] == ('raise_stmt1'):
             return ast[0] == 'expr' and ast[0][0] == 'or'
+        elif rule[0] in ('assert', 'assert2'):
+            jump_inst = ast[1][0]
+            jump_target = jump_inst.attr
+            return not (last >= len(tokens)
+                        or jump_target == tokens[last].offset
+                        or jump_target == next_offset(ast[-1].op, ast[-1].opc, ast[-1].offset))
         elif rule == ('list_if_not', ('expr', 'jmp_true', 'list_iter')):
             jump_inst = ast[1][0]
             jump_offset = jump_inst.attr
@@ -235,6 +258,16 @@ class Python27Parser(Python2Parser):
             jmp_target = jmp_true.offset + jmp_true.attr + 3
             return not (jmp_target == tokens[last].offset or
                         tokens[last].pattr == jmp_true.pattr)
+
+        elif (rule[0] == 'whilestmt' and
+              rule[1][0:-2] ==
+                      ('SETUP_LOOP', 'testexpr', 'l_stmts_opt',
+                       'JUMP_BACK', 'JUMP_BACK')):
+            # Make sure that the jump backs all go to the same place
+            i = last-1
+            while (tokens[i] != 'JUMP_BACK'):
+                i -= 1
+            return tokens[i].attr != tokens[i-1].attr
         # elif rule[0] == ('conditional_true'):
         #     # FIXME: the below is a hack: we check expr for
         #     # nodes that could have possibly been a been a Boolean.
@@ -263,11 +296,10 @@ if __name__ == '__main__':
             """.split()))
         remain_tokens = set(tokens) - opcode_set
         import re
-        remain_tokens = set([re.sub('_\d+$', '', t)
+        remain_tokens = set([re.sub(r'_\d+$', '', t)
                              for t in remain_tokens])
         remain_tokens = set([re.sub('_CONT$', '', t)
                              for t in remain_tokens])
         remain_tokens = set(remain_tokens) - opcode_set
         print(remain_tokens)
-        p.check_grammar()
-        p.dump_grammar()
+        # p.dump_grammar()

uncompyle6/parsers/parse3.py

@@ -1,4 +1,4 @@
-#  Copyright (c) 2015-2018 Rocky Bernstein
+#  Copyright (c) 2015-2019 Rocky Bernstein
 #  Copyright (c) 2005 by Dan Pascu <dan@windowmaker.org>
 #  Copyright (c) 2000-2002 by hartmut Goebel <h.goebel@crazy-compilers.com>
 #  Copyright (c) 1999 John Aycock
@@ -26,11 +26,11 @@ If we succeed in creating a parse tree, then we have a Python program
 that a later phase can turn into a sequence of ASCII text.
 """
 
+from uncompyle6.scanners.tok import Token
 from uncompyle6.parser import PythonParser, PythonParserSingle, nop_func
 from uncompyle6.parsers.treenode import SyntaxTree
 from spark_parser import DEFAULT_DEBUG as PARSER_DEFAULT_DEBUG
 from xdis import PYTHON3
-from itertools import islice,chain,repeat
 
 class Python3Parser(PythonParser):
 
@@ -185,23 +185,11 @@ class Python3Parser(PythonParser):
         # one COME_FROM for Python 2.7 seems to associate the
         # COME_FROM targets from the wrong places
 
-        try_except     ::= SETUP_EXCEPT suite_stmts_opt POP_BLOCK
-                           except_handler opt_come_from_except
-
         # this is nested inside a try_except
         tryfinallystmt ::= SETUP_FINALLY suite_stmts_opt
                            POP_BLOCK LOAD_CONST
                            COME_FROM_FINALLY suite_stmts_opt END_FINALLY
 
-        tryelsestmt    ::= SETUP_EXCEPT suite_stmts_opt POP_BLOCK
-                           except_handler else_suite come_from_except_clauses
-
-        tryelsestmt    ::= SETUP_EXCEPT suite_stmts_opt POP_BLOCK
-                           except_handler else_suite come_froms
-
-        tryelsestmtl   ::= SETUP_EXCEPT suite_stmts_opt POP_BLOCK
-                           except_handler else_suitel come_from_except_clauses
-
         except_handler ::= jmp_abs COME_FROM except_stmts
                            END_FINALLY
         except_handler ::= jmp_abs COME_FROM_EXCEPT except_stmts
@@ -321,7 +309,7 @@ class Python3Parser(PythonParser):
         # FIXME: Common with 2.7
         ret_and  ::= expr JUMP_IF_FALSE_OR_POP ret_expr_or_cond COME_FROM
         ret_or   ::= expr JUMP_IF_TRUE_OR_POP ret_expr_or_cond COME_FROM
-        ret_cond ::= expr POP_JUMP_IF_FALSE expr RETURN_END_IF ret_expr_or_cond
+        ret_cond ::= expr POP_JUMP_IF_FALSE expr RETURN_END_IF COME_FROM ret_expr_or_cond
 
         or   ::= expr JUMP_IF_TRUE_OR_POP expr COME_FROM
         and  ::= expr JUMP_IF_FALSE_OR_POP expr COME_FROM
@@ -564,9 +552,10 @@ class Python3Parser(PythonParser):
         # include instructions that don't need customization,
         # but we'll do a finer check after the rough breakout.
         customize_instruction_basenames = frozenset(
-            ('BUILD', 'CALL', 'CONTINUE', 'DELETE', 'GET',
-             'JUMP',  'LOAD', 'LOOKUP',   'MAKE',
-             'RETURN', 'RAISE', 'UNPACK'))
+            ('BUILD', 'CALL',   'CONTINUE',    'DELETE', 'GET',
+             'JUMP',  'LOAD',   'LOOKUP',      'MAKE',
+             'RETURN', 'RAISE', 'SETUP',
+             'UNPACK'))
 
         # Opcode names in the custom_ops_processed set have rules that get added
         # unconditionally and the rules are constant. So they need to be done
@@ -594,7 +583,6 @@ class Python3Parser(PythonParser):
               stmt ::= assign2_pypy
               assign3_pypy       ::= expr expr expr store store store
               assign2_pypy       ::= expr expr store store
-              return_if_lambda   ::= RETURN_END_IF_LAMBDA
               stmt               ::= conditional_lambda
               stmt               ::= conditional_not_lambda
               conditional_lambda ::= expr jmp_false expr return_if_lambda
@@ -1119,6 +1107,26 @@ class Python3Parser(PythonParser):
                     raise_stmt2 ::= expr expr RAISE_VARARGS_2
                     """, nop_func)
                 custom_ops_processed.add(opname)
+            elif opname == 'SETUP_EXCEPT':
+                self.addRule("""
+                    try_except     ::= SETUP_EXCEPT suite_stmts_opt POP_BLOCK
+                                       except_handler opt_come_from_except
+
+                    tryelsestmt    ::= SETUP_EXCEPT suite_stmts_opt POP_BLOCK
+                                       except_handler else_suite come_from_except_clauses
+
+                    tryelsestmt    ::= SETUP_EXCEPT suite_stmts_opt POP_BLOCK
+                                       except_handler else_suite come_froms
+
+                    tryelsestmtl   ::= SETUP_EXCEPT suite_stmts_opt POP_BLOCK
+                                       except_handler else_suitel come_from_except_clauses
+
+                    stmt             ::= tryelsestmtl3
+                    tryelsestmtl3    ::= SETUP_EXCEPT suite_stmts_opt POP_BLOCK
+                                         except_handler COME_FROM else_suitel
+                                         opt_come_from_except
+                    """, nop_func)
+                custom_ops_processed.add(opname)
             elif opname_base in ('UNPACK_EX',):
                 before_count, after_count = token.attr
                 rule = 'unpack ::= ' + opname + ' store' * (before_count + after_count + 1)
@@ -1137,8 +1145,11 @@ class Python3Parser(PythonParser):
         self.check_reduce['aug_assign2'] = 'AST'
         self.check_reduce['while1stmt'] = 'noAST'
         self.check_reduce['while1elsestmt'] = 'noAST'
+        self.check_reduce['ifelsestmt'] = 'AST'
         self.check_reduce['annotate_tuple'] = 'noAST'
         self.check_reduce['kwarg'] = 'noAST'
+        self.check_reduce['try_except'] = 'AST'
+
         # FIXME: remove parser errors caused by the below
         # self.check_reduce['while1elsestmt'] = 'noAST'
         return
@@ -1180,6 +1191,16 @@ class Python3Parser(PythonParser):
             if last == n:
                 return False
             return tokens[first].attr > tokens[last].offset
+        elif rule == ('try_except',
+                      ('SETUP_EXCEPT', 'suite_stmts_opt', 'POP_BLOCK',
+                       'except_handler', 'opt_come_from_except')):
+            come_from_except = ast[-1]
+            if come_from_except[0] == 'COME_FROM':
+                # There should be at last two COME_FROMs, one from an
+                # exception handler and one from the try. Otherwise
+                # we have a try/else.
+                return True
+            pass
         elif lhs == 'while1stmt':
 
             # If there is a fall through to the COME_FROM_LOOP. then this is
@@ -1195,7 +1216,7 @@ class Python3Parser(PythonParser):
 
             if tokens[cfl-1] != 'JUMP_BACK':
                 cfl_offset = tokens[cfl-1].offset
-                insn = chain((i for i in self.insts if cfl_offset == i.offset), repeat(None)).next()
+                insn = next(i for i in self.insts if cfl_offset == i.offset)
                 if insn and  insn.is_jump_target:
                     return True
 
@@ -1213,6 +1234,14 @@ class Python3Parser(PythonParser):
                 if offset != tokens[first].attr:
                     return True
             return False
+        elif rule == ('ifelsestmt',
+                      ('testexpr', 'c_stmts_opt', 'jump_forward_else', 'else_suite', '_come_froms')):
+            # Make sure the highest/smallest "come from" offset comes inside the "if".
+            come_froms = ast[-1]
+            if not isinstance(come_froms, Token):
+                return tokens[first].offset > come_froms[-1].attr
+            return False
+
         return False
 
 class Python30Parser(Python3Parser):

uncompyle6/parsers/parse30.py

@@ -2,6 +2,7 @@
 """
 spark grammar differences over Python 3.1 for Python 3.0.
 """
+from __future__ import print_function
 
 from uncompyle6.parser import PythonParserSingle
 from uncompyle6.parsers.parse31 import Python31Parser

uncompyle6/parsers/parse31.py

@@ -2,6 +2,7 @@
 """
 spark grammar differences over Python 3.2 for Python 3.1.
 """
+from __future__ import print_function
 
 from uncompyle6.parser import PythonParserSingle
 from uncompyle6.parsers.parse32 import Python32Parser

uncompyle6/parsers/parse32.py

@@ -2,6 +2,8 @@
 """
 spark grammar differences over Python 3 for Python 3.2.
 """
+from __future__ import print_function
+
 from uncompyle6.parser import PythonParserSingle
 from uncompyle6.parsers.parse3 import Python3Parser
 

uncompyle6/parsers/parse33.py

@@ -2,6 +2,7 @@
 """
 spark grammar differences over Python 3.2 for Python 3.3.
 """
+from __future__ import print_function
 
 from uncompyle6.parser import PythonParserSingle
 from uncompyle6.parsers.parse32 import Python32Parser