Merge branch 'master' into python-2.4

Bump python versions used in testing

.circleci/config.yml

@@ -0,0 +1,87 @@
+# This configuration was automatically generated from a CircleCI 1.0 config.
+# It should include any build commands you had along with commands that CircleCI
+# inferred from your project structure. We strongly recommend you read all the
+# comments in this file to understand the structure of CircleCI 2.0, as the idiom
+# for configuration has changed substantially in 2.0 to allow arbitrary jobs rather
+# than the prescribed lifecycle of 1.0. In general, we recommend using this generated
+# configuration as a reference rather than using it in production, though in most
+# cases it should duplicate the execution of your original 1.0 config.
+version: 2
+jobs:
+  build:
+    working_directory: ~/rocky/python-uncompyle6
+    parallelism: 1
+    shell: /bin/bash --login
+    # CircleCI 2.0 does not support environment variables that refer to each other the same way as 1.0 did.
+    # If any of these refer to each other, rewrite them so that they don't or see https://circleci.com/docs/2.0/env-vars/#interpolating-environment-variables-to-set-other-environment-variables .
+    environment:
+      CIRCLE_ARTIFACTS: /tmp/circleci-artifacts
+      CIRCLE_TEST_REPORTS: /tmp/circleci-test-results
+      COMPILE: --compile
+    # In CircleCI 1.0 we used a pre-configured image with a large number of languages and other packages.
+    # In CircleCI 2.0 you can now specify your own image, or use one of our pre-configured images.
+    # The following configuration line tells CircleCI to use the specified docker image as the runtime environment for you job.
+    # We have selected a pre-built image that mirrors the build environment we use on
+    # the 1.0 platform, but we recommend you choose an image more tailored to the needs
+    # of each job. For more information on choosing an image (or alternatively using a
+    # VM instead of a container) see https://circleci.com/docs/2.0/executor-types/
+    # To see the list of pre-built images that CircleCI provides for most common languages see
+    # https://circleci.com/docs/2.0/circleci-images/
+    docker:
+    - image: circleci/build-image:ubuntu-14.04-XXL-upstart-1189-5614f37
+      command: /sbin/init
+    steps:
+    # Machine Setup
+    #   If you break your build into multiple jobs with workflows, you will probably want to do the parts of this that are relevant in each
+    # The following `checkout` command checks out your code to your working directory. In 1.0 we did this implicitly. In 2.0 you can choose where in the course of a job your code should be checked out.
+    - checkout
+    # Prepare for artifact and test results  collection equivalent to how it was done on 1.0.
+    # In many cases you can simplify this from what is generated here.
+    # 'See docs on artifact collection here https://circleci.com/docs/2.0/artifacts/'
+    - run: mkdir -p $CIRCLE_ARTIFACTS $CIRCLE_TEST_REPORTS
+    # This is based on your 1.0 configuration file or project settings
+    - run:
+        working_directory: ~/rocky/python-uncompyle6
+        command: pyenv install 2.4.6 && pyenv local 2.4.6 && pyenv rehash && easy_install nose && pyenv rehash
+    # Dependencies
+    #   This would typically go in either a build or a build-and-test job when using workflows
+    # Restore the dependency cache
+    - restore_cache:
+        keys:
+        # This branch if available
+        - v1-dep-{{ .Branch }}-
+        # Default branch if not
+        - v1-dep-master-
+        # Any branch if there are none on the default branch - this should be unnecessary if you have your default branch configured correctly
+        - v1-dep-
+    # This is based on your 1.0 configuration file or project settings
+    - run: easy_install spark_parser==1.8.5 && easy_install xdis==3.8.4
+    # Save dependency cache
+    - save_cache:
+        key: v1-dep-{{ .Branch }}-{{ epoch }}
+        paths:
+        # This is a broad list of cache paths to include many possible development environments
+        # You can probably delete some of these entries
+        - vendor/bundle
+        - ~/virtualenvs
+        - ~/.m2
+        - ~/.ivy2
+        - ~/.bundle
+        - ~/.go_workspace
+        - ~/.gradle
+        - ~/.cache/bower
+    # Test
+    #   This would typically be a build job when using workflows, possibly combined with build
+    # This is based on your 1.0 configuration file or project settings
+    - run: python ./setup.py develop && make check-2.4
+    - run: cd ./test/stdlib && pyenv local 2.4.6 && bash ./runtests.sh 'test_[p-z]*.py'
+    # Teardown
+    #   If you break your build into multiple jobs with workflows, you will probably want to do the parts of this that are relevant in each
+    # Save test results
+    - store_test_results:
+        path: /tmp/circleci-test-results
+    # Save artifacts
+    - store_artifacts:
+        path: /tmp/circleci-artifacts
+    - store_artifacts:
+        path: /tmp/circleci-test-results

.github/ISSUE_TEMPLATE/bug-report.md

@@ -0,0 +1,77 @@
+---
+name: Bug report
+about: Tell us about uncompyle6 bugs
+
+---
+
+<!-- __Note:__ Have you read https://github.com/rocky/python-uncompyle6/blob/master/HOW-TO-REPORT-A-BUG.md ?
+
+Please remove any of the optional sections if they are not applicable.
+
+Prerequisites
+
+* Make sure the bytecode you have can be disassembled with a
+  disassembler.
+* Don't put bytecode and corresponding source code on any service that
+  requires registration to download.
+* When you open a bug report there is no privacy. If the legitimacy of
+  the activity is deemed suspicous, I may flag it as suspicious,
+  making the issue even more easy to detect.
+
+Bug reports that violate a prerequisite may be discarded.
+
+Note that there are way more bug-fix requestors than there are bug
+fixers. If you want you need more immediate, confidential or urgent
+assistance
+
+http://www.crazy-compilers.com/decompyle/ offers a byte-code
+decompiler service for versions of Python up to 2.6.
+
+-->
+
+## Description
+
+<!-- Add a clear and concise description of the bug. -->
+
+## How to Reproduce
+
+<!-- Please show both the input you gave and the
+output you got in describing how to reproduce the bug:
+
+or give a complete console log with input and output
+
+```console
+$ uncompyle6 <command-line-options>
+...
+$
+```
+
+Provide links to the Python bytecode. For example you can create a
+gist with the information. If you have the correct source code, you
+can add that too.
+
+-->
+
+## Expected behavior
+
+<!-- Add a clear and concise description of what you expected to happen. -->
+
+## Environment
+
+<!-- _This section sometimes is optional but helpful to us._
+
+Please modify for your setup
+
+- Uncompyle6 version: output from  `uncompyle6 --version` or `pip show uncompyle6`
+- Python version: `python -V`
+- OS and Version: [e.g. Ubuntu bionic]
+
+-->
+
+## Additional Environment or Context
+
+<!-- _This section is optional._
+
+Add any other context about the problem here or special environment setup.
+
+-->

.github/ISSUE_TEMPLATE/feature-request.md

@@ -0,0 +1,22 @@
+---
+name: Feature Request
+about: Tell us about a new feature that you would like to see in uncompyle6
+
+---
+
+## Description
+
+<!-- Add a short description of the feature. This might
+include same input and output. -->
+
+## Background
+
+<!-- Add any additional background for the
+feature, for example: user scenarios, or the value of the feature. -->
+
+## Tests
+<!-- _This section is optional._
+
+Add text with suggestions on how to test the feature,
+if it is not obvious.
+-->

HOW-TO-REPORT-A-BUG.md

@@ -1,6 +1,21 @@
-# How to report a Bug
+<!-- markdown-toc start - Don't edit this section. Run M-x markdown-toc-refresh-toc -->
+**Table of Contents**
 
-## The difficulty of the problem
+- [The difficulty of the problem](#the-difficulty-of-the-problem)
+- [Is it really a bug?](#is-it-really-a-bug)
+    - [Do you have valid bytecode?](#do-you-have-valid-bytecode)
+    - [Semantic equivalence vs. exact source code](#semantic-equivalence-vs-exact-source-code)
+- [What to send (minimum requirements)](#what-to-send-minimum-requirements)
+- [What to send (additional helpful information)](#what-to-send-additional-helpful-information)
+    - [But I don't *have* the source code!](#but-i-dont-have-the-source-code)
+    - [But I don't *have* the source code and am incapable of figuring how how to do a hand disassembly!](#but-i-dont-have-the-source-code-and-am-incapable-of-figuring-how-how-to-do-a-hand-disassembly)
+- [Narrowing the problem](#narrowing-the-problem)
+- [Karma](#karma)
+- [Confidentiality of Bug Reports](#confidentiality-of-bug-reports)
+- [Ethics](#ethics)
+
+<!-- markdown-toc end -->
+# The difficulty of the problem
 
 This decompiler is a constant work in progress: Python keeps
 changing, and so does its code generation.
@@ -33,7 +48,7 @@ prescribed cases, the ill-defined amorphous cases as well will get
 handled as well.
 
 In sum, you may need to do some work to have the bug you have found
-handled before the hundreds of other bugs, and things I could be
+handled before the hundreds of other bugs, and other things I could be
 doing.
 
 No one is getting paid to work to work on this project, let alone the
@@ -41,10 +56,10 @@ bugs you may have an interest in. If you require decompiling bytecode
 immediately, consider using a decompilation service, listed further
 down in this document.
 
-## Is it really a bug?
+# Is it really a bug?
 
 
-### Do you have valid bytecode?
+## Do you have valid bytecode?
 
 As mentioned in README.rst, this project doesn't handle obfuscated
 code. See README.rst for suggestions for how to remove some kinds of
@@ -55,11 +70,11 @@ Python comes with a disassembly module called `dis`. A prerequisite
 module for this package, `xdis` has a cross-python version
 disassembler called `pydisasm`.
 
-### Semantic equivalence vs. exact source code
+## Semantic equivalence vs. exact source code
 
 Consider how Python compiles something like "(x*y) + 5". Early on
 Python creates an "abstract syntax tree" (AST) for this. And this is
-"abstract" in the sense that unimportant, redundant or unnecceary
+"abstract" in the sense that unimportant, redundant or unnecessary
 items have been removed. Here, this means that any notion that you
 wrote "x+y" in parenthesis is lost, since in this context they are
 unneeded. Also lost is the fact that the multiplication didn't have
@@ -132,7 +147,7 @@ Python will eliminate the entire "if" statement.
 So just because the text isn't the same, does not
 necessarily mean there's a bug.
 
-## What to send (minimum requirements)
+# What to send (minimum requirements)
 
 The basic requirement is pretty simple:
 
@@ -146,7 +161,7 @@ sending is too large.
 
 Also try to narrow the bug. See below.
 
-## What to send (additional helpful information)
+# What to send (additional helpful information)
 
 Some kind folks also give the invocation they used and the output
 which usually includes an error message produced. This is
@@ -159,7 +174,7 @@ provide the input command and the output from that, please give:
 * Python interpreter version used
 
 
-### But I don't *have* the source code!
+## But I don't *have* the source code!
 
 Sure, I get it. No problem. There is Python assembly code on parse
 errors, so simply by hand decompile that. To get a full disassembly,
@@ -167,7 +182,7 @@ use `pydisasm` from the [xdis](https://pypi.python.org/pypi/xdis)
 package. Opcodes are described in the documentation for
 the [dis](https://docs.python.org/3.6/library/dis.html) module.
 
-### But I don't *have* the source code and am incapable of figuring how how to do a hand disassembly!
+### But I don't *have* the source code and am incapable of figuring how to do a hand disassembly!
 
 Well, you could learn. No one is born into this world knowing how to
 disassemble Python bytecode. And as Richard Feynman once said, "What
@@ -179,7 +194,7 @@ Compilers](http://www.crazy-compilers.com/decompyle/) offers a
 byte-code decompiler service for versions of Python up to 2.6. (If
 there are others around let me know and I'll list them here.)
 
-## Narrowing the problem
+# Narrowing the problem
 
 I don't need or want the entire source code base for the file(s) or
 module(s) can't be decompiled. I just need those file(s) or module(s).
@@ -197,22 +212,53 @@ what doesn't. That is useful. Or maybe the same file will decompile
 properly on a neighboring version of Python. That is helpful too.
 
 In sum, the more you can isolate or narrow the problem, the more
-likley the problem will be fixed and fixed sooner.
+likely the problem will be fixed and fixed sooner.
 
-## Confidentiality of Bug Reports
+# Karma
+
+I realize that following the instructions given herein puts a bit of
+burden on the bug reporter. In my opinion, this is justified as
+attempts to balance somewhat the burden and effort needed to fix the
+bug and the attempts to balance number of would-be bug reporters with
+the number of bug fixers. Better bug reporters are more likely to move
+in the category of bug fixers.
+
+The barrier to reporting a big is pretty small: all you really need is
+a github account, and the ability to type something after clicking
+some buttons. So the reality is that many people just don't bother to
+read these instructions, let alone follow it to any simulacrum.
+
+And the reality is also that bugs sometimes get fixed even though
+these instructions are not followed.
+
+So one factors I may take into consideration is the bug reporter's karma.
+
+* Have you demonstrably contributed to open source? I may look at your
+  github profile to see what contributions you have made, how popular
+  those contributions are, or how popular you are.
+* How appreciative are you? Have you starred this project that you are
+  seeking help from? Have you starred _any_ github project? And the above
+  two kind of feed into ...
+* Attitude. Some people feel that they are doing me and the world a
+  great favor by just pointing out that there is a problem whose solution
+  would greatly benefit them. Perhaps this is why they feel that
+  instructions are not to be followed by them, nor any need for
+  showing evidence gratitude when help is offered them.
+
+# Confidentiality of Bug Reports
 
 When you report a bug, you are giving up confidentiality to the source
 code and the byte code. However, I would imagine that if you have
 narrowed the problem sufficiently, confidentiality of the little that
 remains would not be an issue.
 
-However feel free to remove any commments, and modify variable names
+However feel free to remove any comments, and modify variable names
 or constants in the source code.
 
-## Ethics
+# Ethics
 
 I do not condone using this program for unethical or illegal purposes.
-More detestful, at least to me, is asking for help to assist you in
+More detestable, at least to me, is asking for help to assist you in
 something that might not legitimate.
 
 Don't use the issue tracker for such solicitations. To try to stave

NEWS

@@ -1,3 +1,29 @@
+uncompyle6 3.2.5 2019-12-30 Clearout sale
+
+- 3.7.2 Remove deprecation warning on regexp string that isn't raw
+- main.main() parameter `codes` is not used - note that
+- Improve Python 3.6+ control flow detection
+- More complete fragment instruction annotation for `imports`
+
+uncompyle6 3.2.4 2018-10-27 7x9 release
+
+- Bug fixes #180, #182, #187, #192
+- Enhancements #189
+- Internal improvements
+
+uncompyle6 3.2.3 2018-06-04 Michael Cohen flips and Fleetwood Redux
+
+- Python 1.3 support 3.0 bug and
+- fix botched parameter ordering of 3.x in last release
+
+uncompyle6 3.2.2 2018-06-04 When I'm 64
+
+- Python 3.0 support and bug fixes
+
+uncompyle6 3.2.1 2018-06-04 MF
+
+- Python 1.4 and 1.5 bug fixes
+
 uncompyle6 3.2.0 2018-05-19 Rocket Scientist
 
 - Add rudimentary 1.4 support (still a bit buggy)

README.rst

@@ -1,4 +1,4 @@
-|buildstatus|
+|buildstatus| |Latest Version| |Supported Python Versions|
 
 uncompyle6
 ==========
@@ -11,8 +11,9 @@ Introduction
 ------------
 
 *uncompyle6* translates Python bytecode back into equivalent Python
-source code. It accepts bytecodes from Python version 1.5, and 2.1 to
-3.7 or so, including PyPy bytecode and Dropbox's Python 2.5 bytecode.
+source code. It accepts bytecodes from Python version 1.3 to version
+3.7, spanning over 22 years of Python releases. We include Dropbox's
+Python 2.5 bytecode and some PyPy bytecode.
 
 Why this?
 ---------
@@ -29,11 +30,11 @@ CPython bytecode decompilers is the ability to deparse just
 *fragments* of source code and give source-code information around a
 given bytecode offset.
 
-I use the tree fragments to deparse fragments of code inside my
-trepan_ debuggers_. For that, bytecode offsets are recorded and
-associated with fragments of the source code. This purpose, although
-compatible with the original intention, is yet a little bit different.
-See this_ for more information.
+I use the tree fragments to deparse fragments of code *at run time*
+inside my trepan_ debuggers_. For that, bytecode offsets are recorded
+and associated with fragments of the source code. This purpose,
+although compatible with the original intention, is yet a little bit
+different.  See this_ for more information.
 
 Python fragment deparsing given an instruction offset is useful in
 showing stack traces and can be encorporated into any program that
@@ -58,7 +59,7 @@ provides decompilation for subset of Python versions, we generally do
 demonstrably better for those as well.
 
 How can we tell? By taking Python bytecode that comes distributed with
-that version of Python and decompiling these.  Among htose that
+that version of Python and decompiling these.  Among those that
 successfully decompile, we can then make sure the resulting programs
 are syntactically correct by running the Python interpreter for that
 bytecode version.  Finally, in cases where the program has a test for
@@ -75,7 +76,7 @@ Requirements
 The code here can be run on Python versions 2.6 or later, PyPy 3-2.4,
 or PyPy-5.0.1.  Python versions 2.4-2.7 are supported in the
 python-2.4 branch.  The bytecode files it can read have been tested on
-Python bytecodes from versions 1.5, 2.1-2.7, and 3.0-3.6 and the
+Python bytecodes from versions 1.4, 2.1-2.7, and 3.0-3.6 and the
 above-mentioned PyPy versions.
 
 Installation
@@ -151,12 +152,12 @@ for that bytecode version. Having done this the bytecode produced
 could be compared with the original bytecode. However as Python's code
 generation got better, this is no longer feasible.
 
-There is a kind of *weak verification* that we use that doesn't check
-bytecode for equivalence but does check to see if the resulting
-decompiled source is a valid Python program by running the Python
-interpreter. Because the Python language has changed so much, for best
-results you should use the same Python version in checking as was used
-in creating the bytecode.
+There verification that we use that doesn't check bytecode for
+equivalence but does check to see if the resulting decompiled source
+is a valid Python program by running the Python interpreter. Because
+the Python language has changed so much, for best results you should
+use the same Python version in checking as was used in creating the
+bytecode.
 
 There are however an interesting class of these programs that is
 readily available give stronger verification: those programs that
@@ -174,19 +175,21 @@ that era was minimal)
 
 There is some work to do on the lower end Python versions which is
 more difficult for us to handle since we don't have a Python
-interpreter for versions 1.5, 1.6, and 2.0.
+interpreter for versions 1.6, and 2.0.
 
 In the Python 3 series, Python support is is strongest around 3.4 or
 3.3 and drops off as you move further away from those versions. Python
-3.6 changes things drastically by using word codes rather than byte
-codes. As a result, the jump offset field in a jump instruction
-argument has been reduced. This makes the `EXTENDED_ARG` instructions
-are now more prevalent in jump instruction; previously they had been
-rare.  Perhaps to compensate for the additional `EXTENDED_ARG`
-instructions, additional jump optimization has been added. So in sum
-handling control flow by ad hoc means as is currently done is worse.
+3.0 is weird in that it in some ways resembles 2.6 more than it does
+3.1 or 2.7. Python 3.6 changes things drastically by using word codes
+rather than byte codes. As a result, the jump offset field in a jump
+instruction argument has been reduced. This makes the `EXTENDED_ARG`
+instructions are now more prevalent in jump instruction; previously
+they had been rare.  Perhaps to compensate for the additional
+`EXTENDED_ARG` instructions, additional jump optimization has been
+added. So in sum handling control flow by ad hoc means as is currently
+done is worse.
 
-Also, between Python 3.5, 3.6 and 3.7 there have been major changes to the
+Between Python 3.5, 3.6 and 3.7 there have been major changes to the
 `MAKE_FUNCTION` and `CALL_FUNCTION` instructions.
 
 Currently not all Python magic numbers are supported. Specifically in
@@ -212,11 +215,12 @@ There is lots to do, so please dig in and help.
 See Also
 --------
 
-* https://github.com/zrax/pycdc : supports all versions of Python and is written in C++. Support for Python 3 is a bit lacking though.
-* https://code.google.com/archive/p/unpyc3/ : supports Python 3.2 only. The above projects use a different decompiling technique than what is used here.
-* https://github.com/figment/unpyc3/ : fork of above, but supports Python 3.3 only. Includes some fixes like supporting function annotations
-* The HISTORY_ file.
+* https://github.com/zrax/pycdc : purports to support all versions of Python. It is written in C++ and is most accurate for Python versions around 2.7 and 3.3 when the code was more actively developed. Accuracy for more recent versions of Python 3 and early versions of Python are especially lacking. See its `issue tracker <https://github.com/zrax/pycdc/issues>`_ for details. Currently lightly maintained.
+* https://code.google.com/archive/p/unpyc3/ : supports Python 3.2 only. The above projects use a different decompiling technique than what is used here. Currently unmaintained.
+* https://github.com/figment/unpyc3/ : fork of above, but supports Python 3.3 only. Includes some fixes like supporting function annotations. Currently unmaintained.
+* https://github.com/wibiti/uncompyle2 : supports Python 2.7 only, but does that fairly well. There are situtations where `uncompyle6` results are incorrect while `uncompyle2` results are not, but more often uncompyle6 is correct when uncompyle2 is not. Because `uncompyle6` adheres to accuracy over idiomatic Python, `uncompyle2` can produce more natural-looking code when it is correct. Currently `uncompyle2` is lightly maintained. See its issue `tracker <https://github.com/wibiti/uncompyle2/issues>`_ for more details
 * `How to report a bug <https://github.com/rocky/python-uncompyle6/blob/master/HOW-TO-REPORT-A-BUG.md>`_
+* The HISTORY_ file.
 * https://github.com/rocky/python-xdis : Cross Python version disassembler
 * https://github.com/rocky/python-xasm : Cross Python version assembler
 * https://github.com/rocky/python-uncompyle6/wiki : Wiki Documents which describe the code and aspects of it in more detail
@@ -234,3 +238,6 @@ See Also
 .. _PJOrion: http://www.koreanrandom.com/forum/topic/15280-pjorion-%D1%80%D0%B5%D0%B4%D0%B0%D0%BA%D1%82%D0%B8%D1%80%D0%BE%D0%B2%D0%B0%D0%BD%D0%B8%D0%B5-%D0%BA%D0%BE%D0%BC%D0%BF%D0%B8%D0%BB%D1%8F%D1%86%D0%B8%D1%8F-%D0%B4%D0%B5%D0%BA%D0%BE%D0%BC%D0%BF%D0%B8%D0%BB%D1%8F%D1%86%D0%B8%D1%8F-%D0%BE%D0%B1%D1%84
 .. _Deobfuscator: https://github.com/extremecoders-re/PjOrion-Deobfuscator
 .. _Py2EXE: https://en.wikipedia.org/wiki/Py2exe
+.. |Supported Python Versions| image:: https://img.shields.io/pypi/pyversions/uncompyle6.svg
+.. |Latest Version| image:: https://badge.fury.io/py/uncompyle6.svg
+		 :target: https://badge.fury.io/py/uncompyle6

__pkginfo__.py

@@ -35,6 +35,7 @@ classifiers =  ['Development Status :: 5 - Production/Stable',
                 'Programming Language :: Python :: 2.5',
                 'Programming Language :: Python :: 2.6',
                 'Programming Language :: Python :: 2.7',
+                'Programming Language :: Python :: 3.0',
                 'Programming Language :: Python :: 3.1',
                 'Programming Language :: Python :: 3.2',
                 'Programming Language :: Python :: 3.3',
@@ -55,8 +56,9 @@ entry_points       = {
         'pydisassemble=uncompyle6.bin.pydisassemble:main',
     ]}
 ftp_url            = None
-install_requires   = ['spark-parser >= 1.8.5, < 1.9.0',
-                      'xdis >= 3.8.2, < 3.9.0']
+install_requires   = ['spark-parser >= 1.8.7, < 1.9.0',
+                      'xdis >= 3.8.9, < 3.9.0']
+
 license            = 'GPL3'
 mailing_list       = 'python-debugger@googlegroups.com'
 modname            = 'uncompyle6'

admin-tools/check-newer-versions.sh

@@ -16,6 +16,7 @@ if ! source ./setup-master.sh ; then
 fi
 cd ..
 for version in $PYVERSIONS; do
+    echo --- $version ---
     if ! pyenv local $version ; then
 	exit $?
     fi
@@ -23,4 +24,5 @@ for version in $PYVERSIONS; do
     if ! make check; then
 	exit $?
     fi
+    echo === $version ===
 done

admin-tools/check-older-versions.sh

@@ -15,6 +15,7 @@ fi
 
 cd ..
 for version in $PYVERSIONS; do
+    echo --- $version ---
     if ! pyenv local $version ; then
 	exit $?
     fi
@@ -22,4 +23,5 @@ for version in $PYVERSIONS; do
     if ! make check ; then
 	exit $?
     fi
+    echo === $version ===
 done

admin-tools/pyenv-newer-versions

@@ -5,4 +5,4 @@ if [[ $0 == ${BASH_SOURCE[0]} ]] ; then
     echo "This script should be *sourced* rather than run directly through bash"
     exit 1
 fi
-export PYVERSIONS='3.5.5 3.6.5 2.6.9 3.3.7 2.7.14 3.2.6 3.1.5 3.4.8'
+export PYVERSIONS='3.5.5 3.6.8 3.7.2 2.6.9 3.3.7 2.7.15 3.2.6 3.1.5 3.4.8'

appveyor.yml

@@ -47,7 +47,7 @@ install:
 
   # Upgrade to the latest version of pip to avoid it displaying warnings
   # about it being out of date.
-  - "pip install --disable-pip-version-check --user --upgrade pip"
+  - "%PYTHON%\\python.exe -m pip install --disable-pip-version-check --user --upgrade pip"
 
   # Install the build dependencies of the project. If some dependencies contain
   # compiled extensions and are not provided as pre-built wheel packages,

circle.yml

@@ -1,15 +0,0 @@
-machine:
-    python:
-        version: 2.7.10
-    environment:
-        COMPILE: --compile
-
-dependencies:
-    override:
-        - pip install --upgrade setuptools
-        - pip install -e .
-        - pip install pytest==3.2.5 hypothesis
-test:
-    override:
-        - python ./setup.py develop && make check-2.7
-        - cd ./test/stdlib && pyenv local 2.7.10 && bash ./runtests.sh 'test_[f-i]*.py'

pytest/test_grammar.py

@@ -46,13 +46,11 @@ def test_grammar():
             unused_rhs.add("mkfunc_annotate")
             unused_rhs.add("dict_comp")
             unused_rhs.add("classdefdeco1")
-            if PYTHON_VERSION != 3.6:
-                if PYTHON_VERSION in (3.5, 3.7):
-                    expect_right_recursive.add((('l_stmts',
-                                                 ('lastl_stmt', 'come_froms', 'l_stmts'))))
-                    pass
+            if PYTHON_VERSION >= 3.5:
+                expect_right_recursive.add((('l_stmts',
+                                             ('lastl_stmt', 'come_froms', 'l_stmts'))))
                 pass
-            else:
+            elif 3.0 < PYTHON_VERSION < 3.3:
                 expect_right_recursive.add((('l_stmts',
                                              ('lastl_stmt', 'COME_FROM', 'l_stmts'))))
                 pass

pytest/test_single_compile.py

@@ -1,6 +1,6 @@
-from uncompyle6 import PYTHON_VERSION, deparse_code
+from uncompyle6 import PYTHON_VERSION, code_deparse
 
-if PYTHON_VERSION >= 2.6:
+if PYTHON_VERSION > 2.6:
     def test_single_mode():
         single_expressions = (
             'i = 1',
@@ -16,4 +16,4 @@ if PYTHON_VERSION >= 2.6:
 
         for expr in single_expressions:
             code = compile(expr + '\n', '<string>', 'single')
-            assert deparse_code(PYTHON_VERSION, code, compile_mode='single').text == expr + '\n'
+            assert code_deparse(code, compile_mode='single').text == expr + '\n'

pytest/validate.py

@@ -3,14 +3,15 @@ import os
 import difflib
 import subprocess
 import tempfile
-
 from StringIO import StringIO
+
 # uncompyle6 / xdis
 from uncompyle6 import PYTHON_VERSION, IS_PYPY, deparse_code
 # TODO : I think we can get xdis to support the dis api (python 3 version) by doing something like this there
 from xdis.bytecode import Bytecode
 from xdis.main import get_opcode
 opc = get_opcode(PYTHON_VERSION, IS_PYPY)
+from StringIO import StringIO
 
 try:
     import functools
@@ -20,8 +21,6 @@ try:
 except:
     pass
 
-
-
 def print_diff(original, uncompyled):
     """
     Try and display a pretty html line difference between the original and

requirements-dev.txt

@@ -1,3 +1,2 @@
-pytest>=3.0.0,<=3.0.1
 flake8
 hypothesis<=3.0.0

requirements.txt

@@ -1,2 +1,4 @@
 # Pick up stuff from setup.py
+hypothesis==2.0.0
+pytest
 -e .

test/Makefile

@@ -1,6 +1,6 @@
 PHONY=check clean dist distclean test test-unit test-functional rmChangeLog clean_pyc nosetests \
-      check-bytecode-1  check-bytecode-1.4 check-bytecode-1.5 \
-      check-bytecode-2 check-bytecode-3 \
+      check-bytecode-1  check-bytecode-1.3 check-bytecode-1.4 check-bytecode-1.5 \
+      check-bytecode-2 check-bytecode-3 check-bytecode-3-short \
       check-bytecode-2.2 check-byteocde-2.3 check-bytecode-2.4 \
       check-short check-2.6 check-2.7 check-3.0 check-3.1 check-3.2 check-3.3 \
       check-3.4 check-3.5 check-3.6 check-3.7 check-5.6 5.6 5.8 \
@@ -22,7 +22,7 @@ COVER_DIR=../tmp/grammar-cover
 # Run short tests
 check-short:
 	@$(PYTHON) -V && PYTHON_VERSION=`$(PYTHON) -V 2>&1 | cut -d ' ' -f 2 | cut -d'.' -f1,2`; \
-	$(MAKE) check-bytecode
+	$(MAKE) check-bytecode-short
 
 # Run all tests
 check:
@@ -34,6 +34,7 @@ check-2.4 check-2.5 check-2.6 check-2.7: check-bytecode-2 check-bytecode-3 check
 #: Run working tests from Python 3.0
 check-3.0: check-bytecode
 	$(PYTHON) test_pythonlib.py --bytecode-3.0 --weak-verify $(COMPILE)
+	$(PYTHON) test_pythonlib.py --bytecode-3.0-run --verify-run
 
 #: Run working tests from Python 3.1
 check-3.1: check-bytecode
@@ -91,16 +92,31 @@ check-bytecode-3:
        --bytecode-3.1 --bytecode-3.2 --bytecode-3.3 \
        --bytecode-3.4 --bytecode-3.5 --bytecode-3.6 --bytecode-pypy3.2
 
-#: Check deparsing bytecode that works running Python 2 and Python 3
+#: Check deparsing on selected bytecode 3.x
+check-bytecode-3-short:
+	$(PYTHON) test_pythonlib.py \
+       --bytecode-3.4 --bytecode-3.5 --bytecode-3.6
+
+#: Check deparsing bytecode on all Python 2 and Python 3 versions
 check-bytecode: check-bytecode-3
 	$(PYTHON) test_pythonlib.py \
-	--bytecode-1.4 --bytecode-1.5 \
+	--bytecode-1.3 --bytecode-1.4 --bytecode-1.5 \
         --bytecode-2.2 --bytecode-2.3 --bytecode-2.4 \
 	--bytecode-2.1 --bytecode-2.2 --bytecode-2.3 --bytecode-2.4 \
         --bytecode-2.5 --bytecode-2.6 --bytecode-2.7 \
         --bytecode-pypy2.7
 
 
+#: Check deparsing bytecode on selected Python 2 and Python 3 versions
+check-bytecode-short: check-bytecode-3-short
+	$(PYTHON) test_pythonlib.py \
+        --bytecode-2.6 --bytecode-2.7 --bytecode-pypy2.7
+
+
+#: Check deparsing bytecode 1.3 only
+check-bytecode-1.3:
+	$(PYTHON) test_pythonlib.py --bytecode-1.3
+
 #: Check deparsing bytecode 1.4 only
 check-bytecode-1.4:
 	$(PYTHON) test_pythonlib.py --bytecode-1.4
@@ -209,6 +225,7 @@ check-bytecode-2.7:
 #: Check deparsing Python 3.0
 check-bytecode-3.0:
 	$(PYTHON) test_pythonlib.py --bytecode-3.0 --weak-verify
+	$(PYTHON) test_pythonlib.py --bytecode-3.0-run --verify-run
 
 #: Check deparsing Python 3.1
 check-bytecode-3.1:
@@ -274,7 +291,7 @@ check-3.4-ok:
 2.6:
 
 #: PyPy 5.0.x with Python 2.7 ...
-pypy-2.7 5.0 5.3:
+pypy-2.7 5.0 5.3 6.0:
 	$(PYTHON) test_pythonlib.py --bytecode-pypy2.7 --verify
 
 #: PyPy 2.4.x with Python 3.2 ...

test/simple-uncompyle-code-test.py

@@ -1,13 +1,15 @@
 #!/usr/bin/env python
 
 from uncompyle6 import uncompyle
+from uncompyle6.main import decompile
+from xdis.magics import sysinfo2float
 import sys, inspect
 
 def uncompyle_test():
     frame = inspect.currentframe()
     try:
         co = frame.f_code
-        uncompyle(2.7, co, sys.stdout, 1, 1)
+        decompile(sysinfo2float(), co, sys.stdout, 1, 1)
         print()
     finally:
         del frame

test/simple_source/bug14/00_unpack_list.py

@@ -0,0 +1,3 @@
+# Python 1.4 tzparse.py, but also appears in 1.5
+
+[tzname, delta] = __file__

test/simple_source/bug14/01_print.py

@@ -0,0 +1,6 @@
+# Python 1.4 aifc.py
+# Something weird about the final "print" and PRINT_NL_CONT followed by PRINT_NL
+def _readmark(self, markers):
+    if self._markers: print 'marker',
+    else: print 'markers',
+    print 'instead of', markers

test/simple_source/bug14/02_continue.py

@@ -0,0 +1,11 @@
+# Python 1.4 cgi.py
+# Bug was in "continue" detection.
+# 1.4 doesn't have lnotab and our CONTINUE detection is off.
+def parse_multipart(params, pdict):
+    while params:
+	if params.has_key('name'):
+	   params = None
+	else:
+	    continue
+
+    return None

test/simple_source/bug26/04_lastc_with_loops.py

@@ -0,0 +1,15 @@
+# Bug was the join of two "for" loops at the end of an "if"
+# this happens in Python 2.6 and before
+
+def cheatCogdoMazeGame(self, base, kindOfCheat):
+    if base:
+        maze = kindOfCheat
+        if maze:
+            if kindOfCheat == 0:
+                for suitNum in maze.game.suitsById.keys():
+                    maze.sendUpdate()
+            elif kindOfCheat == 1:
+                for joke in maze.game:
+                    maze.sendUpdate()
+        else:
+           self.sendUpdate()

test/simple_source/bug30/00_chained-compare.py

@@ -0,0 +1,6 @@
+# From 3.0 asyncore.py
+# This is RUNNABLE!
+r, w, e = ([], [], [])
+if [] == r == w == e:
+    r = [1]
+assert r == [1]

test/simple_source/bug30/01_comprehension.py

@@ -0,0 +1,9 @@
+# From abc.py
+def __new__(cls, value, bases, namespace):
+    {name
+     for name, value in namespace.items()
+     if getattr(value, "__isabstractmethod__", False)}
+    return
+
+# From base64.py
+_b32rev = dict([(v[0], k) for k, v in __file__])

test/simple_source/bug30/02_try_except_except.py

@@ -0,0 +1,9 @@
+# From 3.0.1/lib/python3.0/_dummy_thread.py
+
+def start_new_thread(function, args, kwargs={}):
+    try:
+        function()
+    except SystemExit:
+        pass
+    except:
+        args()

test/simple_source/bug30/03_pop_top.py

@@ -0,0 +1,20 @@
+# From 3.0.1 __dummy_thread.py
+# bug was handling else:
+def interrupt_main():
+    """Set _interrupt flag to True to have start_new_thread raise
+    KeyboardInterrupt upon exiting."""
+    if _main:
+        raise KeyboardInterrupt
+    else:
+        global _interrupt
+        _interrupt = True
+
+# From 3.0.1 ast.py bug was mangling prototype
+# def parse(expr, filename='<unknown>', mode='exec'):
+
+# From 3.0.1 bisect
+def bisect_left(a, x, lo=0, hi=None):
+    while lo:
+        if a[mid] < x: lo = mid+1
+        else: hi = mid
+    return lo

test/simple_source/bug30/06_listcomp.py

@@ -0,0 +1,24 @@
+# Python 3.0 comprehensions can produce different code from
+# all other Python versions. Thanks, Python!
+
+# This code is RUNNABLE!
+
+# Adapted from 3.0 ast.py; uses comprehension implemented via CLOSURE
+def _format(node):
+    return [(a, int(b)) for a, b in node.items()]
+
+x = {'a': '1', 'b': '2'}
+assert [('a', 1), ('b', 2)] == _format(x)
+
+# Adapted from 3.0 cmd.py; ises "if" comprehension
+def monthrange(ary, dotext):
+    return [a[3:] for a in ary if a.startswith(dotext)]
+
+ary = ["Monday", "Twoday", "Monmonth"]
+assert ['day', 'month'] == monthrange(ary, "Mon")
+
+# From 3.0 cmd.py; uses "if not" comprehension
+def columnize(l):
+    return [i for i in range(len(l))
+            if not isinstance(l[i], str)]
+assert [0, 2] == columnize([1, 'a', 2])

test/simple_source/bug31/06_listcomp.py

@@ -0,0 +1,17 @@
+# Python 3.0 comprehensions can produce different code from
+# all other Python versions. Thanks, Python!
+
+# Adapted from 3.0 ast.py
+# This code is RUNNABLE!
+def _format(node):
+    return [(a, int(b)) for a, b in node.items()]
+
+# Adapted from 3.0 cmd.py
+def monthrange(ary, dotext):
+    return [a[3:] for a in ary if a.startswith(dotext)]
+
+x = {'a': '1', 'b': '2'}
+assert [('a', 1), ('b', 2)] == _format(x)
+
+ary = ["Monday", "Twoday", "Monmonth"]
+assert ['day', 'month'] == monthrange(ary, "Mon")

test/simple_source/bug36/02_tryfinally_return.py

@@ -0,0 +1,6 @@
+# Python 3.6 sometimes omits END_FINALLY. See issue #182
+def foo():
+    try:
+        x = 1
+    finally:
+        return

test/simple_source/expression/06_huge_list.py

@@ -1,4 +1,8 @@
 # We have more than 1<<16 elements
+# In Python2 this causes an EXTENDED_ARG instruction to be emitted and then we can check
+# handling that.
+
+# It also triggers the of special rules for expr32 and expr1024
 [
     1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,11,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,11,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,
     1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,11,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,11,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,

test/simple_source/stmts/00_assign.py

@@ -1,6 +1,8 @@
 # Tests:
 # assign ::= expr store
+# This code is RUNNABLE!
 
 a = 'None'
 b = None
 c = 556
+assert (a, b, c) == ('None', None, 556)

test/stdlib/compile-file-1x.py

@@ -0,0 +1,22 @@
+#!/usr/bin/env python
+"""byte compiles a Python 1.x program"""
+import sys
+if len(sys.argv) != 2:
+    print("Usage: compile-file.py *python-file*")
+    sys.exit(1)
+source = sys.argv[1]
+
+# assert source.endswith('.py')
+basename = source[:-3]
+
+# We do this crazy way to support Python 1.4 which
+# doesn't support version_info.
+PY_VERSION = sys.version[:3]
+
+bytecode = "%s-%s.pyc" % (basename, PY_VERSION)
+
+import py_compile
+print("# compiling %s to %s" % (source, bytecode))
+py_compile.compile(source, bytecode)
+# import os
+# os.system("../bin/uncompyle6 %s" % bytecode)

test/stdlib/compile-file.py

@@ -1,4 +1,5 @@
 #!/usr/bin/env python
+"""byte compiles a Python program after version 2.2 or so. Also see compile_file_1x.py"""
 import sys
 if len(sys.argv) != 2:
     print("Usage: compile-file.py *python-file*")

test/stdlib/runtests.sh

@@ -35,6 +35,7 @@ case $PYVERSION in
 	    [test_dis.py]=1   # We change line numbers - duh!
 	    [test_grp.py]=1      # Long test - might work Control flow?
 	    [test_pwd.py]=1 # Long test - might work? Control flow?
+	    [test_pep247.py]=1 # Long test - might work? Control flow?
 	    [test_queue.py]=1 # Control flow?
 	    # [test_threading.py]=1 # Long test - works
 	)
@@ -105,6 +106,7 @@ case $PYVERSION in
 	    [test_curses.py]=1  # Possibly fails on its own but not detected
 	    [test_dis.py]=1   # We change line numbers - duh!
 	    [test_doctest.py]=1 # Fails on its own
+	    [test_format.py]=1  # control flow. uncompyle2 does not have problems here
 	    [test_generators.py]=1  # control flow. uncompyle2 has problem here too
 	    [test_grammar.py]=1     # Too many stmts. Handle large stmts
 	    [test_io.py]=1 # Test takes too long to run
@@ -113,13 +115,20 @@ case $PYVERSION in
 	                          # See test/simple_source/bug27+/05_not_unconditional.py
 	    [test_memoryio.py]=1 # FIX
 	    [test_multiprocessing.py]=1 # On uncompyle2, taks 24 secs
-	    [test_pep352.py]=1 # ?
+	    [test_pep352.py]=1  # ?
+	    [test_posix.py]=1   # Bug in try-else detection inside test_initgroups()
+	                        # Deal with when we have better flow-control detection
+	    [test_pwd.py]=1     # Takes too long
+	    [test_pty.py]=1
+	    [test_queue.py]=1   # Control flow?
+	    [test_re.py]=1      # Probably Control flow?
 	    [test_select.py]=1  # Runs okay but takes 11 seconds
 	    [test_socket.py]=1  # Runs ok but takes 22 seconds
 	    [test_subprocess.py]=1 # Runs ok but takes 22 seconds
 	    [test_sys_settrace.py]=1 # Line numbers are expected to be different
 	    [test_strtod.py]=1 # FIX
 	    [test_traceback.py]=1 # Line numbers change - duh.
+	    [test_types.py]=1     # try/else confusions
 	    [test_unicode.py]=1  # Too long to run 11 seconds
 	    [test_xpickle.py]=1 # Runs ok but takes 72 seconds
 	    [test_zipfile64.py]=1  # Runs ok but takes 204 seconds
@@ -143,6 +152,7 @@ case $PYVERSION in
 	if (( batch )) ; then
 	    # Fails in crontab environment?
 	    # Figure out what's up here
+	    SKIP_TESTS[test_exception_variations.py]=1
 	    SKIP_TESTS[test_quopri.py]=1
 	fi
 	;;
@@ -176,7 +186,7 @@ if [[ -e $TESTDIR ]] ; then
     rm -fr $TESTDIR
 fi
 mkdir $TESTDIR || exit $?
-cp -r ~/.pyenv/versions/${PYVERSION}.${MINOR}/lib/python${PYVERSION}/test $TESTDIR
+cp -r ${PYENV_ROOT}/versions/${PYVERSION}.${MINOR}/lib/python${PYVERSION}/test $TESTDIR
 cd $TESTDIR/test
 export PYTHONPATH=$TESTDIR
 

test/test_pyenvlib.py

@@ -110,8 +110,8 @@ def do_tests(src_dir, patterns, target_dir, start_with=None,
         files = [file for file in files if not 'site-packages' in file]
         files = [file for file in files if not 'test' in file]
         if len(files) > max_files:
-            # print("Numer of files %d - truncating to last 200" % len(files))
-            print("Numer of files %d - truncating to first %s" %
+            # print("Number of files %d - truncating to last 200" % len(files))
+            print("Number of files %d - truncating to first %s" %
                   (len(files), max_files))
             files = files[:max_files]
 

test/test_pythonlib.py

@@ -76,7 +76,7 @@ for  vers in (2.7, 3.4, 3.5, 3.6):
     test_options[key] = (os.path.join(src_dir, pythonlib), PYOC, key, vers)
     pass
 
-for  vers in (1.4, 1.5,
+for  vers in (1.3, 1.4, 1.5,
               2.1, 2.2, 2.3, 2.4, 2.5, 2.6, 2.7,
               3.0, 3.1, 3.2, 3.3,
               3.4, 3.5, 3.6, 3.7, 'pypy3.2', 'pypy2.7'):

uncompyle6/__init__.py

@@ -54,7 +54,7 @@ from uncompyle6.main import decompile_file
 # For compatibility
 uncompyle_file = decompile_file
 
-# Conventience functions so you can say:
+# Convenience functions so you can say:
 # from uncompyle6 import (code_deparse, deparse_code2str)
 
 code_deparse = uncompyle6.semantics.pysource.code_deparse

uncompyle6/bin/uncompile.py

@@ -68,11 +68,9 @@ def usage():
 
 
 def main_bin():
-    if not (sys.version_info[0:2] in ((2, 4), (2, 5), (2, 6), (2, 7),
-                                      (3, 2), (3, 3),
-                                      (3, 4), (3, 5), (3, 6), (3, 7))):
-        sys.stderr.write('Error: %s requires Python 2.4 2.5 2.6, 2.7, '
-                         '3.2, 3.3, 3.4, 3.5, or 3.6' % program)
+    if not (sys.version_info[0:2] in ((2, 4), (2, 5), (2, 6), (2, 7))):
+        sys.stderr.write('Error: this branch of %s requires Python 2.4, 2.5, 2.6 or 2.7'
+                         % program)
         sys.exit(-1)
 
     do_verify = recurse_dirs = False
@@ -174,7 +172,7 @@ def main_bin():
 
     if numproc <= 1:
         try:
-            result = main(src_base, out_base, files, codes, outfile,
+            result = main(src_base, out_base, files, None, outfile,
                           **options)
             result = list(result) + [options.get('do_verify', None)]
             if len(files) > 1:
@@ -209,7 +207,7 @@ def main_bin():
                     if f is None:
                         break
                     (t, o, f, v) = \
-                      main(src_base, out_base, [f], codes, outfile, **options)
+                      main(src_base, out_base, [f], None, outfile, **options)
                     tot_files += t
                     okay_files += o
                     failed_files += f

uncompyle6/disas.py

@@ -73,6 +73,19 @@ def disco_loop(disasm, queue, real_out):
             pass
         pass
 
+def disassemble_fp(fp, outstream=None):
+    """
+    disassemble Python byte-code from an open file
+    """
+    (version, timestamp, magic_int, co, is_pypy,
+     source_size) = load_from_fp(fp)
+    if type(co) == list:
+        for con in co:
+            disco(version, con, outstream)
+    else:
+        disco(version, co, outstream, is_pypy=is_pypy)
+    co = None
+
 def disassemble_file(filename, outstream=None):
     """
     disassemble Python byte-code file (.pyc)

uncompyle6/main.py

@@ -17,6 +17,7 @@ import datetime, os, subprocess, sys
 
 from uncompyle6 import verify, IS_PYPY
 from xdis.code import iscode
+from xdis.magics import sysinfo2float
 from uncompyle6.disas import check_object_path
 from uncompyle6.semantics import pysource
 from uncompyle6.parser import ParserError
@@ -48,8 +49,14 @@ def decompile(
     """
     ingests and deparses a given code block 'co'
 
+    if `bytecode_version` is None, use the current Python intepreter
+    version.
+
     Caller is responsible for closing `out` and `mapstream`
     """
+    if bytecode_version is None:
+        bytecode_version = sysinfo2float()
+
     # store final output stream for case of error
     real_out = out or sys.stdout
 
@@ -162,9 +169,11 @@ def main(in_base, out_base, files, codes, outfile=None,
     """
     in_base	base directory for input files
     out_base	base directory for output files (ignored when
-    files	list of filenames to be uncompyled (relative to src_base)
+    files	list of filenames to be uncompyled (relative to in_base)
     outfile	write output to this filename (overwrites out_base)
 
+    Note: `codes` is not use. Historical compatability?
+
     For redirecting output to
     - <filename>		outfile=<filename> (out_base is ignored)
     - files below out_base	out_base=...

uncompyle6/parser.py

@@ -39,9 +39,14 @@ nop_func = lambda self, args: None
 
 class PythonParser(GenericASTBuilder):
 
-    def __init__(self, AST, start, debug):
-        super(PythonParser, self).__init__(AST, start, debug)
+    def __init__(self, SyntaxTree, start, debug):
+        super(PythonParser, self).__init__(SyntaxTree, start, debug)
         # FIXME: customize per python parser version
+
+        # These are the non-terminals we should collect into a list.
+        # For example instead of:
+        #   stmts -> stmts stmt -> stmts stmt stmt ...
+        # collect as stmts -> stmt stmt ...
         nt_list = [
             'stmts', 'except_stmts', '_stmts', 'attributes',
             'exprlist', 'kvlist', 'kwargs', 'come_froms', '_come_froms',
@@ -618,18 +623,31 @@ def get_python_parser(
     # a lazy way of doing the import?
 
     if version < 3.0:
-        if version == 1.5:
-            import uncompyle6.parsers.parse15 as parse15
-            if compile_mode == 'exec':
-                p = parse15.Python15Parser(debug_parser)
-            else:
-                p = parse15.Python15ParserSingle(debug_parser)
-        elif version == 2.1:
-            import uncompyle6.parsers.parse21 as parse21
-            if compile_mode == 'exec':
-                p = parse21.Python21Parser(debug_parser)
-            else:
-                p = parse21.Python21ParserSingle(debug_parser)
+        if version < 2.2:
+            if version == 1.3:
+                import uncompyle6.parsers.parse13 as parse13
+                if compile_mode == 'exec':
+                    p = parse13.Python14Parser(debug_parser)
+                else:
+                    p = parse13.Python14ParserSingle(debug_parser)
+            elif version == 1.4:
+                import uncompyle6.parsers.parse14 as parse14
+                if compile_mode == 'exec':
+                    p = parse14.Python14Parser(debug_parser)
+                else:
+                    p = parse14.Python14ParserSingle(debug_parser)
+            elif version == 1.5:
+                import uncompyle6.parsers.parse15 as parse15
+                if compile_mode == 'exec':
+                    p = parse15.Python15Parser(debug_parser)
+                else:
+                    p = parse15.Python15ParserSingle(debug_parser)
+            elif version == 2.1:
+                import uncompyle6.parsers.parse21 as parse21
+                if compile_mode == 'exec':
+                    p = parse21.Python21Parser(debug_parser)
+                else:
+                    p = parse21.Python21ParserSingle(debug_parser)
         elif version == 2.2:
             import uncompyle6.parsers.parse22 as parse22
             if compile_mode == 'exec':

uncompyle6/parsers/parse13.py

@@ -0,0 +1,49 @@
+#  Copyright (c) 2018 Rocky Bernstein
+
+from spark_parser import DEFAULT_DEBUG as PARSER_DEFAULT_DEBUG
+from uncompyle6.parser import PythonParserSingle
+from uncompyle6.parsers.parse14 import Python14Parser
+
+class Python13Parser(Python14Parser):
+
+    def p_misc13(self, args):
+        """
+        # Nothing here yet, but will need to add LOAD_GLOBALS
+        """
+
+    def __init__(self, debug_parser=PARSER_DEFAULT_DEBUG):
+        super(Python13Parser, self).__init__(debug_parser)
+        self.customized = {}
+
+    # def customize_grammar_rules(self, tokens, customize):
+    #     super(Python13Parser, self).customize_grammar_rules(tokens, customize)
+    #     self.remove_rules("""
+    #     whileelsestmt ::= SETUP_LOOP testexpr l_stmts_opt
+    #                       jb_pop
+    #                       POP_BLOCK else_suitel COME_FROM
+    #     """)
+    #     self.check_reduce['doc_junk'] = 'tokens'
+
+
+    # def reduce_is_invalid(self, rule, ast, tokens, first, last):
+    #     invalid = super(Python14Parser,
+    #                     self).reduce_is_invalid(rule, ast,
+    #                                             tokens, first, last)
+    #     if invalid or tokens is None:
+    #         return invalid
+    #     if rule[0] == 'doc_junk':
+    #         return not isinstance(tokens[first].pattr, str)
+
+
+
+class Python13ParserSingle(Python13Parser, PythonParserSingle):
+    pass
+
+if __name__ == '__main__':
+    # Check grammar
+    p = Python13Parser()
+    p.check_grammar()
+    p.dump_grammar()
+
+# local variables:
+# tab-width: 4

uncompyle6/parsers/parse14.py

@@ -8,15 +8,53 @@ class Python14Parser(Python15Parser):
 
     def p_misc14(self, args):
         """
-        # Nothing here yet, but will need to add UNARY_CALL, BINARY_CALL,
+        # Not much here yet, but will probably need to add UNARY_CALL, BINARY_CALL,
         # RAISE_EXCEPTION, BUILD_FUNCTION, UNPACK_ARG, UNPACK_VARARG, LOAD_LOCAL,
         # SET_FUNC_ARGS, and RESERVE_FAST
+
+        # Not strictly needed, but tidies up output
+        stmt     ::= doc_junk
+        doc_junk ::= LOAD_CONST POP_TOP
+
+        # Not sure why later Python's omit the COME_FROM
+        jb_pop14  ::= JUMP_BACK COME_FROM POP_TOP
+
+        whileelsestmt ::= SETUP_LOOP testexpr l_stmts_opt
+                          jb_pop14
+                          POP_BLOCK else_suitel COME_FROM
+
+        print_items_nl_stmt ::= expr PRINT_ITEM_CONT print_items_opt PRINT_NEWLINE_CONT
+
+
+        # 1.4 doesn't have linenotab, and although this shouldn't
+        # be a show stopper, our CONTINUE detection is off here.
+        continue ::= JUMP_BACK
         """
 
     def __init__(self, debug_parser=PARSER_DEFAULT_DEBUG):
         super(Python14Parser, self).__init__(debug_parser)
         self.customized = {}
 
+    def customize_grammar_rules(self, tokens, customize):
+        super(Python14Parser, self).customize_grammar_rules(tokens, customize)
+        self.remove_rules("""
+        whileelsestmt ::= SETUP_LOOP testexpr l_stmts_opt
+                          jb_pop
+                          POP_BLOCK else_suitel COME_FROM
+        """)
+        self.check_reduce['doc_junk'] = 'tokens'
+
+
+    def reduce_is_invalid(self, rule, ast, tokens, first, last):
+        invalid = super(Python14Parser,
+                        self).reduce_is_invalid(rule, ast,
+                                                tokens, first, last)
+        if invalid or tokens is None:
+            return invalid
+        if rule[0] == 'doc_junk':
+            return not isinstance(tokens[first].pattr, str)
+
+
 
 class Python14ParserSingle(Python14Parser, PythonParserSingle):
     pass

uncompyle6/parsers/parse15.py

@@ -2,7 +2,7 @@
 #  Copyright (c) 2000-2002 by hartmut Goebel <hartmut@goebel.noris.de>
 
 from spark_parser import DEFAULT_DEBUG as PARSER_DEFAULT_DEBUG
-from uncompyle6.parser import PythonParserSingle
+from uncompyle6.parser import PythonParserSingle, nop_func
 from uncompyle6.parsers.parse21 import Python21Parser
 
 class Python15Parser(Python21Parser):
@@ -23,6 +23,17 @@ class Python15Parser(Python21Parser):
         importlist  ::= IMPORT_FROM
         """
 
+    def customize_grammar_rules(self, tokens, customize):
+        super(Python15Parser, self).customize_grammar_rules(tokens, customize)
+        for i, token in enumerate(tokens):
+            opname = token.kind
+            opname_base = opname[:opname.rfind('_')]
+
+            if opname_base == 'UNPACK_LIST':
+                self.addRule("store ::= unpack_list", nop_func)
+
+
+
 class Python15ParserSingle(Python15Parser, PythonParserSingle):
     pass
 

uncompyle6/parsers/parse2.py

@@ -1,4 +1,4 @@
-#  Copyright (c) 2015-2017 Rocky Bernstein
+#  Copyright (c) 2015-2018 Rocky Bernstein
 #  Copyright (c) 2000-2002 by hartmut Goebel <h.goebel@crazy-compilers.com>
 #
 #  Copyright (c) 1999 John Aycock
@@ -26,13 +26,13 @@ that a later phase can turn into a sequence of ASCII text.
 """
 
 from uncompyle6.parser import PythonParser, PythonParserSingle, nop_func
-from uncompyle6.parsers.astnode import AST
+from uncompyle6.parsers.treenode import SyntaxTree
 from spark_parser import DEFAULT_DEBUG as PARSER_DEFAULT_DEBUG
 
 class Python2Parser(PythonParser):
 
     def __init__(self, debug_parser=PARSER_DEFAULT_DEBUG):
-        super(Python2Parser, self).__init__(AST, 'stmts', debug=debug_parser)
+        super(Python2Parser, self).__init__(SyntaxTree, 'stmts', debug=debug_parser)
         self.new_rules = set()
 
     def p_print2(self, args):
@@ -156,8 +156,10 @@ class Python2Parser(PythonParser):
         try_except      ::= SETUP_EXCEPT suite_stmts_opt POP_BLOCK
                             except_handler COME_FROM
 
+        # Note: except_stmts may have many jumps after END_FINALLY
         except_handler  ::= JUMP_FORWARD COME_FROM except_stmts
-                            END_FINALLY COME_FROM
+                            END_FINALLY come_froms
+
         except_handler  ::= jmp_abs COME_FROM except_stmts
                              END_FINALLY
 
@@ -292,9 +294,20 @@ class Python2Parser(PythonParser):
 
             # The order of opname listed is roughly sorted below
             if opname_base in ('BUILD_LIST', 'BUILD_SET', 'BUILD_TUPLE'):
-                v = token.attr
+                # We do this complicated test to speed up parsing of
+                # pathelogically long literals, especially those over 1024.
+                build_count = token.attr
+                thousands = (build_count//1024)
+                thirty32s = ((build_count//32) % 32)
+                if thirty32s > 0:
+                    rule = "expr32 ::=%s" % (' expr' * 32)
+                    self.add_unique_rule(rule, opname_base, build_count, customize)
+                if thousands > 0:
+                    self.add_unique_rule("expr1024 ::=%s" % (' expr32' * 32),
+                                         opname_base, build_count, customize)
                 collection = opname_base[opname_base.find('_')+1:].lower()
-                rule = '%s ::= %s%s' % (collection, (token.attr * 'expr '), opname)
+                rule = (('%s ::= ' % collection) + 'expr1024 '*thousands +
+                        'expr32 '*thirty32s + 'expr '*(build_count % 32) + opname)
                 self.add_unique_rules([
                     "expr ::= %s" % collection,
                     rule], customize)
@@ -393,7 +406,6 @@ class Python2Parser(PythonParser):
                     """, nop_func)
                 continue
             elif opname == 'JUMP_IF_NOT_DEBUG':
-                v = token.attr
                 self.addRule("""
                     jmp_true_false ::= POP_JUMP_IF_TRUE
                     jmp_true_false ::= POP_JUMP_IF_FALSE

uncompyle6/parsers/parse24.py

@@ -32,6 +32,11 @@ class Python24Parser(Python25Parser):
         importmultiple ::= filler LOAD_CONST alias imports_cont
         import_cont    ::= filler LOAD_CONST alias
 
+        # Handle "if true else: ..." in Python 2.4
+        stmt            ::= iftrue_stmt24
+        iftrue_stmt24   ::= _ifstmts_jump24 suite_stmts COME_FROM
+        _ifstmts_jump24 ::= c_stmts_opt JUMP_FORWARD POP_TOP
+
         # Python 2.5+ omits POP_TOP POP_BLOCK
         while1stmt ::= SETUP_LOOP l_stmts_opt JUMP_BACK POP_TOP POP_BLOCK COME_FROM
         while1stmt ::= SETUP_LOOP l_stmts_opt JUMP_BACK POP_TOP POP_BLOCK

uncompyle6/parsers/parse26.py

@@ -155,7 +155,7 @@ class Python26Parser(Python2Parser):
         iflaststmtl ::= testexpr c_stmts_opt jb_cf_pop
         iflaststmt  ::= testexpr c_stmts_opt JUMP_ABSOLUTE come_from_pop
 
-        lastc_stmt ::= iflaststmt COME_FROM
+        lastc_stmt ::= iflaststmt come_froms
 
         ifstmt         ::= testexpr_then _ifstmts_jump
 
@@ -265,22 +265,21 @@ class Python26Parser(Python2Parser):
         kvlist ::= kvlist kv3
 
         # Note: preserve positions 0 2 and 4 for semantic actions
-        conditional_not ::= expr jmp_true  expr jf_cf_pop expr COME_FROM
-        conditional     ::= expr jmp_false expr jf_cf_pop expr come_from_opt
-        expr            ::= conditional_not
+        conditional_not    ::= expr jmp_true  expr jf_cf_pop expr COME_FROM
+        conditional        ::= expr jmp_false expr jf_cf_pop expr come_from_opt
+        expr               ::= conditional_not
 
-        and          ::= expr JUMP_IF_FALSE POP_TOP expr JUMP_IF_FALSE POP_TOP
+        and                ::= expr JUMP_IF_FALSE POP_TOP expr JUMP_IF_FALSE POP_TOP
 
         # compare_chained is like x <= y <= z
         compare_chained    ::= expr compare_chained1 ROT_TWO COME_FROM POP_TOP _come_froms
-        compare_chained1   ::= expr DUP_TOP ROT_THREE COMPARE_OP
-                               jmp_false_then compare_chained1 _come_froms
-        compare_chained1   ::= expr DUP_TOP ROT_THREE COMPARE_OP
-                               jmp_false_then compare_chained2 _come_froms
         compare_chained1   ::= expr DUP_TOP ROT_THREE COMPARE_OP
                                jmp_false compare_chained1 _come_froms
         compare_chained1   ::= expr DUP_TOP ROT_THREE COMPARE_OP
                                jmp_false compare_chained2 _come_froms
+
+        compare_chained1   ::= expr DUP_TOP ROT_THREE COMPARE_OP
+                               jmp_false_then compare_chained1 _come_froms
         compare_chained1   ::= expr DUP_TOP ROT_THREE COMPARE_OP
                                jmp_false_then compare_chained2 _come_froms
 
@@ -349,7 +348,7 @@ class Python26Parser(Python2Parser):
             # For now, we won't let the 2nd 'expr' be a "conditional_not"
             # However in < 2.6 where we don't have if/else expression it *can*
             # be.
-            if ast[2][0] == 'conditional_not':
+            if self.version >= 2.6 and ast[2][0] == 'conditional_not':
                 return True
 
             test_index = last

uncompyle6/parsers/parse27.py

@@ -52,6 +52,8 @@ class Python27Parser(Python2Parser):
     def p_try27(self, args):
         """
         # If the last except is a "raise" we might not have a final COME_FROM
+        # FIXME: need a check on this rule since this accepts try_except when
+        # we shouldn't
         try_except      ::= SETUP_EXCEPT suite_stmts_opt POP_BLOCK
                             except_handler
 
@@ -261,7 +263,7 @@ if __name__ == '__main__':
             """.split()))
         remain_tokens = set(tokens) - opcode_set
         import re
-        remain_tokens = set([re.sub('_\d+$', '', t)
+        remain_tokens = set([re.sub(r'_\d+$', '', t)
                              for t in remain_tokens])
         remain_tokens = set([re.sub('_CONT$', '', t)
                              for t in remain_tokens])

uncompyle6/parsers/parse3.py

@@ -27,7 +27,7 @@ that a later phase can turn into a sequence of ASCII text.
 """
 
 from uncompyle6.parser import PythonParser, PythonParserSingle, nop_func
-from uncompyle6.parsers.astnode import AST
+from uncompyle6.parsers.treenode import SyntaxTree
 from spark_parser import DEFAULT_DEBUG as PARSER_DEFAULT_DEBUG
 from xdis import PYTHON3
 from itertools import islice,chain,repeat
@@ -36,7 +36,7 @@ class Python3Parser(PythonParser):
 
     def __init__(self, debug_parser=PARSER_DEFAULT_DEBUG):
         self.added_rules = set()
-        super(Python3Parser, self).__init__(AST, 'stmts', debug=debug_parser)
+        super(Python3Parser, self).__init__(SyntaxTree, 'stmts', debug=debug_parser)
         self.new_rules = set()
 
     def p_comprehension3(self, args):
@@ -710,11 +710,26 @@ class Python3Parser(PythonParser):
                         rule = ('load_closure ::= %s%s' % (('LOAD_CLOSURE ' * v), opname))
                         self.add_unique_rule(rule, opname, token.attr, customize)
                 if not is_LOAD_CLOSURE or v == 0:
+                    # We do this complicated test to speed up parsing of
+                    # pathelogically long literals, especially those over 1024.
+                    build_count = token.attr
+                    thousands = (build_count//1024)
+                    thirty32s = ((build_count//32) % 32)
+                    if thirty32s > 0:
+                        rule = "expr32 ::=%s" % (' expr' * 32)
+                        self.add_unique_rule(rule, opname_base, build_count, customize)
+                        pass
+                    if thousands > 0:
+                        self.add_unique_rule("expr1024 ::=%s" % (' expr32' * 32),
+                                             opname_base, build_count, customize)
+                        pass
                     collection = opname_base[opname_base.find('_')+1:].lower()
-                    rule = '%s ::= %s%s' % (collection, 'expr ' * v, opname)
+                    rule = (('%s ::= ' % collection) + 'expr1024 '*thousands +
+                            'expr32 '*thirty32s + 'expr '*(build_count % 32) + opname)
                     self.add_unique_rules([
-                        'expr ::= %s' % collection,
+                        "expr ::= %s" % collection,
                         rule], customize)
+                    continue
                 continue
             elif opname_base == 'BUILD_SLICE':
                 if token.attr == 2:

uncompyle6/parsers/parse30.py

@@ -11,6 +11,10 @@ class Python30Parser(Python31Parser):
     def p_30(self, args):
         """
 
+        assert            ::= assert_expr jmp_true LOAD_ASSERT RAISE_VARARGS_1 POP_TOP
+        return_if_lambda  ::= RETURN_END_IF_LAMBDA POP_TOP
+        compare_chained2  ::= expr COMPARE_OP RETURN_END_IF_LAMBDA
+
         # FIXME: combine with parse3.2
         whileTruestmt     ::= SETUP_LOOP l_stmts_opt JUMP_BACK
                               COME_FROM_LOOP
@@ -19,29 +23,131 @@ class Python30Parser(Python31Parser):
 
         # In many ways Python 3.0 code generation is more like Python 2.6 than
         # it is 2.7 or 3.1. So we have a number of 2.6ish (and before) rules below
+        # Specifically POP_TOP is more prevelant since there is no POP_JUMP_IF_...
+        # instructions
 
-        _ifstmts_jump  ::= c_stmts_opt JUMP_FORWARD _come_froms POP_TOP COME_FROM
-        jmp_true       ::= JUMP_IF_TRUE POP_TOP
-        jmp_false      ::= JUMP_IF_FALSE POP_TOP
+        _ifstmts_jump  ::= c_stmts JUMP_FORWARD _come_froms POP_TOP COME_FROM
+        _ifstmts_jump  ::= c_stmts POP_TOP
 
         # Used to keep index order the same in semantic actions
         jb_pop_top     ::= JUMP_BACK POP_TOP
 
-        while1stmt ::= SETUP_LOOP l_stmts COME_FROM_LOOP
+        while1stmt     ::= SETUP_LOOP l_stmts COME_FROM_LOOP
+        whileelsestmt  ::= SETUP_LOOP testexpr l_stmts
+                           jb_pop_top POP_BLOCK
+                           else_suitel COME_FROM_LOOP
+        # while1elsestmt ::= SETUP_LOOP l_stmts
+        #                    jb_pop_top POP_BLOCK
+        #                    else_suitel COME_FROM_LOOP
 
         else_suitel ::= l_stmts COME_FROM_LOOP JUMP_BACK
 
         ifelsestmtl ::= testexpr c_stmts_opt jb_pop_top else_suitel
+        iflaststmtl ::= testexpr c_stmts_opt jb_pop_top
+        iflaststmt  ::= testexpr c_stmts_opt JUMP_ABSOLUTE POP_TOP
 
         withasstmt    ::= expr setupwithas store suite_stmts_opt
                           POP_BLOCK LOAD_CONST COME_FROM_FINALLY
                           LOAD_FAST DELETE_FAST WITH_CLEANUP END_FINALLY
         setupwithas   ::= DUP_TOP LOAD_ATTR STORE_FAST LOAD_ATTR CALL_FUNCTION_0 setup_finally
         setup_finally ::= STORE_FAST SETUP_FINALLY LOAD_FAST DELETE_FAST
+
+        # Need to keep LOAD_FAST as index 1
+        set_comp_func_header ::=  BUILD_SET_0 DUP_TOP STORE_FAST
+        set_comp_func ::= set_comp_func_header
+                          LOAD_FAST FOR_ITER store comp_iter
+                          JUMP_BACK POP_TOP JUMP_BACK RETURN_VALUE RETURN_LAST
+
+        list_comp_header ::= BUILD_LIST_0 DUP_TOP STORE_FAST
+        list_comp        ::= list_comp_header
+                             LOAD_FAST FOR_ITER store comp_iter
+                             JUMP_BACK
+
+        set_comp_header  ::= BUILD_SET_0 DUP_TOP STORE_FAST
+        set_comp         ::= set_comp_header
+                             LOAD_FAST FOR_ITER store comp_iter
+                             JUMP_BACK
+
+        dict_comp_header ::= BUILD_MAP_0 DUP_TOP STORE_FAST
+        dict_comp        ::= dict_comp_header
+                             LOAD_FAST FOR_ITER store dict_comp_iter
+                             JUMP_BACK
+
+        dict_comp_iter   ::= expr expr ROT_TWO expr STORE_SUBSCR
+
+        # JUMP_IF_TRUE POP_TOP as a replacement
+        comp_if       ::= expr jmp_false comp_iter
+        comp_if       ::= expr jmp_false comp_iter JUMP_BACK POP_TOP
+        comp_if_not   ::= expr jmp_true  comp_iter JUMP_BACK POP_TOP
+        comp_iter     ::= expr expr SET_ADD
+        comp_iter     ::= expr expr LIST_APPEND
+
+        jump_forward_else     ::= JUMP_FORWARD POP_TOP
+        jump_absolute_else    ::= JUMP_ABSOLUTE POP_TOP
+        except_suite          ::= c_stmts POP_EXCEPT jump_except POP_TOP
+        except_suite_finalize ::= SETUP_FINALLY c_stmts_opt except_var_finalize END_FINALLY
+                                  _jump POP_TOP
+        jump_except           ::= JUMP_FORWARD POP_TOP
+        or                    ::= expr jmp_false expr jmp_true expr
+        or                    ::= expr jmp_true expr
+
+        ################################################################################
+        # In many ways 3.0 is like 2.6. One similarity is there is no JUMP_IF_TRUE and
+        # JUMP_IF_FALSE
+        # The below rules in fact are the same or similar.
+
+        jmp_true       ::= JUMP_IF_TRUE POP_TOP
+        jmp_false      ::= JUMP_IF_FALSE POP_TOP
+
+        for_block      ::= l_stmts_opt _come_froms POP_TOP JUMP_BACK
+
+        except_handler ::= JUMP_FORWARD COME_FROM_EXCEPT except_stmts
+                           POP_TOP END_FINALLY come_froms
+        except_handler ::= jmp_abs COME_FROM_EXCEPT except_stmts
+                           POP_TOP END_FINALLY
+
+        return_if_stmt ::= ret_expr RETURN_END_IF POP_TOP
+        and            ::= expr jmp_false expr come_from_opt
+        whilestmt      ::= SETUP_LOOP testexpr l_stmts_opt come_from_opt
+                           JUMP_BACK POP_TOP POP_BLOCK COME_FROM_LOOP
+        whilestmt      ::= SETUP_LOOP testexpr returns
+                           POP_TOP POP_BLOCK COME_FROM_LOOP
+
+
+        # compare_chained is like x <= y <= z
+        compare_chained1  ::= expr DUP_TOP ROT_THREE COMPARE_OP
+                              jmp_false compare_chained1 _come_froms
+        compare_chained1  ::= expr DUP_TOP ROT_THREE COMPARE_OP
+                              jmp_false compare_chained2 _come_froms
+        compare_chained2 ::= expr COMPARE_OP RETURN_END_IF
         """
 
     def customize_grammar_rules(self, tokens, customize):
         super(Python30Parser, self).customize_grammar_rules(tokens, customize)
+        self.remove_rules("""
+        iflaststmtl        ::= testexpr c_stmts_opt JUMP_BACK COME_FROM_LOOP
+        ifelsestmtl        ::= testexpr c_stmts_opt JUMP_BACK else_suitel
+        iflaststmt         ::= testexpr c_stmts_opt JUMP_ABSOLUTE
+        _ifstmts_jump      ::= c_stmts_opt JUMP_FORWARD _come_froms
+        jump_forward_else  ::= JUMP_FORWARD ELSE
+        jump_absolute_else ::= JUMP_ABSOLUTE ELSE
+        whilestmt          ::= SETUP_LOOP testexpr l_stmts_opt COME_FROM JUMP_BACK POP_BLOCK
+                               COME_FROM_LOOP
+        whilestmt          ::= SETUP_LOOP testexpr returns
+                               POP_BLOCK COME_FROM_LOOP
+        assert             ::= assert_expr jmp_true LOAD_ASSERT RAISE_VARARGS_1
+        return_if_lambda   ::= RETURN_END_IF_LAMBDA
+        except_suite       ::= c_stmts POP_EXCEPT jump_except
+        whileelsestmt      ::= SETUP_LOOP testexpr l_stmts JUMP_BACK POP_BLOCK
+                               else_suitel COME_FROM_LOOP
+
+        # No JUMP_IF_FALSE_OR_POP
+        compare_chained1 ::= expr DUP_TOP ROT_THREE COMPARE_OP JUMP_IF_FALSE_OR_POP
+                             compare_chained1 COME_FROM
+        compare_chained1 ::= expr DUP_TOP ROT_THREE COMPARE_OP JUMP_IF_FALSE_OR_POP
+                             compare_chained2 COME_FROM
+        """)
+
         return
     pass
 

uncompyle6/parsers/parse36.py

@@ -41,6 +41,11 @@ class Python36Parser(Python35Parser):
         whilestmt       ::= SETUP_LOOP testexpr l_stmts_opt
                             JUMP_BACK come_froms POP_BLOCK COME_FROM_LOOP
 
+        # 3.6 due to jump optimization, we sometimes add RETURN_END_IF where
+        # RETURN_VALUE is meant. Specifcally this can happen in
+        # ifelsestmt -> ...else_suite _. suite_stmts... (last) stmt
+        return ::= ret_expr RETURN_END_IF
+
         # A COME_FROM is dropped off because of JUMP-to-JUMP optimization
         and  ::= expr jmp_false expr
         and  ::= expr jmp_false expr jmp_false
@@ -105,6 +110,10 @@ class Python36Parser(Python35Parser):
                              COME_FROM_FINALLY suite_stmts_opt END_FINALLY
         except_suite_finalize ::= SETUP_FINALLY returns
                                   COME_FROM_FINALLY suite_stmts_opt END_FINALLY _jump
+
+        stmt ::= tryfinally_return_stmt
+        tryfinally_return_stmt ::= SETUP_FINALLY suite_stmts_opt POP_BLOCK LOAD_CONST
+                                   COME_FROM_FINALLY
         """
 
     def customize_grammar_rules(self, tokens, customize):

uncompyle6/parsers/treenode.py

@@ -6,10 +6,10 @@ from spark_parser.ast import AST as spark_AST
 if PYTHON3:
     intern = sys.intern
 
-class AST(spark_AST):
+class SyntaxTree(spark_AST):
     def isNone(self):
-        """An AST None token. We can't use regular list comparisons
-        because AST token offsets might be different"""
+        """An SyntaxTree None token. We can't use regular list comparisons
+        because SyntaxTree token offsets might be different"""
         return len(self.data) == 1 and NoneToken == self.data[0]
 
     def __repr__(self):

uncompyle6/scanner.py

@@ -39,7 +39,7 @@ else:
 
 # The byte code versions we support.
 # Note: these all have to be floats
-PYTHON_VERSIONS = frozenset((1.4, 1.5,
+PYTHON_VERSIONS = frozenset((1.3, 1.4, 1.5,
                              2.1, 2.2, 2.3, 2.4, 2.5, 2.6, 2.7,
                              3.0, 3.1, 3.2, 3.3, 3.4, 3.5, 3.6, 3.7))
 
@@ -505,22 +505,26 @@ def get_scanner(version, is_pypy=False, show_asm=None):
     # Pick up appropriate scanner
     if version in PYTHON_VERSIONS:
         v_str = "%s" % (int(version * 10))
-        if PYTHON3:
+        try:
             import importlib
             if is_pypy:
                 scan = importlib.import_module("uncompyle6.scanners.pypy%s" % v_str)
             else:
                 scan = importlib.import_module("uncompyle6.scanners.scanner%s" % v_str)
             if False: print(scan)  # Avoid unused scan
-        else:
+        except ImportError:
             if is_pypy:
-                exec("import uncompyle6.scanners.pypy%s as scan" % v_str)
+                exec("import uncompyle6.scanners.pypy%s as scan" % v_str,
+                     locals(), globals())
             else:
-                exec("import uncompyle6.scanners.scanner%s as scan" % v_str)
+                exec("import uncompyle6.scanners.scanner%s as scan" % v_str,
+                      locals(), globals())
         if is_pypy:
-            scanner = eval("scan.ScannerPyPy%s(show_asm=show_asm)" % v_str)
+            scanner = eval("scan.ScannerPyPy%s(show_asm=show_asm)" % v_str,
+                           locals(), globals())
         else:
-            scanner = eval("scan.Scanner%s(show_asm=show_asm)" % v_str)
+            scanner = eval("scan.Scanner%s(show_asm=show_asm)" % v_str,
+                           locals(), globals())
     else:
         raise RuntimeError("Unsupported Python version %s" % version)
     return scanner

uncompyle6/scanners/scanner13.py

@@ -0,0 +1,35 @@
+#  Copyright (c) 2018 by Rocky Bernstein
+"""
+Python 1.3 bytecode decompiler massaging.
+
+This massages tokenized 1.3 bytecode to make it more amenable for
+grammar parsing.
+"""
+
+import uncompyle6.scanners.scanner14 as scan
+# from uncompyle6.scanners.scanner26 import ingest as  ingest26
+
+# bytecode verification, verify(), uses JUMP_OPs from here
+from xdis.opcodes import opcode_13
+JUMP_OPS = opcode_13.JUMP_OPS
+
+# We base this off of 1.4 instead of the other way around
+# because we cleaned things up this way.
+# The history is that 2.7 support is the cleanest,
+# then from that we got 2.6 and so on.
+class Scanner13(scan.Scanner14):
+    def __init__(self, show_asm=False):
+        scan.Scanner14.__init__(self, show_asm)
+        self.opc = opcode_13
+        self.opname = opcode_13.opname
+        self.version = 1.3
+        return
+
+    # def ingest22(self, co, classname=None, code_objects={}, show_asm=None):
+    #     tokens, customize = self.parent_ingest(co, classname, code_objects, show_asm)
+    #     tokens = [t for t in tokens if t.kind != 'SET_LINENO']
+
+    #     # for t in tokens:
+    #     #     print(t)
+    #
+    #   return tokens, customize

uncompyle6/scanners/scanner14.py

@@ -32,5 +32,5 @@ class Scanner14(scan.Scanner15):
 
     #     # for t in tokens:
     #     #     print(t)
-
-        return tokens, customize
+    #
+    #   return tokens, customize

uncompyle6/scanners/scanner15.py

@@ -25,3 +25,17 @@ class Scanner15(scan.Scanner21):
         self.version = 1.5
         self.genexpr_name = '<generator expression>'
         return
+
+    def ingest(self, co, classname=None, code_objects={}, show_asm=None):
+        """
+        Pick out tokens from an uncompyle6 code object, and transform them,
+        returning a list of uncompyle6 Token's.
+
+        The transformations are made to assist the deparsing grammar.
+        """
+        tokens, customize = scan.Scanner21.ingest(self, co, classname, code_objects, show_asm)
+        for t in tokens:
+            if t.op == self.opc.UNPACK_LIST:
+                t.kind = 'UNPACK_LIST_%d' % t.attr
+            pass
+        return tokens, customize

uncompyle6/scanners/scanner2.py

@@ -43,6 +43,10 @@ from xdis.bytecode import (
     _get_const_info)
 from xdis.util import code2num
 
+from uncompyle6 import PYTHON3
+if PYTHON3:
+    from sys import intern
+
 from uncompyle6.scanner import Scanner, Token
 
 class Scanner2(Scanner):
@@ -106,6 +110,17 @@ class Scanner2(Scanner):
 
 
 
+    @staticmethod
+    def extended_arg_val(arg):
+        """Return integer value of an EXTENDED_ARG operand.
+        In Python2 this always the operand value shifted 16 bits since
+        the operand is always 2 bytes. In Python 3.6+ this changes to one byte.
+        """
+        if PYTHON3:
+            return (arg << 16)
+        else:
+            return (arg << long(16))
+
     @staticmethod
     def unmangle_name(name, classname):
         """Remove __ from the end of _name_ if it starts with __classname__
@@ -1081,7 +1096,7 @@ class Scanner2(Scanner):
                 # FIXME FIXME FIXME
                 # All the conditions are horrible, and I am not sure I
                 # undestand fully what's going l
-                # WeR REALLY REALLY  need a better way to handle control flow
+                # We REALLY REALLY  need a better way to handle control flow
                 # Expecially for < 2.7
                 if label is not None and label != -1:
                     if self.version == 2.7:

uncompyle6/scanners/scanner3.py

@@ -78,8 +78,10 @@ class Scanner3(Scanner):
 
         if self.version == 3.0:
             self.pop_jump_tf = frozenset([self.opc.JUMP_IF_FALSE, self.opc.JUMP_IF_TRUE])
+            self.not_continue_follow = ('END_FINALLY', 'POP_BLOCK', 'POP_TOP')
         else:
             self.pop_jump_tf = frozenset([self.opc.PJIF, self.opc.PJIT])
+            self.not_continue_follow = ('END_FINALLY', 'POP_BLOCK')
 
         self.setup_ops_no_loop = frozenset(setup_ops) - frozenset([self.opc.SETUP_LOOP])
 
@@ -217,7 +219,12 @@ class Scanner3(Scanner):
             # If we have a JUMP_FORWARD after the
             # RAISE_VARARGS then we have a "raise" statement
             # else we have an "assert" statement.
-            if inst.opname == 'POP_JUMP_IF_TRUE' and i+1 < n:
+            if self.version == 3.0:
+                # There is a an implied JUMP_IF_TRUE that we are not testing for (yet?) here
+                assert_can_follow = inst.opname == 'POP_TOP' and i+1 < n
+            else:
+                assert_can_follow = inst.opname == 'POP_JUMP_IF_TRUE' and i+1 < n
+            if assert_can_follow:
                 next_inst = self.insts[i+1]
                 if (next_inst.opname == 'LOAD_GLOBAL' and
                     next_inst.argval == 'AssertionError'):
@@ -395,13 +402,9 @@ class Scanner3(Scanner):
                                   .opname == 'FOR_ITER'
                                   and self.insts[i+1].opname == 'JUMP_FORWARD')
 
-
                     if (is_continue or
-                        (inst.offset in self.stmts and
-                        (self.version != 3.0 or (hasattr(inst, 'linestart'))) and
-                        (next_opname not in ('END_FINALLY', 'POP_BLOCK',
-                                            # Python 3.0 only uses POP_TOP
-                                            'POP_TOP')))):
+                        (inst.offset in self.stmts and (inst.starts_line and
+                        next_opname not in self.not_continue_follow))):
                         opname = 'CONTINUE'
                     else:
                         opname = 'JUMP_BACK'
@@ -663,11 +666,14 @@ class Scanner3(Scanner):
                      and code[return_val_offset1] == self.opc.RETURN_VALUE)):
                     jump_back = None
             if not jump_back:
+                # loop suite ends in return
                 jump_back = self.last_instr(start, end, self.opc.RETURN_VALUE)
                 if not jump_back:
                     return
 
-                jump_back += 2  # FIXME ???
+                jb_inst = self.get_inst(jump_back)
+                jump_back = self.next_offset(jb_inst.opcode, jump_back)
+
                 if_offset = None
                 if code[self.prev_op[next_line_byte]] not in self.pop_jump_tf:
                     if_offset = self.prev[next_line_byte]
@@ -700,18 +706,15 @@ class Scanner3(Scanner):
                     loop_type = 'for'
                 else:
                     loop_type = 'while'
-                    if next_line_byte < len(code):
-                        test_inst = self.insts[self.offset2inst_index[next_line_byte]-1]
-                        if test_inst.offset == offset:
-                            loop_type = 'while 1'
-                        elif test_inst.opcode in self.opc.JUMP_OPs:
-                            self.ignore_if.add(test_inst.offset)
-                            test_target = self.get_target(test_inst.offset)
-                            if test_target > (jump_back+3):
-                                jump_back = test_target
-                                pass
-                            pass
-                        pass
+                    test = self.prev_op[next_line_byte]
+
+                    if test == offset:
+                        loop_type = 'while 1'
+                    elif self.code[test] in self.opc.JUMP_OPs:
+                        self.ignore_if.add(test)
+                        test_target = self.get_target(test)
+                        if test_target > (jump_back+3):
+                            jump_back = test_target
                 self.not_continue.add(jump_back)
             self.loops.append(target)
             self.structs.append({'type': loop_type + '-loop',
@@ -962,8 +965,6 @@ class Scanner3(Scanner):
         elif op == self.opc.POP_EXCEPT:
             next_offset = xdis.next_offset(op, self.opc, offset)
             target = self.get_target(next_offset)
-            if target is None:
-                from trepan.api import debug; debug()
             if target > next_offset:
                 next_op = code[next_offset]
                 if (self.opc.JUMP_ABSOLUTE == next_op and
@@ -991,8 +992,9 @@ class Scanner3(Scanner):
             # In RETURN_VALUE, JUMP_ABSOLUTE, RETURN_VALUE is never RETURN_END_IF
             if op == self.opc.RETURN_VALUE:
                 next_offset = xdis.next_offset(op, self.opc, offset)
-                if (next_offset < len(code) and code[next_offset] == self.opc.JUMP_ABSOLUTE and
-                    offset in self.return_end_ifs):
+                if ( next_offset < len(code) and
+                     (code[next_offset] == self.opc.JUMP_ABSOLUTE and
+                      offset in self.return_end_ifs) ):
                     self.return_end_ifs.remove(offset)
                     pass
                 pass