Get ready for release 3.7.4

build_ast() added a code argument
n_dict(): was out of sync with corresponding pysource routine
pysource.py: small doc typo

.circleci/config.yml

@@ -1,6 +1,7 @@
 version: 2
 jobs:
   build:
+    working_directory: ~/rocky/python-uncompyle6
     parallelism: 1
     shell: /bin/bash --login
     # CircleCI 2.0 does not support environment variables that refer to each other the same way as 1.0 did.
@@ -12,7 +13,7 @@ jobs:
     # To see the list of pre-built images that CircleCI provides for most common languages see
     # https://circleci.com/docs/2.0/circleci-images/
     docker:
-      - image: circleci/python:2.7
+      - image: circleci/python:3.6.9
     steps:
     # Machine Setup
     #   If you break your build into multiple jobs with workflows, you will probably want to do the parts of this that are relevant in each
@@ -22,6 +23,10 @@ jobs:
     # In many cases you can simplify this from what is generated here.
     # 'See docs on artifact collection here https://circleci.com/docs/2.0/artifacts/'
     - run: mkdir -p $CIRCLE_ARTIFACTS $CIRCLE_TEST_REPORTS
+    # This is based on your 1.0 configuration file or project settings
+    - run:
+        working_directory: ~/rocky/python-uncompyle6
+        command: pip install --user virtualenv && pip install --user nose && pip install --user pep8
     # Dependencies
     #   This would typically go in either a build or a build-and-test job when using workflows
     # Restore the dependency cache
@@ -32,7 +37,10 @@ jobs:
           - v2-dependencies-
 
     - run:
-        command: sudo easy_install xdis spark-parser && sudo pip install -e . && sudo pip install -r requirements-dev.txt
+        command: | # Use pip to install dependengcies
+          pip install --user --upgrade setuptools
+          pip install --user -e .
+          pip install --user -r requirements-dev.txt
 
     # Save dependency cache
     - save_cache:
@@ -50,8 +58,8 @@ jobs:
     # Test
     #   This would typically be a build job when using workflows, possibly combined with build
     # This is based on your 1.0 configuration file or project settings
-    - run: sudo python ./setup.py develop && make check-2.7
-    - run: cd test/stdlib && bash ./runtests.sh 'test_[p-z]*.py'
+    - run: sudo python ./setup.py develop && make check-3.6
+    - run: cd ./test/stdlib && bash ./runtests.sh 'test_[p-z]*.py'
     # Teardown
     #   If you break your build into multiple jobs with workflows, you will probably want to do the parts of this that are relevant in each
     # Save test results

.github/ISSUE_TEMPLATE/bug-report.md

@@ -4,28 +4,40 @@ about: Tell us about uncompyle6 bugs
 
 ---
 
-<!-- __Note:__ Have you read https://github.com/rocky/python-uncompyle6/blob/master/HOW-TO-REPORT-A-BUG.md ?
+<!-- __Note:__ Bugs are not for asking questions about a problem you
+are trying to solve that involve the use of uncompyle6 along the way,
+although I may be more tolerent of this if you sponsor the project.
+
+Also, the unless you are a sponsor of the project, it may take a
+while, maybe a week or so, before the bug report is noticed, let alone
+acted upon.
+
+To set expectations, some legitimate bugs can take years
+to fix, but they eventually do get fixed. Funding the project was
+added to address the problem that there are lots of people seeking
+help and reporting bugs, but few people who are willing or capable of
+providing help or fixing bugs.
+
+Finally, have you read https://github.com/rocky/python-uncompyle6/blob/master/HOW-TO-REPORT-A-BUG.md
+?
+
 
 Please remove any of the optional sections if they are not applicable.
 
-Prerequisites
+Prerequisites/Caveats
 
 * Make sure the bytecode you have can be disassembled with a
   disassembler and produces valid results.
 * Don't put bytecode and corresponding source code on any service that
   requires registration to download.
-* When you open a bug report there is no privacy. If the legitimacy of
-  the activity is deemed suspicous, I may flag it as suspicious,
+* When you open a bug report there is no privacy. If you need privacy, then
+  contact me by email and explain who you are and the need for privacy.
+  But be mindful that you may be asked to sponsor the project for the
+  personal and private help that you are requesting.
+* If the legitimacy of the activity is deemed suspicous, I may flag it as suspicious,
   making the issue even more easy to detect.
 
-Bug reports that violate a prerequisite may be discarded.
-
-Note that there are way more bug-fix requestors than there are bug
-fixers. If you want you need more immediate, confidential or urgent
-assistance
-
-http://www.crazy-compilers.com/decompyle/ offers a byte-code
-decompiler service for versions of Python up to 2.6.
+Bug reports that violate the above may be discarded.
 
 -->
 

.travis.yml

@@ -1,7 +1,16 @@
 language: python
 
 python:
-  - 2.7 # this is a cheat here because travis doesn't do 2.4-2.6
+#  - '3.5'
+#  - '2.7'
+#  - '3.4'
+  - '3.6'
+  - '3.8'
+
+matrix:
+  include:
+    - python: '3.7'
+      dist: xenial  # required for Python >= 3.7 (travis-ci/travis-ci#9069)
 
 install:
 - pip install -e .

HOW-TO-REPORT-A-BUG.md

@@ -30,31 +30,9 @@ files that I know of that will cause problems. See, for example, the
 list in
 [`test/stdlib/runtests.sh`](https://github.com/rocky/python-uncompyle6/blob/master/test/stdlib/runtests.sh).
 
-But I understand: you would the bugs _you_ encounter addressed before
-all the other known bugs.
+There are far more bug reporters than there are bug fixers.
 
-From my standpoint, the good thing about the bugs listed in
-`runtests.sh` is that each test case is small and isolated to a single
-kind of problem. And I'll tend to fix easier, more isolated cases than
-generic "something's wrong" kinds of bugs where I'd have to do a bit
-of work to figure out what's up, if not use some sort of mind reading,
-make some guesses, and perform some experiments to see if the guesses
-are correct. I can't read minds, nor am I into guessing games; I'd
-rather devote the effort spent instead towards fixing bugs that are
-precisely defined.
-
-And it often turns out that by just fixing the well-defined and
-prescribed cases, the ill-defined amorphous cases as well will get
-handled as well.
-
-In sum, you may need to do some work to have the bug you have found
-handled before the hundreds of other bugs, and other things I could be
-doing.
-
-No one is getting paid to work to work on this project, let alone the
-bugs you may have an interest in. If you require decompiling bytecode
-immediately, consider using a decompilation service, listed further
-down in this document.
+Unless you are a sponsor of this project, it may take a while, maybe a week or so, before the bug report is noticed, let alone acted upon. Things eventually get fixed, but it may take years.
 
 # Is it really a bug?
 
@@ -176,11 +154,7 @@ provide the input command and the output from that, please give:
 
 ## But I don't *have* the source code!
 
-Sure, I get it. No problem. There is Python assembly code on parse
-errors, so simply by hand decompile that. To get a full disassembly,
-use `pydisasm` from the [xdis](https://pypi.python.org/pypi/xdis)
-package. Opcodes are described in the documentation for
-the [dis](https://docs.python.org/3.6/library/dis.html) module.
+There is Python assembly code on parse errors, so simply by hand decompile that. To get a full disassembly, use `pydisasm` from the [xdis](https://pypi.python.org/pypi/xdis) package. Opcodes are described in the documentation for the[dis](https://docs.python.org/3.6/library/dis.html) module.
 
 ### But I don't *have* the source code and am incapable of figuring how to do a hand disassembly!
 
@@ -189,7 +163,7 @@ disassemble Python bytecode. And as Richard Feynman once said, "What
 one fool can learn, so can another."
 
 If this is too difficult, or too time consuming, or not of interest to
-you, then perhaps what require is a decompilation service. [Crazy
+you, then you might consider sponsoring the project. [Crazy
 Compilers](http://www.crazy-compilers.com/decompyle/) offers a
 byte-code decompiler service for versions of Python up to 2.6. (If
 there are others around let me know and I'll list them here.)
@@ -217,10 +191,7 @@ likely the problem will be fixed and fixed sooner.
 # Karma
 
 I realize that following the instructions given herein puts a bit of
-burden on the bug reporter. In my opinion, this is justified as
-attempts to balance somewhat the burden and effort needed to fix the
-bug and the attempts to balance number of would-be bug reporters with
-the number of bug fixers. Better bug reporters are more likely to move
+burden on the bug reporter. This is justified as attempts to balance somewhat the burden and effort needed to fix the bug and the attempts to balance number of would-be bug reporters with the number of bug fixers. Better bug reporters are more likely to move
 in the category of bug fixers.
 
 The barrier to reporting a big is pretty small: all you really need is
@@ -228,22 +199,14 @@ a github account, and the ability to type something after clicking
 some buttons. So the reality is that many people just don't bother to
 read these instructions, let alone follow it to any simulacrum.
 
-And the reality is also that bugs sometimes get fixed even though
-these instructions are not followed.
+That said, bugs sometimes get fixed even though these instructions are not followed.
 
-So one factors I may take into consideration is the bug reporter's karma.
+I may take into consideration is the bug reporter's karma.
+
+* Have you demonstrably contributed to open source? I may look at your  github profile to see what contributions you have made, how popular those contributions are, or how popular you are.
+* How appreciative are you? Have you starred this project that you are   seeking help from? Have you starred _any_ github project? And the above  two kind of feed into ...
+* Attitude. Some people feel that they are doing me and the world a great favor by just pointing out that there is a problem whose solution would greatly benefit them. (This might account partially those that have this attitude often don't read or follow instructions such as those given here.)
 
-* Have you demonstrably contributed to open source? I may look at your
-  github profile to see what contributions you have made, how popular
-  those contributions are, or how popular you are.
-* How appreciative are you? Have you starred this project that you are
-  seeking help from? Have you starred _any_ github project? And the above
-  two kind of feed into ...
-* Attitude. Some people feel that they are doing me and the world a
-  great favor by just pointing out that there is a problem whose solution
-  would greatly benefit them. Perhaps this is why they feel that
-  instructions are not to be followed by them, nor any need for
-  showing evidence gratitude when help is offered them.
 
 # Confidentiality of Bug Reports
 
@@ -255,6 +218,8 @@ remains would not be an issue.
 However feel free to remove any comments, and modify variable names
 or constants in the source code.
 
+If there is some legitimate reason to keep confidentiality, you can contact me by email to explain the extenuating circumstances. However I tend to discard without reading anonymous email.
+
 # Ethics
 
 I do not condone using this program for unethical or illegal purposes.

Makefile

@@ -40,9 +40,8 @@ check-3.0 check-3.1 check-3.2 check-3.6:
 check-3.7: pytest
 	$(MAKE) -C test check
 
-#:Tests for Python 2.4-2.5 (don't have pytest)
-check-2.4 check-2.5:
-	$(MAKE) -C test $@
+check-3.8:
+	$(MAKE) -C test check
 
 #:PyPy 2.6.1 PyPy 5.0.1, or PyPy 5.8.0-beta0
 # Skip for now

NEWS.md

@@ -1,3 +1,44 @@
+3.7.4: 2020-8-05
+================
+
+* Fragment parsing was borked. This means deparsing in trepan2/trepan3k was broken
+* 3.7+: narrow precedence for call tatement
+* del_stmt -> delete to better match Python AST
+* 3.8+ Add another `forelsestmt` (found only in a loop)
+* 3.8+ Add precedence on walrus operator
+* More files blackened
+* bump min xdis version
+
+3.7.3: 2020-7-25
+================
+
+Mostly small miscellaneous bug fixes
+
+* `__doc__ = DocDescr()` from `test_descr.py` was getting confused as a docstring.
+* detect 2.7 exchandler range better
+* Add for .. else reduction checks on 2.6 and before
+* Add reduce check for 2.7 augmented assign
+* Add VERSION in a pydoc-friendly way
+
+
+3.7.2: 2020-6-27
+================
+
+* Use newer xdis
+* Docstrings (again) which were broken again on earlier Python
+* Fix 2.6 and 2.7 decompilation bug in handling "list if" comprehensions
+
+
+
+3.7.1: 2020-6-12 Fleetwood66
+====================================================
+
+Released to pick up new xdis version which has fixes to read bytestings better on 3.x
+
+* Handle 3.7+ "else" branch removal adAs seen in `_cmp()` of `python3.8/distutils/version.py` with optimization `-O2`
+* 3.6+ "with" and "with .. as" grammar improvements
+* ast-check for "for" loop was missing some grammar rules
+
 3.7.0: 2020-5-19 Primidi 1st Prairial - Alfalfa - HF
 ====================================================
 

README.rst

@@ -68,10 +68,9 @@ are syntactically correct by running the Python interpreter for that
 bytecode version.  Finally, in cases where the program has a test for
 itself, we can run the check on the decompiled code.
 
-We are serious about testing, and use automated processes to find
-bugs. In the issue trackers for other decompilers, you will find a
-number of bugs we've found along the way. Very few to none of them are
-fixed in the other decompilers.
+We use an automated processes to find bugs. In the issue trackers for
+other decompilers, you will find a number of bugs we've found along
+the way. Very few to none of them are fixed in the other decompilers.
 
 Requirements
 ------------
@@ -89,7 +88,8 @@ This uses setup.py, so it follows the standard Python routine:
 
 ::
 
-    $ pip install -e .  # set up to run from source tree, or...
+    $ pip install -e .  # set up to run from source tree
+                        # Or if you want to install instead
     $ python setup.py install # may need sudo
 
 A GNU makefile is also provided so :code:`make install` (possibly as root or
@@ -100,7 +100,7 @@ Running Tests
 
 ::
 
-   $  make check
+   make check
 
 A GNU makefile has been added to smooth over setting running the right
 command, and running tests from fastest to slowest.
@@ -171,15 +171,11 @@ All of the Python decompilers that I have looked at have problems
 decompiling Python's control flow. In some cases we can detect an
 erroneous decompilation and report that.
 
-Python support is strongest in Python 2 for 2.7 and drops off as you
-get further away from that. Support is also probably pretty good for
-python 2.3-2.4 since a lot of the goodness of early the version of the
-decompiler from that era has been preserved (and Python compilation in
-that era was minimal)
+Python support is pretty good for Python 2
 
-There is some work to do on the lower end Python versions which is
-more difficult for us to handle since we don't have a Python
-interpreter for versions 1.6, and 2.0.
+On the lower end of Python versions, decompilation seems pretty good although
+we don't have any automated testing in place for Python's distributed tests.
+Also, we don't have a Python interpreter for versions 1.6, and 2.0.
 
 In the Python 3 series, Python support is is strongest around 3.4 or
 3.3 and drops off as you move further away from those versions. Python
@@ -194,7 +190,7 @@ added. So in sum handling control flow by ad hoc means as is currently
 done is worse.
 
 Between Python 3.5, 3.6, 3.7 there have been major changes to the
-:code:`MAKE_FUNCTION` and :code:`CALL_FUNCTION` instructions.  Python
+:code:`MAKE_FUNCTION` and :code:`CALL_FUNCTION` instructions.
 
 Python 3.8 removes :code:`SETUP_LOOP`, :code:`SETUP_EXCEPT`,
 :code:`BREAK_LOOP`, and :code:`CONTINUE_LOOP`, instructions which may
@@ -210,26 +206,52 @@ however that the magic of a released version is usually the same as
 the *last* candidate version prior to release.
 
 There are also customized Python interpreters, notably Dropbox,
-which use their own magic and encrypt bytcode. With the exception of
+which use their own magic and encrypt bytecode. With the exception of
 the Dropbox's old Python 2.5 interpreter this kind of thing is not
 handled.
 
-We also don't handle PJOrion_ obfuscated code. For that try: PJOrion
-Deobfuscator_ to unscramble the bytecode to get valid bytecode before
-trying this tool. This program can't decompile Microsoft Windows EXE
-files created by Py2EXE_, although we can probably decompile the code
-after you extract the bytecode properly. For situations like this, you
-might want to consider a decompilation service like `Crazy Compilers
-<http://www.crazy-compilers.com/decompyle/>`_.  Handling
-pathologically long lists of expressions or statements is slow.
+We also don't handle PJOrion_ or otherwise obfuscated code. For
+PJOrion try: PJOrion Deobfuscator_ to unscramble the bytecode to get
+valid bytecode before trying this tool. This program can't decompile
+Microsoft Windows EXE files created by Py2EXE_, although we can
+probably decompile the code after you extract the bytecode
+properly. Handling pathologically long lists of expressions or
+statements is slow. We don't handle Cython_ or MicroPython which don't
+use bytecode.
 
+There are numerous bugs in decompilation. And that's true for every
+other CPython decompiler I have encountered, even the ones that
+claimed to be "perfect" on some particular version like 2.4.
 
-There is lots to do, so please dig in and help.
+As Python progresses decompilation also gets harder because the
+compilation is more sophisticated and the language itself is more
+sophisticated. I suspect that attempts there will be fewer ad-hoc
+attempts like unpyc37_ (which is based on a 3.3 decompiler) simply
+because it is harder to do so. The good news, at least from my
+standpoint, is that I think I understand what's needed to address the
+problems in a more robust way. But right now until such time as
+project is better funded, I do not intend to make any serious effort
+to support Python versions 3.8 or 3.9, including bugs that might come
+in. I imagine at some point I may be interested in it.
+
+You can easily find bugs by running the tests against the standard
+test suite that Python uses to check itself. At any given time, there are
+dozens of known problems that are pretty well isolated and that could
+be solved if one were to put in the time to do so. The problem is that
+there aren't that many people who have been working on bug fixing.
+
+Some of the bugs in 3.7 and 3.8 are simply a matter of back-porting
+the fixes in decmopyle3.
+
+You may run across a bug, that you want to report. Please do so. But
+be aware that it might not get my attention for a while. If you
+sponsor or support the project in some way, I'll prioritize your
+issues above the queue of other things I might be doing instead.
 
 See Also
 --------
 
-* https://github.com/rocky/python-decompile3 : Much smaller and more modern code, focusing on 3.7+. Changes in that will get migrated back ehre.
+* https://github.com/rocky/python-decompile3 : Much smaller and more modern code, focusing on 3.7+. Changes in that will get migrated back here.
 * https://code.google.com/archive/p/unpyc3/ : supports Python 3.2 only. The above projects use a different decompiling technique than what is used here. Currently unmaintained.
 * https://github.com/figment/unpyc3/ : fork of above, but supports Python 3.3 only. Includes some fixes like supporting function annotations. Currently unmaintained.
 * https://github.com/wibiti/uncompyle2 : supports Python 2.7 only, but does that fairly well. There are situations where :code:`uncompyle6` results are incorrect while :code:`uncompyle2` results are not, but more often uncompyle6 is correct when uncompyle2 is not. Because :code:`uncompyle6` adheres to accuracy over idiomatic Python, :code:`uncompyle2` can produce more natural-looking code when it is correct. Currently :code:`uncompyle2` is lightly maintained. See its issue `tracker <https://github.com/wibiti/uncompyle2/issues>`_ for more details
@@ -241,6 +263,7 @@ See Also
 * https://github.com/zrax/pycdc : The README for this C++ code says it aims to support all versions of Python. It is best for Python versions around 2.7 and 3.3 when the code was initially developed. Accuracy for current versions of Python3 and early versions of Python is lacking. Without major effort, it is unlikely it can be made to support current Python 3. See its `issue tracker <https://github.com/zrax/pycdc/issues>`_ for details. Currently lightly maintained.
 
 
+.. _Cython: https://en.wikipedia.org/wiki/Cython
 .. _trepan: https://pypi.python.org/pypi/trepan2g
 .. _compiler: https://pypi.python.org/pypi/spark_parser
 .. _HISTORY: https://github.com/rocky/python-uncompyle6/blob/master/HISTORY.md
@@ -251,11 +274,14 @@ See Also
 .. _uncompyle2: https://github.com/wibiti/uncompyle2
 .. _unpyc37: https://github.com/andrew-tavera/unpyc37
 .. _this: https://github.com/rocky/python-uncompyle6/wiki/Deparsing-technology-and-its-use-in-exact-location-reporting
-.. |buildstatus| image:: https://travis-ci.org/rocky/python-uncompyle6.svg :target: https://travis-ci.org/rocky/python-uncompyle6
-.. |packagestatus| image:: https://repology.org/badge/vertical-allrepos/python:uncompyle6.svg :target: https://repology.org/project/python:uncompyle6/versions
+.. |buildstatus| image:: https://travis-ci.org/rocky/python-uncompyle6.svg
+		 :target: https://travis-ci.org/rocky/python-uncompyle6
+.. |packagestatus| image:: https://repology.org/badge/vertical-allrepos/python:uncompyle6.svg
+		 :target: https://repology.org/project/python:uncompyle6/versions
 .. _PJOrion: http://www.koreanrandom.com/forum/topic/15280-pjorion-%D1%80%D0%B5%D0%B4%D0%B0%D0%BA%D1%82%D0%B8%D1%80%D0%BE%D0%B2%D0%B0%D0%BD%D0%B8%D0%B5-%D0%BA%D0%BE%D0%BC%D0%BF%D0%B8%D0%BB%D1%8F%D1%86%D0%B8%D1%8F-%D0%B4%D0%B5%D0%BA%D0%BE%D0%BC%D0%BF%D0%B8%D0%BB%D1%8F%D1%86%D0%B8%D1%8F-%D0%BE%D0%B1%D1%84
 .. _Deobfuscator: https://github.com/extremecoders-re/PjOrion-Deobfuscator
 .. _Py2EXE: https://en.wikipedia.org/wiki/Py2exe
 .. |Supported Python Versions| image:: https://img.shields.io/pypi/pyversions/uncompyle6.svg
-.. |Latest Version| image:: https://badge.fury.io/py/uncompyle6.svg :target: https://badge.fury.io/py/uncompyle6
+.. |Latest Version| image:: https://badge.fury.io/py/uncompyle6.svg
+		 :target: https://badge.fury.io/py/uncompyle6
 .. |Pypi Installs| image:: https://pepy.tech/badge/uncompyle6/month

__pkginfo__.py

@@ -33,65 +33,71 @@
 # 3.4            | pip     | 19.1.1       |
 
 # Things that change more often go here.
-copyright   = """
+copyright = """
 Copyright (C) 2015-2020 Rocky Bernstein <rb@dustyfeet.com>.
 """
 
-classifiers =  ["Development Status :: 5 - Production/Stable",
-                "Intended Audience :: Developers",
-                "License :: OSI Approved :: GNU General Public License v3 (GPLv3)",
-                "Operating System :: OS Independent",
-                "Programming Language :: Python",
-                "Programming Language :: Python :: 2.4",
-                "Programming Language :: Python :: 2.5",
-                "Programming Language :: Python :: 2.6",
-                "Programming Language :: Python :: 2.7",
-                "Programming Language :: Python :: 3.0",
-                "Programming Language :: Python :: 3.1",
-                "Programming Language :: Python :: 3.2",
-                "Programming Language :: Python :: 3.3",
-                "Programming Language :: Python :: 3.4",
-                "Programming Language :: Python :: 3.5",
-                "Programming Language :: Python :: 3.6",
-                "Programming Language :: Python :: 3.7",
-                "Programming Language :: Python :: 3.8",
-                "Topic :: Software Development :: Debuggers",
-                "Topic :: Software Development :: Libraries :: Python Modules",
-                ]
+classifiers = [
+    "Development Status :: 5 - Production/Stable",
+    "Intended Audience :: Developers",
+    "License :: OSI Approved :: GNU General Public License v3 (GPLv3)",
+    "Operating System :: OS Independent",
+    "Programming Language :: Python",
+    "Programming Language :: Python :: 2.4",
+    "Programming Language :: Python :: 2.5",
+    "Programming Language :: Python :: 2.6",
+    "Programming Language :: Python :: 2.7",
+    "Programming Language :: Python :: 3.0",
+    "Programming Language :: Python :: 3.1",
+    "Programming Language :: Python :: 3.2",
+    "Programming Language :: Python :: 3.3",
+    "Programming Language :: Python :: 3.4",
+    "Programming Language :: Python :: 3.5",
+    "Programming Language :: Python :: 3.6",
+    "Programming Language :: Python :: 3.7",
+    "Programming Language :: Python :: 3.8",
+    "Topic :: Software Development :: Debuggers",
+    "Topic :: Software Development :: Libraries :: Python Modules",
+]
 
 # The rest in alphabetic order
-author             = "Rocky Bernstein, Hartmut Goebel, John Aycock, and others"
-author_email       = "rb@dustyfeet.com"
-entry_points       = {
+author = "Rocky Bernstein, Hartmut Goebel, John Aycock, and others"
+author_email = "rb@dustyfeet.com"
+entry_points = {
     "console_scripts": [
         "uncompyle6=uncompyle6.bin.uncompile:main_bin",
         "pydisassemble=uncompyle6.bin.pydisassemble:main",
-    ]}
-ftp_url            = None
-install_requires   = ["spark-parser >= 1.8.9, < 1.9.0",
-                      "xdis >= 4.6.0, < 4.7.0"]
+    ]
+}
+ftp_url = None
+install_requires = ["spark-parser >= 1.8.9, < 1.9.0", "xdis >= 5.0.4, <5.1.0"]
 
-license            = "GPL3"
-mailing_list       = "python-debugger@googlegroups.com"
-modname            = "uncompyle6"
-py_modules         = None
-short_desc         = "Python cross-version byte-code decompiler"
-web                = "https://github.com/rocky/python-uncompyle6/"
+license = "GPL3"
+mailing_list = "python-debugger@googlegroups.com"
+modname = "uncompyle6"
+py_modules = None
+short_desc = "Python cross-version byte-code decompiler"
+web = "https://github.com/rocky/python-uncompyle6/"
 
 # tracebacks in zip files are funky and not debuggable
 zip_safe = True
 
 
 import os.path
+
+
 def get_srcdir():
     filename = os.path.normcase(os.path.dirname(os.path.abspath(__file__)))
     return os.path.realpath(filename)
 
+
 srcdir = get_srcdir()
 
+
 def read(*rnames):
     return open(os.path.join(srcdir, *rnames)).read()
 
+
 # Get info from files; set: long_description and VERSION
-long_description   = ( read("README.rst") + "\n" )
+long_description = read("README.rst") + "\n"
 exec(read("uncompyle6/version.py"))

admin-tools/check-older-versions.sh

@@ -20,9 +20,8 @@ for version in $PYVERSIONS; do
 	exit $?
     fi
     make clean && python setup.py develop
-    if ! make check-short ; then
+    if ! make check ; then
 	exit $?
     fi
     echo === $version ===
 done
-make check

admin-tools/how-to-make-a-release.md

@@ -2,17 +2,17 @@
 **Table of Contents**
 
 - [Get latest sources:](#get-latest-sources)
-- [Change version in uncompyle6/version.py](#change-version-in-uncompyle6versionpy)
+- [Change version in uncompyle6/version.py:](#change-version-in-uncompyle6versionpy)
 - [Update ChangeLog:](#update-changelog)
-- [Update NEWS from ChangeLog:](#update-news-from-changelog)
+- [Update NEWS.md from ChangeLog:](#update-newsmd-from-changelog)
 - [Make sure pyenv is running and check newer versions](#make-sure-pyenv-is-running-and-check-newer-versions)
 - [Switch to python-2.4, sync that up and build that first since it creates a tarball which we don't want.](#switch-to-python-24-sync-that-up-and-build-that-first-since-it-creates-a-tarball-which-we-dont-want)
-- [Update NEWS from master branch](#update-news-from-master-branch)
-- [Check against all versions](#check-against-all-versions)
+- [Check against older versions](#check-against-older-versions)
 - [Make packages and tag](#make-packages-and-tag)
-- [Upload single package and look at Rst Formating](#upload-single-package-and-look-at-rst-formating)
-- [Upload rest of versions](#upload-rest-of-versions)
-- [Push tags:](#push-tags)
+- [Check package on github](#check-package-on-github)
+- [Release on Github](#release-on-github)
+- [Get onto PyPI](#get-onto-pypi)
+- [Update tags:](#update-tags)
 
 <!-- markdown-toc end -->
 # Get latest sources:
@@ -39,7 +39,7 @@
 
 # Make sure pyenv is running and check newer versions
 
-    $ pyenv local && source admin-tools/check-newer-versions.sh
+    $ admin-tools/check-newer-versions.sh
 
 # Switch to python-2.4, sync that up and build that first since it creates a tarball which we don't want.
 
@@ -50,38 +50,51 @@
 
 # Check against older versions
 
-    $ source admin-tools/check-older-versions.sh
+    $ admin-tools/check-older-versions.sh
 
 # Make packages and tag
 
     $ . ./admin-tools/make-dist-older.sh
-	$ pyenv local 3.8.3
+	$ pyenv local 3.8.5
 	$ twine check dist/uncompyle6-$VERSION*
     $ git tag release-python-2.4-$VERSION
+    $ ./admin-tools/make-dist-newer.sh
 	$ twine check dist/uncompyle6-$VERSION*
-    $ . ./admin-tools/make-dist-newer.sh
 
-# Upload single package and look at Rst Formating
+# Check package on github
 
-	$ twine check dist/uncompyle6-${VERSION}*
-    $ twine upload dist/uncompyle6-${VERSION}-py3.3.egg
+	$ [[ ! -d /tmp/gittest ]] && mkdir /tmp/gittest; pushd /tmp/gittest
+	$ pyenv local 3.8.3
+	$ pip install -e git://github.com/rocky/python-uncompyle6.git#egg=uncompyle6
+	$ uncompyle6 --help
+	$ pip uninstall uncompyle6
+	$ popd
 
-# Upload rest of versions
-
-    $ twine upload dist/uncompyle6-${VERSION}*
+# Release on Github
 
 Goto https://github.com/rocky/python-uncompyle6/releases
 
-# Push tags:
+Now check the *tagged* release. (Checking the untagged release was previously done).
+
+Todo: turn this into a script in `admin-tools`
+
+	$ pushd /tmp/gittest
+	$ pip install -e git://github.com/rocky/python-uncompyle6.git@$VERSION#egg=uncompyle6
+	$ uncompyle6 --help
+	$ pip uninstall uncompyle6
+	$ popd
+
+
+# Get onto PyPI
+
+    $ twine upload dist/uncompyle6-${VERSION}*
+
+
+# Update tags:
 
     $ git push --tags
+    $ git pull --tags
 
-# Check on a VM
+# Move dist files to uploaded
 
-    $ cd /virtual/vagrant/virtual/vagrant/ubuntu-zesty
-	$ vagrant up
-	$ vagrant ssh
-	$ pyenv local 3.5.2
-	$ pip install --upgrade uncompyle6
-	$ exit
-	$ vagrant halt
+	$ mv -v dist/uncompyle6-${VERSION}* dist/uploaded

admin-tools/how-to-make-a-release.txt

@@ -1,46 +0,0 @@
-git pull
-
-Change version in uncompyle6/version.py
-source uncompyle6/version.py
-echo $VERSION
-git commit -m"Get ready for release $VERSION" .
-
-Update ChangeLog:
-  make ChangeLog
-
-Update NEWS from ChangeLog
-make check
-
-git commit --amend .
-
-git push
-
-Make sure pyenv is running
-# Pyenv
-
-source admin-tools/check-newer-versions.sh
-
-
-# Switch to python-2.4 and build that first...
-source admin-tools/setup-python-2.4
-
-rm ChangeLog
-git merge master
-
-Update NEWS from master branch
-
-git commit -m"Get ready for release $VERSION" .
-
-source admin-tools/check-older-versions.sh
-source admin-tools/check-newer-versions.sh
-
-make-dist-older.sh
-
-git tag release-python-2.4-$VERSION
-
-./make-dist-newer.sh
-
-git tag release-$VERSION
-
-
-twine upload dist/uncompyle6-${VERSION}*

admin-tools/pyenv-newer-versions

@@ -5,4 +5,4 @@ if [[ $0 == ${BASH_SOURCE[0]} ]] ; then
     echo "This script should be *sourced* rather than run directly through bash"
     exit 1
 fi
-export PYVERSIONS='3.5.9 3.6.10 2.6.9 3.3.7 2.7.18 3.2.6 3.1.5 3.4.10 3.7.7 3.8.3'
+export PYVERSIONS='3.5.9 3.6.12 2.6.9 3.3.7 2.7.18 3.2.6 3.1.5 3.4.10 3.7.9 3.8.5'

admin-tools/pyenv-older-versions

@@ -6,4 +6,4 @@ if [[ $0 == ${BASH_SOURCE[0]} ]] ; then
     echo "This script should be *sourced* rather than run directly through bash"
     exit 1
 fi
-export PYVERSIONS='2.4.6 2.5.6'
+export PYVERSIONS='2.4.6 2.5.6 2.6.9'

admin-tools/setup-python-2.4.sh

@@ -15,4 +15,3 @@ cd $fulldir/..
     git checkout python-2.4 &&  pyenv local $PYTHON_VERSION && git pull
 cd $owd
 rm -v */.python-version || true
-pyenv local $PYTHON_VERSION

admin-tools/update-sources.sh

@@ -1,3 +0,0 @@
-#!/bin/bash
-cd $(dirname ${BASH_SOURCE[0]})/..
-git pull

pytest/test_deparse.py

@@ -30,7 +30,7 @@ def list_comp():
     [y for y in range(3)]
 
 def get_parsed_for_fn(fn):
-    code = fn.func_code
+    code = fn.__code__ if PYTHON3 else fn.func_code
     return deparse(code, version=PYTHON_VERSION)
 
 def check_expect(expect, parsed, fn_name):

pytest/test_fjt.py

@@ -20,7 +20,7 @@ def bug_loop(disassemble, tb=None):
     disassemble(tb)
 
 def test_if_in_for():
-    code = bug.func_code
+    code = bug.__code__
     scan = get_scanner(PYTHON_VERSION)
     if 2.7 <= PYTHON_VERSION <= 3.0 and not IS_PYPY:
         scan.build_instructions(code)

pytest/test_grammar.py

@@ -45,9 +45,6 @@ def test_grammar():
         expect_lhs.add("kvlist")
         expect_lhs.add("kv3")
         unused_rhs.add("dict")
-    else:
-        # NOTE: this may disappear
-        expect_lhs.add("except_handler_else")
 
         if PYTHON_VERSION < 3.7 and PYTHON_VERSION != 2.7:
             # NOTE: this may disappear

pytest/test_pysource.py-notyet

@@ -8,8 +8,12 @@ from uncompyle6.semantics.consts import (
 
 if PYTHON3:
     from io import StringIO
+    def iteritems(d):
+        return d.items()
 else:
     from StringIO import StringIO
+    def iteritems(d):
+        return d.iteritems()
 
 from uncompyle6.semantics.pysource import (SourceWalker, deparse_code2str)
 
@@ -26,7 +30,7 @@ def test_template_engine():
     # FIXME: and so on...
 
 from uncompyle6.semantics.consts import (
-    TABLE_R, TABLE_DIRECT,
+    TABLE_DIRECT, TABLE_R,
     )
 
 from uncompyle6.semantics.fragments import (
@@ -40,7 +44,7 @@ def test_tables():
             (TABLE_DIRECT, 'TABLE_DIRECT', False),
             (TABLE_R, 'TABLE_R', False),
             (TABLE_DIRECT_FRAGMENT, 'TABLE_DIRECT_FRAGMENT', True)):
-        for k, entry in t.iteritems():
+        for k, entry in iteritems(t):
             if k in skip_for_now:
                 continue
             fmt = entry[0]

pytest/test_single_compile.py

@@ -1,4 +1,7 @@
+import pytest
 from uncompyle6 import PYTHON_VERSION, code_deparse
+pytestmark = pytest.mark.skip(PYTHON_VERSION < 2.7,
+                              reason="need at least Python 2.7")
 
 if PYTHON_VERSION > 2.6:
     def test_single_mode():

pytest/testdata/if-2.7.right

@@ -9,4 +9,4 @@
                12  JUMP_FORWARD          0  'to 15'
              15_0  COME_FROM            12  '12'
                15  LOAD_CONST               None
-               18  RETURN_VALUE
+               18  RETURN_VALUE     

pytest/testdata/ifelse-2.7.right

@@ -12,4 +12,4 @@
                18  STORE_NAME            2  'd'
              21_0  COME_FROM            12  '12'
                21  LOAD_CONST               None
-               24  RETURN_VALUE
+               24  RETURN_VALUE     

setup.py

@@ -1,17 +1,20 @@
 #!/usr/bin/env python
-"""Setup script for the 'uncompyle6' distribution."""
-
 import sys
 
+"""Setup script for the 'uncompyle6' distribution."""
+
 SYS_VERSION = sys.version_info[0:2]
-if not ((2, 4) <= SYS_VERSION  <= (2, 7)):
-    mess = "Python Release 2.4 .. 2.7 are supported in this code branch."
-    if ((3, 2) <= SYS_VERSION  <= (3, 8)):
-        mess += ("\nFor your Python, version %s, use the master code/branch." %
-                 sys.version[0:3])
-    else:
-        mess += ("\nThis package is not supported before Python 2.4. Your Python version is %s."
-                 % sys.version[0:3])
+if not ((2, 6) <= SYS_VERSION <= (3, 9)):
+    mess = "Python Release 2.6 .. 3.9 are supported in this code branch."
+    if (2, 4) <= SYS_VERSION <= (2, 7):
+        mess += (
+            "\nFor your Python, version %s, use the python-2.4 code/branch."
+            % sys.version[0:3]
+        )
+    elif SYS_VERSION < (2, 4):
+        mess += (
+            "\nThis package is not supported for Python version %s." % sys.version[0:3]
+        )
     print(mess)
     raise Exception(mess)
 

test-unit/test_grammar.py

@@ -1,55 +0,0 @@
-import re
-import unittest
-from uncompyle6 import PYTHON_VERSION, IS_PYPY # , PYTHON_VERSION
-from uncompyle6.parser import get_python_parser, python_parser
-
-class TestGrammar(unittest.TestCase):
-    def test_grammar(self):
-
-        def check_tokens(tokens, opcode_set):
-            remain_tokens = set(tokens) - opcode_set
-            remain_tokens = set([re.sub('_\d+$','', t) for t in remain_tokens])
-            remain_tokens = set([re.sub('_CONT$','', t) for t in remain_tokens])
-            remain_tokens = set(remain_tokens) - opcode_set
-            self.assertEqual(remain_tokens, set([]),
-                    "Remaining tokens %s\n====\n%s" % (remain_tokens, p.dump_grammar()))
-
-        p = get_python_parser(PYTHON_VERSION, is_pypy=IS_PYPY)
-        (lhs, rhs, tokens,
-         right_recursive, dup_rhs) = p.check_sets()
-        expect_lhs = set(['pos_arg', 'get_iter', 'attribute'])
-        unused_rhs = set(['list', 'call', 'mkfunc',
-                          'mklambda',
-                          'unpack',])
-
-        expect_right_recursive = frozenset([('designList',
-                                             ('store', 'DUP_TOP', 'designList'))])
-        expect_lhs.add('kwarg')
-
-        self.assertEqual(expect_lhs, set(lhs))
-        self.assertEqual(unused_rhs, set(rhs))
-        self.assertEqual(expect_right_recursive, right_recursive)
-
-        expect_dup_rhs = frozenset([('COME_FROM',), ('CONTINUE',), ('JUMP_ABSOLUTE',),
-                                    ('LOAD_CONST',),
-                                    ('JUMP_BACK',), ('JUMP_FORWARD',)])
-
-        reduced_dup_rhs = {}
-        for k in dup_rhs:
-            if k not in expect_dup_rhs:
-                reduced_dup_rhs[k] = dup_rhs[k]
-                pass
-            pass
-        for k in reduced_dup_rhs:
-            print(k, reduced_dup_rhs[k])
-        # assert not reduced_dup_rhs, reduced_dup_rhs
-
-    def test_dup_rule(self):
-        import inspect
-        python_parser(PYTHON_VERSION, inspect.currentframe().f_code,
-                      is_pypy=IS_PYPY,
-                      parser_debug={
-                          'dups': True, 'transition': False, 'reduce': False,
-                          'rules': False, 'errorstack': None, 'context': True})
-if __name__ == '__main__':
-    unittest.main()

test/Makefile

@@ -30,7 +30,7 @@ check:
 	$(MAKE) check-$(PYTHON_VERSION)
 
 #: Run working tests from Python 2.6 or 2.7
-check-2.4 check-2.5 check-2.6 check-2.7: check-bytecode-2 check-bytecode-3 check-bytecode-1  check-native-short
+check-2.6 check-2.7: check-bytecode-2 check-bytecode-3 check-bytecode-1 check-native-short
 
 #: Run working tests from Python 3.0
 check-3.0: check-bytecode
@@ -72,10 +72,10 @@ check-3.7: check-bytecode
 	$(PYTHON) test_pythonlib.py --bytecode-3.7-run --verify-run
 	$(PYTHON) test_pythonlib.py --bytecode-3.7 --syntax-verify $(COMPILE)
 
-# #: Run working tests from Python 3.8
-# check-3.8: check-bytecode
-# 	$(PYTHON) test_pythonlib.py --bytecode-3.8-run --verify-run
-# 	$(PYTHON) test_pythonlib.py --bytecode-3.8 --syntax-verify $(COMPILE)
+#: Run working tests from Python 3.8
+check-3.8: check-bytecode
+	$(PYTHON) test_pythonlib.py --bytecode-3.8-run --verify-run
+	$(PYTHON) test_pythonlib.py --bytecode-3.8 --syntax-verify $(COMPILE)
 
 # FIXME
 #: this is called when running under pypy3.5-5.8.0, pypy2-5.6.0, or pypy3.6-7.3.0
@@ -173,7 +173,6 @@ check-bytecode-2.3:
 
 #: Check deparsing Python 2.4
 check-bytecode-2.4:
-	$(PYTHON) test_pythonlib.py --bytecode-2.4-run --verify-run
 	$(PYTHON) test_pythonlib.py --bytecode-2.4
 
 #: Check deparsing Python 2.5
@@ -302,20 +301,16 @@ check-bytecode-3.7:
 	$(PYTHON) test_pythonlib.py --bytecode-3.7-run --verify-run
 	$(PYTHON) test_pythonlib.py --bytecode-3.7 --syntax-verify
 
-# #: Check deparsing Python 3.8
-# check-bytecode-3.8:
-# 	$(PYTHON) test_pythonlib.py --bytecode-3.8-run --verify-run
-# 	$(PYTHON) test_pythonlib.py --bytecode-3.8 --syntax-verify
+#: Check deparsing Python 3.8
+check-bytecode-3.8:
+	$(PYTHON) test_pythonlib.py --bytecode-3.8-run --verify-run
+	$(PYTHON) test_pythonlib.py --bytecode-3.8 --syntax-verify
 
 #: short tests for bytecodes only for this version of Python
 check-native-short:
 	$(PYTHON) test_pythonlib.py --bytecode-$(PYTHON_VERSION) --syntax-verify $(COMPILE)
 	$(PYTHON) test_pythonlib.py --bytecode-$(PYTHON_VERSION)-run --verify-run $(COMPILE)
 
-#: Run longer Python 2.6's lib files known to be okay
-check-2.4-ok:
-	$(PYTHON) test_pythonlib.py --ok-2.4 --verify $(COMPILE)
-
 #: Run longer Python 2.6's lib files known to be okay
 check-2.6-ok:
 	$(PYTHON) test_pythonlib.py --ok-2.6 --syntax-verify $(COMPILE)

test/add-test.py

@@ -3,14 +3,22 @@
 """
 import os, sys, py_compile
 
-assert len(sys.argv) >= 2
+assert (2 <= len(sys.argv) <= 4)
 version = sys.version[0:3]
+vers = sys.version_info[:2]
 if sys.argv[1] in ("--run", "-r"):
     suffix = "_run"
     py_source = sys.argv[2:]
+    i = 2
 else:
     suffix = ""
     py_source = sys.argv[1:]
+    i = 1
+try:
+    optimize = int(sys.argv[-1])
+    py_source = sys.argv[i:-1]
+except:
+    optimize = 2
 
 for path in py_source:
     short = os.path.basename(path)
@@ -19,6 +27,10 @@ for path in py_source:
     else:
         cfile = "bytecode_%s%s/%s" % (version, suffix, short) + "c"
     print("byte-compiling %s to %s" % (path, cfile))
-    py_compile.compile(path, cfile)
-    if isinstance(version, str) or version >= (2, 6, 0):
+    optimize = optimize
+    if vers > (3, 1):
+        py_compile.compile(path, cfile, optimize=optimize)
+    else:
+        py_compile.compile(path, cfile)
+    if vers >= (2, 6):
         os.system("../bin/uncompyle6 -a -T %s" % cfile)

test/dis-compare.py

@@ -1,8 +1,9 @@
 #!/usr/bin/env python
 # Mode: -*- python -*-
 #
-# Copyright (c) 2015, 2017 by Rocky Bernstein <rb@dustyfeet.com>
+# Copyright (c) 2015 by Rocky Bernstein <rb@dustyfeet.com>
 #
+from __future__ import print_function
 
 
 import dis, os.path

test/run-and-email.sh

@@ -34,9 +34,10 @@ else
 fi
 
 MAIN="test_pyenvlib.py"
+
 USER=${USER:-rocky}
 EMAIL=${EMAIL:-rb@dustyfeet.com}
-WHAT="uncompyle6 2.4 ${MAIN}"
+WHAT="uncompyle6 ${MAIN}"
 MAX_TESTS=${MAX_TESTS:-800}
 export BATCH=1
 
@@ -44,13 +45,13 @@ typeset -i RUN_STARTTIME=$(date +%s)
 
 # PYVERSIONS="3.5.6"
 MAILBODY=/tmp/${MAIN}-mailbody-$$.txt
-# for VERSION in 2.4.6 2.5.9 ; do
+# for VERSION in 3.3.7 ; do
 for VERSION in $PYVERSIONS ; do
     typeset -i rc=0
     LOGFILE=/tmp/${MAIN}-$VERSION-$$.log
 
     case "$VERSION" in
-	3.7.7 | 3.8.2 | 3.1.5 | 3.0.1 )
+	3.7.8 | 3.8.5 | 3.1.5 | 3.0.1 )
 	    continue
 	    ;;
 	3.5.9 )
@@ -65,7 +66,7 @@ for VERSION in $PYVERSIONS ; do
 	3.4.10 )
 	    MAX_TESTS=800
 	    ;;
-	3.6.10 )
+	3.6.11 )
 	    # MAX_TESTS=1300  # about 2139 exist
 	    # fails on _pyio.cpython-36.opt-1.pyc
 	    MAX_TESTS=34

test/simple-decompile-code-test.py

@@ -1,6 +1,7 @@
 #!/usr/bin/env python
 
-from uncompyle6 import uncompyle
+from __future__ import print_function
+
 from uncompyle6.main import decompile
 from xdis import sysinfo2float
 import sys, inspect

test/simple_source/bug14/01_for_else_try_else.py

@@ -0,0 +1,17 @@
+# Adapted from 1.4 anydbm
+"""This program is self-checking!"""
+def scan(items):
+    for i in items:
+            try:
+                    5 / i
+            except:
+                    continue
+            else:
+                    break
+    else:
+            return 2
+    return i
+
+assert scan((0, 1)) == 1
+assert scan((0, 0)) == 2
+assert scan((3, 2, 1)) == 3

test/simple_source/bug25/02_yield_bug.py

@@ -1,7 +0,0 @@
-# From 2.4 test_array.py
-# In Python 2.4 and earlier "yield" is not valid and instead
-# we must use "yield None". Bug was not adding "None"
-
-def yield_bug():
-    yield None
-    return

test/simple_source/bug25/04_try_else_confuse.py

@@ -1,19 +0,0 @@
-# From 2.4 test_sax.py
-# Bug was distinguishing try from try/else
-
-def verify_empty_attrs():
-    gvqk = 3
-    try:
-        gvk = 1/0
-    except ZeroDivisionError:
-        gvk = 1
-
-    try:
-        gvqk = 0
-    except KeyError:
-        gvqk = 1
-
-    # If try/else was used above the return will be 4
-    return gvk + gvqk
-
-assert 1 == verify_empty_attrs()