Another RsT typo

More work is needed. Not done in Python 3 yet

.gitignore

@@ -1,3 +1,5 @@
+*.pyo
+*.pyc
 *_dis
 *~
 /.cache
@@ -8,7 +10,12 @@
 /__pkginfo__.pyc
 /dist
 /how-to-make-a-release.txt
+/nose-*.egg
 /tmp
 /uncompyle6.egg-info
+/unpyc
 __pycache__
 build
+/.venv*
+/.idea
+/.hypothesis

.travis.yml

@@ -3,14 +3,22 @@ language: python
 sudo: false
 
 python:
-  - '2.6'
-  - '2.7'
-  - '3.4'
   - '3.5'
+  - '2.7.12'
+  - '2.6'
+  - '3.3'
+  - '3.4'
+  - '3.2'
+  - '3.6'
 
 install:
-- pip install -r requirements.txt
+- pip install -e .
 - pip install -r requirements-dev.txt
 
 script:
 - python ./setup.py develop && COMPILE='--compile' make check
+
+# blacklist
+branches:
+  except:
+    - data-driven-pytest

DECOMPYLE-2.4-CHANGELOG.txt

@@ -0,0 +1,131 @@
+This is the changelog from *decompyle*'s release 2.4 and before
+passed on by Dan Pascu
+
+
+release 2.4 (Dan Pascu)
+  - Replaced the way code structures are identified by the parser.
+    Previously, the scanner introduced some COME_FROM entries in the
+    dissasembly output to mark all the destinations of jump instructions.
+    Using these COME_FROM labels the parser was then able to identify the
+    code structures (if tests, while loops, etc). Up to python-2.3 this was
+    possible because the code structures were clearly defined and jump
+    targets were always to the same points in a given strcuture making it
+    easy to identify the structure. Python 2.3 however introduced optimized
+    jumps to increase code performance. In the previous version of decompyle
+    (2.3) we used a technique to identify the code structures and then used
+    these structures to determine where the jump targets would have been if
+    not optimized. Using this information we then added COME_FROM labels at
+    the points where they would have been if not optimized, thus emulating
+    the way decompyle worked with versions before python 2.3. However with
+    the introduction of even more optimizations in python 2.4 this technique
+    no longer works. Not only the jump targets are no longer an effective
+    mean for the parser to identify the code structures, but also trying to
+    emulate the old way things were solved when it clearly no longer works
+    is not the right solution. To solve this issue, the code to identify the
+    structures that we had developed in version 2.3, was used to add real
+    start/end points for strcuture identification, instead of the COME_FROM
+    labels. Now these new start/end labels are used by the parser to more
+    precisely identify the structures and the COME_FROM labels were removed
+    completely. The scanner is responsible to identify these code structures
+    and use any knowledge of optimizations that python applies to determine
+    the start/end points of any structure and then mark them with certain
+    keywords that are understood by the parser.
+  - Correctly identify certain `while 1' structures that were not
+    recognized in the previous version.
+  - Added support for new byte code constructs used by python 2.4
+
+release 2.3.2
+  - tidied up copyright and changelog information for releases 2.3 and later
+
+release 2.3.1 (Dan Pascu)
+  - implemented a structure detection technique that fixes problems with
+    optimised jumps in Python >= 2.3. In the previous release (decompyle 2.3),
+    these problems meant that some files were incorrectly decompiled and
+    others could not be decompiled at all. With this new structure detection
+    technique, thorough testing over the standard python libraries suggests
+    that decompyle 2.3.1 can handle everything that decompyle 2.2beta1 could,
+    plus new Python 2.3 bytecodes and constructs.
+
+release 2.3 (Dan Pascu)
+  - support for Python 2.3 added
+  - use the marshal and disassembly code from their respective python
+    versions, so that decompyle can manipulate bytecode independently
+    of the interpreter that runs decompyle itself (for example it can
+    decompile python2.3 bytecode even when running under python2.2)
+
+——————————————————
+
+release 2.2beta1 (hartmut Goebel)
+  - support for Python 1.5 up to Python 2.2
+  - no longer requires to be run with the Python interpreter version
+    which generated the byte-code.
+  - requires Python 2.2
+  - pretty-prints docstrings, hashes, lists and tuples
+  - decompyle is now a script and a package
+  - added emacs mode-hint and tab-width for each file output
+  - enhanced test suite: more test patterns, .pyc/.pyo included
+  - avoids unnecessary 'global' statements
+  - still untested: EXTENDED_ARG
+
+  internal changes:
+  - major code overhoul: splitted into several modules, clean-ups
+  - use a list of valid magics instead of the single one from imp.py
+  - uses copies of 'dis.py' for every supported version. This ensures
+    correct disassemling of the byte-code.
+  - use a single Walker and a single Parser, thus saving time and memory
+  - use augmented assign and 'print >>' internally
+  - optimized 'Walker.engine', the main part of code generation
+release 0.6.0: (hartmut Goebel)
+  - extended print (Python 2.0)
+  - extended import (Python 2.0) (may not cover all cases)
+  - augmented assign (Python 2.0) (may not cover all cases)
+  - list comprehensions (Python 2.0)
+  - equivalent for 'apply' (Python 1.6)
+  - if .. elif .. else are now nested as expected
+  - assert test, data
+  - unpack list corrected (was the same as unpack tuple)
+  - fixed unpack tuple (trailing semicolon was missing)
+  - major speed up :-)
+  - reduced memory usage (pre-alpha-0.5 has increased it a lot)
+  - still missing: EXTENDED_ARG
+
+pre-alpha-0.5: (hartmut Goebel)
+  - *args, **kwargs
+  - global
+  - formal tuple parameters (eg. def a(self, (x,y,z)) )
+  - actual lambda parameters (eg.  X(lambda z: z**2) )
+  - remove last 'return None' in procedures
+  - remove last 'return locals()' in class definitions
+  - docstrings
+
+pre-alpha-0.4: (hartmut Goebel)
+  - assert
+  - try/except/finally
+  - parentheses in expressions
+  - nested expressions
+  - extracted dissassemble() from module dis and
+    removed ugly redirect of stdout, thus saved a lot of
+    ugly code and a lot of memory
+
+pre-alpha-0.3: (hartmut Goebel)
+  - keyword arguments
+  - some boolean expressions
+  - and/or
+  - complex conditions in if/while
+  - read byte-code from .pyc without importing
+  - access to the body of classes and modules
+  - class and function definitions
+  - a = b = c = xxx
+
+pre-alpha-0.1 -> pre-alpha-0.2:
+  -  SET_LINENO filtered out in lexer now
+  -  added support for subscripts (just for Christian Tismer :-)
+  -  fixed bug with handling of BUILD_{LIST,TUPLE} & CALL_FUNCTION
+  -  dict-building support
+  -  comparison support
+  -  exec support
+  -  del support
+  -  pass support
+  -  slice support
+  -  no more extraneous (albeit legal) commas
+  -  finally, it excepts try [sic] but not all 42 variations of it

HISTORY.md

@@ -4,7 +4,8 @@ There have been a number of people who have worked on this. I am awed
 by the amount of work, number of people who have contributed to this,
 and the cleverness in the code.
 
-The below is an annotated history from my reading of the sources cited.
+The below is an annotated history from talking to participants
+involved and my reading of the code and sources cited.
 
 In 1998, John Aycock first wrote a grammar parser in Python,
 eventually called SPARK, that was usable inside a Python program. This
@@ -23,10 +24,14 @@ working on his thesis, John realized SPARK could be used to deparse
 Python bytecode. In the fall of 1999, he started writing the Python
 program, "decompyle", to do this.
 
-This code introduced another clever idea: using table-driven
-semantics routines, using format specifiers.
+To help with control structure deparsing the instruction sequence was
+augmented with pseudo instruction COME_FROM. This code introduced
+another clever idea: using table-driven semantics routines, using
+format specifiers.
 
-The last mention of a release of SPARK from John is around 2002.
+The last mention of a release of SPARK from John is around 2002. As
+released, although the Earley Algorithm parser was in good shape, this
+code was woefully lacking as serious Python deparser.
 
 In the fall of 2000, Hartmut Goebel
 [took over maintaining the code](https://groups.google.com/forum/#!searchin/comp.lang.python/hartmut$20goebel/comp.lang.python/35s3mp4-nuY/UZALti6ujnQJ). The
@@ -36,10 +41,11 @@ first subsequent public release announcement that I can find is
 From the CHANGES file found in
 [the tarball for that release](http://old-releases.ubuntu.com/ubuntu/pool/universe/d/decompyle2.2/decompyle2.2_2.2beta1.orig.tar.gz),
 it appears that Hartmut did most of the work to get this code to
-accept the full Python language. He added precidence to the table
+accept the full Python language. He added precedence to the table
 specifiers, support for multiple versions of Python, the
 pretty-printing of docstrings, lists, and hashes. He also wrote test and verification routines of
-deparsed bytecode, and used this in an extensive set of tests that he also wrote. He could verify against the entire Python library.
+deparsed bytecode, and used this in an extensive set of tests that he also wrote. He says he could verify against the
+entire Python library. However I have subsequently found small and relatively obscure bugs in the decompilation code.
 
 decompyle2.2 was packaged for Debian (sarge) by
 [Ben Burton around 2002](https://packages.qa.debian.org/d/decompyle.html). As
@@ -55,32 +61,88 @@ it doesn't look like he's done anything compiler-wise since SPARK). So
 I hope people will use the crazy-compilers service. I wish them the
 success that his good work deserves.
 
-Next we get to
-["uncompyle" and PyPI](https://pypi.python.org/pypi/uncompyle/1.1) and
-the era of git repositories. In contrast to decompyle, this now runs
-only on Python 2.7 although it accepts bytecode back to Python
+Dan Pascu did a bit of work from late 2004 to early 2006 to get this
+code to handle first Python 2.3 and then 2.4 bytecodes. Because of
+jump optimization introduced in the CPython bytecode compiler at that
+time, various JUMP instructions were classifed as going backwards, and
+COME FROM instructions were reintroduced.  See
+[RELEASE-2.4-CHANGELOG.txt](https://github.com/rocky/python-uncompyle6/blob/master/DECOMPYLE-2.4-CHANGELOG.txt)
+for more details here. There wasn't a public
+release of RELEASE-2.4 and bytecodes other than Python 2.4 weren't
+supported. Dan says the Python 2.3 version could verify the entire
+Python library. But given subsequent bugs found like simply
+recognizing complex-number constants in bytecode, decompilation wasn't perfect.
+
+Next we get to ["uncompyle" and
+PyPI](https://pypi.python.org/pypi/uncompyle/1.1) and the era of
+public version control. (Dan's code although not public used
+[darcs](http://darcs.net/) for version control.)
+
+In contrast to _decompyle_, _uncompyle_ at least in its final versions,
+runs only on Python 2.7. However it accepts bytecode back to Python
 2.5. Thomas Grainger is the package owner of this, although Hartmut is
-listed as the author.
+still listed as the author.
 
 The project exists not only on
 [github](https://github.com/gstarnberger/uncompyle) but also on
-[bitbucket](https://bitbucket.org/gstarnberger/uncompyle) where the
-git history goes back to 2009. Somewhere in there the name was changed
-from "decompyle" to "uncompyle".
+[bitbucket](https://bitbucket.org/gstarnberger/uncompyle) and later
+the defunct [google
+code](https://code.google.com/archive/p/unpyc/). The git/svn history
+goes back to 2009. Somewhere in there the name was changed from
+"decompyle" to "unpyc" by Keknehv, and then to "uncompyle" by Guenther Starnberger.
 
 The name Thomas Grainger isn't found in (m)any of the commits in the
-several years of active development. Guenther Starnberger, Keknehv,
-hamled, and Eike Siewertsen are principle committers here.
+several years of active development. First Keknehv worked on this up
+to Python 2.5 or so while acceping Python bytecode back to 2.0 or
+so. Then hamled made a few commits earler on, while Eike Siewertsen
+made a few commits later on. But mostly wibiti, and Guenther
+Starnberger got the code to where uncompyle2 was around 2012.
+
+While John Aycock and Hartmut Goebel were well versed in compiler
+technology, those that have come afterwards don't seem to have been as
+facile in it.  Furthermore, documentation or guidance on how the
+decompiler code worked, comparison to a conventional compiler
+pipeline, how to add new constructs, or debug grammars was weak. Some
+of the grammar tracing and error reporting was a bit weak as well.
+
+Given this, perhaps it is not surprising that subsequent changes
+tended to shy away from using the built-in compiler technology
+mechanisms and addressed problems and extensions by some other means.
+
+Specifically, in `uncompyle`, decompilation of python bytecode 2.5 & 2.6
+is done by transforming the byte code into a pseudo-2.7 Python
+bytecode and is based on code from Eloi Vanderbeken.
+
+This project, `uncompyle6`, abandons that approach for various
+reasons. However the main reason is that we need offsets in fragment
+deparsing to be exactly the same, and the transformation process can
+remove instructions.  _Adding_ instructions with psuedo offsets is
+however okay.
+
+`Uncompyle6` however owes its existence to the fork of `uncompyle2` by
+Myst herie (Mysterie) whose first commit picks up at
+2012. I chose this since it seemed to have been at that time the most
+actively, if briefly, worked on. Also starting around 2012 is Dark
+Fenx's uncompyle3 which I used for inspiration for Python3 support.
+
+I started working on this late 2015, mostly to add fragment support.
+In that, I decided to make this runnable on Python 3.2+ and Python 2.6+
+while, handling Python bytecodes from Python versions 2.5+ and
+3.2+. In doing so, it has been expedient to separate this into three
+projects:
+
+* marshaling/unmarshaling, bytecode loading and disassembly ([xdis](https://pypi.python.org/pypi/xdis)),
+* parsing and tree building ([spark_parser](https://pypi.python.org/pypi/spark_parser)),
+* this project - grammar and semantic actions for decompiling
+  ([uncompyle6](https://pypi.python.org/pypi/uncompyle6)).
 
-This project, uncompyle6, however owes its existence to uncompyle2 by
-Myst herie (Mysterie) whose first commit seems to goes back to 2012;
-it is also based on Hartmut's code. I chose this as it seems had been
-the most actively worked on most recently.
 
 Over the many years, code styles and Python features have
 changed. However brilliant the code was and still is, it hasn't really
 had a single public active maintainer. And there have been many forks
-of the code.
+of the code.  I have spent a great deal of time trying to organize and
+modularize the code so that it can handle more Python versions more
+gracefully (with still only moderate success).
 
 That it has been in need of an overhaul has been recognized by the
 Hartmut a decade an a half ago:
@@ -90,15 +152,29 @@ Hartmut a decade an a half ago:
     NB. This is not a masterpiece of software, but became more like a hack.
     Probably a complete rewrite would be sensefull. hG/2000-12-27
 
-Lastly, I should mention [unpyc](https://code.google.com/p/unpyc3/)
-and most especially [pycdc](https://github.com/zrax/pycdc), largely by
-Michael Hansen and Darryl Pogue. If they supported getting source-code
-fragments and I could call it from Python, I'd probably ditch this and
-use that. From what I've seen, the code runs blindingly fast and spans
-all versions of Python.
+This project deparses using an Earley-algorithm parse with lots of
+massaging of tokens and the grammar in the scanner
+phase. Earley-algorithm parsers are context free and tend to be linear
+if the grammar is LR or left recursive.
+
+Another approach that doesn't use grammars is to do something like
+simulate execution symbolically and build expression trees off of
+stack results. Control flow in that apprproach still needs to be
+handled somewhat ad hoc.  The two important projects that work this
+way are [unpyc3](https://code.google.com/p/unpyc3/) and most
+especially [pycdc](https://github.com/zrax/pycdc) The latter project
+is largely by Michael Hansen and Darryl Pogue. If they supported
+getting source-code fragments, did a better job in supporting Python
+more fully, and had a way I could call it from Python, I'd probably
+would have ditched this and used that. The code runs blindingly fast
+and spans all versions of Python, although more recently Python 3
+support has been lagging.
 
 Tests for the project have been, or are being, culled from all of the
 projects mentioned.
 
-NB. If you find mistakes, want corrections, or want your name added (or removed),
-please contact me.
+For a little bit of the history of changes to the Early-algorithm parser,
+see the file [NEW-FEATURES.rst](https://github.com/rocky/python-spark/blob/master/NEW-FEATURES.rst) in the [python-spark github repository](https://github.com/rocky/python-spark).
+
+NB. If you find mistakes, want corrections, or want your name added
+(or removed), please contact me.

HOW-TO-REPORT-A-BUG.md

@@ -0,0 +1,187 @@
+# How to report a Bug
+
+## The difficulty of the problem
+
+This decompiler is a constant work in progress: Python keeps
+changing, and so does its code generation.
+
+There is no Python decompiler yet that I know about that will
+decompile everything. Overall, I think this one probably does the best
+job of *any* Python decompiler that handles such a wide range of
+versions.
+
+But at any given time, there are maybe dozens of valid Python bytecode
+files that I know of that will cause problems. And when I get through
+those and all the issues of decompiler bugs that are currently logged,
+I could probably easily find dozens more bugs just by doing a
+decompile of all the Python bytecode on any one of my
+computers. Unless you want to help out by _fixing_ bugs, or are
+willing to do work by isolating and narrowing bugs, don't feel you are
+doing me a favor by doing scans on your favorite sets of bytecode
+files.
+
+In sum, it is not uncommon that you will find a mistranslation in
+decompiling. Furthermore, you may be expected to do some work in order
+to have your bug worthy of being considered above other bugs.
+
+No one is getting paid to work to work on this project, let alone bugs
+you may have an interest in. If you require decompiling bytecode
+immediately, consider using a decompilation service.
+
+## Is it really a bug?
+
+
+### Do you have valid bytecode?
+
+As mentioned in README.rst, this project doesn't handle obfuscated
+code. See README.rst for suggestions for how to remove some kinds of
+obfuscation.
+
+Checking if bytecode is valid is pretty simple: disassemble the code.
+Python comes with a disassembly module called `dis`. A prerequisite
+module for this package, `xdis` has a cross-python version
+disassembler.
+
+### Semantic equivalence vs. exact source code
+
+Almost all versions of Python can perform some sort of code
+improvement that can't be undone. In earlier versions of Python it is
+rare; in later Python versions, it is more common.
+
+If the code emitted is semantically equivalent, then this isn't a bug.
+
+
+For example the code might be
+
+```
+if a:
+  if b:
+     x = 1
+```
+
+and we might produce:
+
+```
+if a and b:
+  x = 1
+```
+
+These are equivalent. Sometimes
+
+```
+else:
+   if ...
+
+```
+
+may come out as `elif`.
+
+
+As mentioned in the README, It is possible that Python changes what
+you write to be more efficient. For example, for:
+
+
+```
+if True:
+  x = 5
+```
+
+Python will generate code like:
+
+```
+x = 5
+```
+
+So just because the text isn't the same, does not
+necessarily mean there's a bug.
+
+## What to send (minimum requirements)
+
+The basic requirement is pretty simple:
+
+* Python bytecode
+* Python source text
+
+Please don't put files on download services that one has to register
+for or can't get to by issuing a simple `curl` or `wget`. If you can't
+attach it to the issue, or create a github gist, then the code you are
+sending is too large.
+
+Also try to narrow the bug. See below.
+
+## What to send (additional helpful information)
+
+Some kind folks also give the invocation they used and the output
+which usually includes an error message produced. This is
+helpful. From this, I can figure out what OS you are running this on
+and what version of *uncomplye6* was used. Therefore, if you don't
+provide the input command and the output from that, please give:
+
+* _uncompyle6_ version used
+* OS that you used this on
+* Python interpreter version used
+
+
+### But I don't *have* the source code!
+
+Sure, I get it. No problem. There is Python assembly code on parse
+errors, so simply by hand decompile that. To get a full disassembly,
+use pydisasm from the [xdis](https://pypi.python.org/pypi/xdis)
+package. Opcodes are described in the documentation for
+the [dis](https://docs.python.org/3.6/library/dis.html) module.
+
+### But I don't *have* the source code and am incapable of figuring how how to do a hand disassembly!
+
+Well, you could learn. No one is born into this world knowing how to
+disassemble Python bytecode. And as Richard Feynman once said, "What
+one fool can learn, so can another."
+
+If this is too difficult, or too time consuming, or not of interest to
+you, then perhaps what require is a decompilation service. [Crazy
+Compilers](http://www.crazy-compilers.com/decompyle/) offers a
+byte-code decompiler service for versions of Python up to 2.6. (If
+there are others around let me know and I'll list them here.)
+
+## Narrowing the problem
+
+I don't need or want the entire source code base for the file(s) or
+module(s) can't be decompiled. I just need those file(s) or module(s).
+If there are problems in several files, file a bug report for each
+file.
+
+Python modules can get quite large, and usually decompilation problems
+occur in a single function or maybe the main-line code but not any of
+the functions or classes. So please chop down the source code by
+removing those parts that do to decompile properly.
+
+By doing this, you'll probably have a better sense of what exactly is
+the problem. Perhaps you can find the boundary of what decompiles, and
+what doesn't. That is useful. Or maybe the same file will decompile
+properly on a neighboring version of Python. That is helpful too.
+
+In sum, the more you can isolate or narrow the problem, the more
+likley the problem will be fixed and fixed sooner.
+
+## Confidentiality of Bug Reports
+
+When you report a bug, you are giving up confidentiality to the source
+code and the byte code. However, I would imagine that if you have
+narrowed the problem sufficiently, confidentiality of the little that
+remains would not be an issue.
+
+However feel free to remove any commments, and modify variable names
+or constants in the source code.
+
+## Ethics
+
+I do not condone using this program for unethical or illegal purposes.
+More detestful, at least to me, is asking for help to assist you in
+something that might not legitimate.
+
+Don't use the issue tracker for such solicitations. To try to stave
+off illegitimate behavior, you should note that the issue tracker, the
+code, and bugs mentioned in that are in the open: there is no
+confidentiality. You may be asked about the authorship or claimed
+ownership of the bytecode. If I think something is not quite right, I
+may label the issue questionable which may make the it easier those
+who are looking for illegal activity.

LICENSE

@@ -1,6 +1,6 @@
-Copyright (c) 1998-2002 John Aycock
-Copyright (c) 2000 by hartmut Goebel <h.goebel@crazy-compilers.com>
 Copyright (c) 2015 by Rocky Bernstein
+Copyright (c) 2000 by hartmut Goebel <h.goebel@crazy-compilers.com>
+Copyright (c) 1998-2002 John Aycock
 
   Permission is hereby granted, free of charge, to any person obtaining
   a copy of this software and associated documentation files (the

MANIFEST.in

@@ -1,9 +1,18 @@
 include README.rst
-include HISTORY.md
 include ChangeLog
+include HISTORY.md
+include HOW-TO-REPORT-A-BUG.md
+include LICENSE
+include Makefile
+include requirements.txt
+include requirements-dev.txt
+include DECOMPYLE-2.4-CHANGELOG.txt
 include __pkginfo__.py
 recursive-include uncompyle6 *.py
 include bin/uncompyle6
 include bin/pydisassemble
-recursive-include test *.py
+include pytest/Makefile
+include test/Makefile
+recursive-include test *.py *.pyc
 recursive-include pytest *.py
+recursive-include pytest/testdata *

Makefile

@@ -11,7 +11,7 @@ RM      ?= rm
 LINT    = flake8
 
 #EXTRA_DIST=ipython/ipy_trepan.py trepan
-PHONY=all check clean pytest check-long dist distclean lint flake8 test rmChangeLog clean_pyc
+PHONY=all check clean distcheck pytest check-long dist distclean lint flake8 test rmChangeLog clean_pyc
 
 TEST_TYPES=check-long check-short check-2.7 check-3.4
 
@@ -23,18 +23,33 @@ check:
 	@PYTHON_VERSION=`$(PYTHON) -V 2>&1 | cut -d ' ' -f 2 | cut -d'.' -f1,2`; \
 	$(MAKE) check-$$PYTHON_VERSION
 
+# Run all quick tests
+check-short: pytest
+	$(MAKE) -C test check-short
+
 #: Tests for Python 2.7, 3.3 and 3.4
 check-2.7 check-3.3 check-3.4: pytest
 	$(MAKE) -C test $@
 
-#: Tests for Python 3.5 - pytest doesn't work here
-check-3.5:
+#: Tests for Python 3.2 and 3.5 - pytest doesn't work here
+# Or rather 3.5 doesn't work not on Travis
+check-3.0 check-3.1 check-3.2 check-3.5 check-3.6:
 	$(MAKE) -C test $@
 
+check-3.7: pytest
+
 #:Tests for Python 2.6 (doesn't have pytest)
 check-2.6:
 	$(MAKE) -C test $@
 
+#:PyPy 2.6.1 PyPy 5.0.1, or PyPy 5.8.0-beta0
+# Skip for now
+2.6 5.0 5.3 5.6 5.8:
+
+#:PyPy pypy3-2.4.0 Python 3:
+pypy-3.2 2.4:
+	$(MAKE) -C test $@
+
 #: Run py.test tests
 pytest:
 	$(MAKE) -C pytest check
@@ -44,9 +59,13 @@ clean: clean_pyc
 	$(PYTHON) ./setup.py $@
 	(cd test && $(MAKE) clean)
 
-#: Create source (tarball) and binary (egg) distribution
-dist:
-	$(PYTHON) ./setup.py sdist bdist_egg
+#: Create source (tarball) and wheel distribution
+dist: distcheck
+	$(PYTHON) ./setup.py sdist bdist_wheel
+
+# perform some checks on the package via setup.py
+distcheck:
+	$(PYTHON) ./setup.py check
 
 #: Remove .pyc files
 clean_pyc:
@@ -73,6 +92,11 @@ bdist_egg:
 	$(PYTHON) ./setup.py bdist_egg
 
 
+#: Create binary wheel distribution
+wheel:
+	$(PYTHON) ./setup.py bdist_wheel
+
+
 # It is too much work to figure out how to add a new command to distutils
 # to do the following. I'm sure distutils will someday get there.
 DISTCLEAN_FILES = build dist *.pyc

NEWS

@@ -1,3 +1,401 @@
+uncompyle6 2.14.0 2017-11-26 johnnybamazing
+
+- Start to isolate grammar rules between versions
+  and remove used grammar rules
+- Fix a number of bytecode decompile problems
+  (many more remain)
+- Add stdlib/runtests.sh for even more rigourous testing
+
+uncompyle6 2.13.3 2017-11-13
+
+Overall: better 3.6 decompiling and some much needed code refactoring and cleanup
+
+
+- Start noting names in for template-action names; these are
+  used to check/assert we have the right node type
+- Simplify <import_from> rule
+- Pypy 5.80-beta testing tolerance
+- Start to clean up instruction mangling phase by using 3.6-style instructions
+  rather trying to parse the bytecode array. This largely been done in for versions 3.x;
+  3.0 custom mangling code has been reduced;
+  some 2.x conversion has been done, but more is desired. This make it possible to...
+- Handle EXTENDED_ARGS better. While relevant to all Python versions it is most noticeable in
+  version 3.6+ where in switching to wordcodes the size of operands has been reduced from 2**16
+  to 2**8. JUMP instruction then often need EXTENDED_ARGS.
+- Refactor find_jump_targets() with via working of of instructions rather the bytecode array.
+- use --weak-verify more and additional fuzzing on verify()
+- fragment parser now ignores errors in nested function definitions; an parameter was
+  added to assist here. Ignoring errors may be okay because the fragment parser often just needs,
+  well, *fragments*.
+- Distinguish RETURN_VALUE from RETURN_END_IF in exception bodies better in 3.6
+- bug in 3.x language changes: import queue va import Queue
+- reinstate some bytecode tests since decompiling has gotten better
+- Revise how to report a bug
+
+uncompyle6 2.13.2 2017-10-12
+
+- Re-release using a more automated approach
+
+uncompyle6 2.13.1 2017-10-11
+
+- Re-release because Python 2.4 source uploaded rather than 2.6-3.6
+
+uncompyle6 2.13.0 2017-10-10
+
+- Fixes in deparsing lambda expressions
+- Improve table-semantics descriptions
+- Document hacky customize arg count better (until we can remove it)
+- Update to use xdis 3.7.0 or greater
+
+uncompyle6 2.12.0 2017-09-26
+
+- Use xdis 3.6.0 or greater now
+- Small semantic table cleanups
+- Python 3.4's terms a little names better
+- Slightly more Python 3.7, but still failing a lot
+- Cross Python 2/3 compatibility with annotation arguments
+
+uncompyle6 2.11.5 2017-08-31
+
+- Skeletal support for Python 3.7
+
+uncompyle6 2.11.4 2017-08-15
+
+* scanner and parser now allow 3-part version string lookups,
+  e.g. 2.7.1 We allow a float here, but if passed a string like '2.7'. or
+* unpin 3.5.1. xdis 3.5.4 has been releasd and fixes the problems we had. Use that.
+* some routnes here moved to xdis. Use the xdis version
+* README.rst: Link typo Name is trepan2 now not trepan
+* xdis-forced change adjust for COMPARE_OP "is-not" in
+  semanatic routines. We need "is not".
+* Some PyPy tolerance in validate testing.
+* Some pyston tolerance
+
+uncompyle6 2.11.3 2017-08-09
+
+Very minor changes
+
+- RsT doc fixes and updates
+- use newer xdis, but not too new; 3.5.2 breaks uncompyle6
+- use xdis opcode sets
+- xdis "exception match" is now "exception-match"
+
+uncompyle6 2.11.2 2017-07-09
+
+- Start supporting Pypy 3.5 (5.7.1-beta)
+- use xdis 3.5.0's opcode sets and require xdis 3.5.0
+- Correct some Python 2.4-2.6 loop detection
+- guard against badly formatted bytecode
+
+uncompyle6 2.11.1 2017-06-25
+
+- Python 3.x annotation and function signature fixes
+- Bump xdis version
+- Small pysource bug fixes
+
+uncompyle6 2.11.0 2017-06-18 Fleetwood
+- Major improvements in fragment tracking
+  * Add nonterminal node in extractInfo
+  * tag more offsets in expressions
+  * tag array subscripts
+  * set YIELD value offset in a <yield> expr
+  * fix a long-standing bug in not adjusting final AST when melding other deparse ASTs
+- Fixes yet again for make_function node handling; document what's up here
+- Fix bug in snowflake Python 3.5 *args kwargs
+
+uncompyle6 2.10.1 2017-06-3 Marylin Frankel
+
+- fix some fragments parsing bugs
+  - was returning the wrong type sometimes in deparse_code_around_offset()
+  - capture function name in offsets
+  - track changes to ifelstrmtr node from pysource into fragments
+
+uncompyle6 2.10.0 2017-05-30 Elaine Gordon
+
+- Add fuzzy offset deparse look up
+- 3.6 bug fixes
+   - fix EXTENDED_ARGS handling (and in 2.6 and others)
+   - semantic routine make_function fragments.py
+   - MAKE_FUNCTION handling
+   - CALL_FUNCTION_EX handling
+   - async property on defs
+   - support for CALL_FUNCTION_KW (moagstar)
+- 3.5+ UNMAP_PACK and BUILD_UNMAP_PACK handling
+- 3.5 FUNCTION_VAR bug
+- 3.x pass statement insdie while True
+- Improve 3.2 decompilation
+- Fixed -o argument processing (grkov90)
+- Reduce scope of LOAD_ASSERT as expr to 3.4+
+- "await" statement fixes
+- 2.3, 2.4 "if 1 .." fixes
+- 3.x annotation fixes
+
+uncompyle6 2.9.11 2017-04-06
+
+- Better support for Python 3.5+ BUILD_MAP_UNPACK
+- Start 3.6 CALL_FUNCTION_EX support
+- Many decompilation bug fixes. (Many more remain). See ChangeLog
+
+uncompyle6 2.9.10 2017-02-25
+
+- Python grammar rule fixes
+- Add ability to get grammar coverage on runs
+- Handle Python 3.6 opcode BUILD_CONST_KEYMAP
+
+uncompyle6 2.9.9 2016-12-16
+
+- Remaining Python 3.5 ops handled
+  (this also means more Python 3.6 ops are handled)
+- Python 3.5 and 3.6 async and await handled
+- Python 3.0 decompilation improved
+- Python 3 annotations fixed
+- Better control-flow detection
+- Code cleanups and misc bug fixes
+
+uncompyle6 2.9.8 2016-12-16
+
+- Better control-flow detection
+- pseudo instruction THEN in 2.x
+  to disambiguate if from and
+- fix bug in --verify option
+- DRY (a little) control-flow detection
+- fix syntax in tuples with one element
+- if AST rule inheritence in Python 2.5
+- NAME_MODULE removal for Python <= 2.4
+- verifycall fixes for Python <= 2.4
+- more Python lint
+
+uncompyle6 2.9.7 2016-12-16
+
+- Start to handle 3.5/3.6 build_map_unpack_with_call
+- Some Python 3.6 bytecode to wordcode conversion fixes
+- option -g: show start-end range when possible
+- track print_docstring move to help (used in python 3.1)
+- verify: allow RETURN_VALUE to match RETURN_END_IF
+- some 3.2 compatibility
+- Better Python 3 control flow detection by adding Pseudo ELSE opcodes
+
+uncompyle6 2.9.6 2016-12-04
+
+- Shorten Python3 grammars with + and *
+  this requires spark parser 1.5.1
+- Add some AST reduction checks to improve
+  decompile accuracy. This too requires
+  spark parser 1.5.1
+
+uncompyle6 2.9.6 2016-11-20
+
+- Correct MANIFEST.in
+- More AST grammar checking
+- --linemapping option or linenumbers.line_number_mapping()
+  Shows correspondence of lines between source
+  and decompiled source
+- Some control flow adjustments in code for 2.x.
+  This is probably an improvement in 2.6 and before.
+  For 2.7 things are just shuffled around a little. Sigh.
+  Overall I think we are getting more precise in
+  or analysis even if it is not always reflected
+  in the results.
+- better control flow debugging output
+- Python 2 and 3 detect structure code is more similar
+- Handle Docstrings with embedded tiple quotes (""")
+
+uncompyle6 2.9.5 2016-11-13
+
+- Fix Python 3 bugs:
+  * improper while 1 else
+  * docstring indent
+  * 3.3 default values in lambda expressions
+  * start 3.0 decompilation (needs newer xdis)
+- Start grammar misparse checking
+
+
+uncompyle6 2.9.4 2016-11-02
+
+- Handle Python 3.x function annotations
+- track def keyword-parameter line-splitting in source code better
+- bump min xdis version to mask previous xdis bug
+
+uncompyle6 2.9.3 2016-10-26
+
+Release forced by incompatibility change in xdis 3.2.0.
+
+- Python 3.1 bugs:
+  * handle "with ... as"
+  * handle "with"
+  * Start handling def (...) -> yy (has bugs still)
+
+- DRY Python 3.x via inheritance
+- Python 3.6 work (from Daniel Bradburn)
+  * Handle 3.6 buildstring
+  * Handle 3.6 handle single and multiple fstring better
+
+
+uncompyle6 2.9.2 2016-10-15
+
+- use source-code line breaks to assist in where to break
+  in tuples and maps
+- Fix Python 1.5 decompyle bugs
+- Fix some Python 2.6 and below bugs
+- DRY fragments.py code a little
+
+uncompyle6 2.9.1 2016-10-09
+
+- Improved Python 1.5 decompiling
+- Handle old-style pre Python 2.2 classes
+
+uncompyle6 2.9.0 2016-10-09
+
+- Use xdis 3.0.0 protocol load_module.
+  this Forces change in requirements.txt and _pkg_info_.py
+- Start Python 1.5 decompiling; another round of work is needed to
+  remove bugs
+- Simplify python 2.1 grammar
+- Fix bug with -t ...  Wasn't showing source text when -t option was given
+- Fix 2.1-2.6 bug in list comprehension
+
+uncompyle6 2.8.4 2016-10-08
+
+- Python 3 disassembly bug fixes
+- Python 3.6 fstring bug fixes (from moagstar)
+- Python 2.1 disassembly
+- COME_FROM suffixes added in Python3
+- use .py extension in verification disassembly
+
+uncompyle6 2.8.3 2016-09-11 live from NYC!
+
+NOTE: this is possibly the last release before a major reworking of
+control-flow structure detection is done.
+
+- Lots of bug fixes in decompilation:
+  * 3.0 .. 3.4 whileTrue bug
+  * 3.x function declaration deparsing:
+    . 3.0 .. 3.2 *args processing
+    . 3.0 .. 3.2 call name and kwargs bug
+    . 3.0 .. getting parameter of *
+    . 3.0 .. handling variable number of args
+    . 3.0 .. "if" structure bugs
+  * 3.5+ if/else bugs
+  * 2.2-2.6 bugs
+    . try/except control flow
+    . a == b == c -like detection
+    . generator detection
+    . "while .. and" statement bugs
+    . handle "except <cond>, <var>"
+    . use older raise format in 2.x
+- scanner "disassemble" is now "ingest". True disassembly is done by xdis
+- Start accepting Python 3.1 bytecode
+- Add --weak-verify option on test_pyenvlib and test_pythonlib. This
+  catches more bugs more easily
+- bump xdis requirement so we can deparse dropbox 2.5 code
+- Added H. Goebel's changes before 2.4 in DECOMPYLE-2.4-CHANGELOG.txt
+
+uncompyle6 2.8.2 2016-08-29
+
+- Handle Python 3.6 format string conversions !r, !s, !a
+- Start to handle 3.1 bytecode
+- Fix some PyPy translation bugs
+- We now only handle 3.6.0a3+ since that is incompatible with 3.6 before that
+
+uncompyle6 2.8.1 2016-08-20
+
+- Add Python 2.2 decompilation
+
+- Fix bugs
+ * PyPy LOOKUP_METHOD bug
+ * Python 3.6 FORMAT_VALUE handles expressions now
+
+uncompyle6 2.8.0 2016-08-03
+
+- Start Python 3.6 support (moagstar)
+  more work on PEP 498 needed
+- tidy bytecode/word output
+- numerous decompiling bugs fixed
+- grammar testing started
+- show magic number in deparsed output
+- better grammar and semantic action segregation based
+  on python bytecode version
+
+uncompyle6 2.7.1 2016-07-26
+
+- PyPy bytecodes for 2.7 and 3.2 added
+- Instruction formatting improved slightly
+- 2.7 bytecode "continue" bug fixed
+
+uncompyle6 2.7.0 2016-07-15
+
+- Many Syntax and verification bugs removed
+  tested on standard libraries from 2.3.7 to 3.5.1
+  and they all decompile and verify fine.
+  I'm sure there are more bugs though.
+
+uncompyle6 2.6.2 2016-07-11 Manhattenhenge
+
+- Extend bytecodes back to 2.3
+- Fix bugs:
+  * 3.x and 2.7 set comprehensions,
+  * while1 loops
+  * continue statements
+- DRY and segregate grammar more
+
+uncompyle6 2.6.1 2016-07-08
+
+- Go over Python 2.5 bytecode deparsing
+  all library programs now deparse
+- Fix a couple bugs in 2.6 deparsing
+
+uncompyle6 2.6.0 2016-07-07
+
+- Improve Python 2.6 bytecode deparsing:
+  stdlib now will deparse something
+- Better <2.6 vs. 2.7 grammar separation
+- Fix some 2.7 deparsing bugs
+- Fix bug in installing uncompyle6 script
+- Doc improvements
+
+uncompyle6 2.5.0 2016-06-22  Summer Solstice
+
+- Much better Python 3.2-3.5 coverage.
+  3.4.6 is probably the best;3.2 and 3.5 are weaker
+- Better AST printing with -t
+- Better error reporting
+- Better fragment offset tracking
+- Some (much-needed) code refactoring
+
+uncompyle6 2.4.0 2016-05-18 (in memory of Lewis Bernstein)
+
+- Many Python 3 bugs fixed:
+  * Python 3.2 to 3.5 libraries largely
+    uncompyle and most verify
+- pydisassembler:
+  * disassembles all code objects in a file
+  * can select showing bytecode before
+    or after uncompyle mangling, option -U
+- DRY scanner code (but more is desired)
+- Some code cleanup (but more is desired)
+- Misc Bugs fixed:
+  * handle complex number unmarshaling
+  * Running on Python 2 to works on Python 3.5 bytecodes now
+
+uncompyle6 2.3.5 and 2.3.6 2016-05-14
+
+- Python 2 class decorator fix (thanks to Tey)
+- Fix fragment parsing bugs
+- Fix some Python 3 parsing bugs:
+  * Handling single in * parameter
+  * "while True"
+  * escape from for inside if
+  * yield expressions
+- Correct history based on info from Dan Pascu
+- Fix up pip packaging, ugh.
+
+uncompyle6 2.3.4 2016-05-5
+
+- More Python 3.5 parsing bugs addressed
+- decompiling Python 3.5 from other Python versions works
+- test from Python 3.2
+- remove "__module__ = __name__" in 3.0 <= Python 3.2
+
 uncompyle6 2.3.3 2016-05-3
 
 - Fix bug in running uncompyle6 script on Python 3
@@ -25,7 +423,7 @@ uncompyle6 2.2.0 2016-04-30
 
 uncompyle6 2.2.0 2016-04-02
 
-- Support single-mode (in addtion to exec-mode) compilation
+- Support single-mode (in addition to exec-mode) compilation
 - Start to DRY Python 2 and Python 3 grammars
 - Fix bug in if else ternary  construct
 - Fix bug in uncomplye6 -d and -r options (via lelicopter)

PKG-INFO

@@ -5,6 +5,6 @@ Summary: Python byte-code to source-code converter
 Home-page: http://github.com/rocky/python-uncompyle6
 Author: Rocky
 Author-email: rb@dustyfeet.com
-License: GPLv3
+License: MIT
 Description: UNKNOWN
 Platform: UNKNOWN

README.rst

@@ -3,29 +3,37 @@
 uncompyle6
 ==========
 
-A native Python bytecode Disassembler, Decompiler, Fragment Decompiler
-and bytecode library. Follows in the tradition of decompyle, uncompyle, and uncompyle2.
+A native Python cross-version Decompiler and Fragment Decompiler.
+The successor to decompyle, uncompyle, and uncompyle2.
 
 
 Introduction
 ------------
 
 *uncompyle6* translates Python bytecode back into equivalent Python
-source code. It accepts bytecodes from Python version 2.5 to 3.4 or
-so. The code requires Python 2.6 or later and has been tested on Python
-running versions 2.6, 2.7, 3.3, 3.4 and 3.5.
+source code. It accepts bytecodes from Python version 1.5, and 2.1 to
+3.7 or so, including PyPy bytecode and Dropbox's Python 2.5 bytecode.
 
 Why this?
 ---------
 
-There were a number of decompyle, uncompile, uncompyle2, uncompyle3
-forks around. All of them come basically from the same code base, and
-almost all of them not maintained very well. This code pulls these together
-and addresses a number of open issues in those.
+There were a number of decompyle, uncompyle, uncompyle2, uncompyle3
+forks around. All of them came basically from the same code base, and
+almost all of them no were no longer actively maintained. Only one
+handled Python 3, and even there, only 3.2 or 3.3 depending on which
+code is used. This code pulls these together and moves forward.
 
-What makes this different from other CPython bytecode decompilers?  Its
-ability to deparse just fragments and give source-code information
-around a given bytecode offset.
+This project has the most complete support for Python 3.3 and above
+and the best all-around Python support.
+
+We are serious about testing, and use automated processes to find
+bugs. In the issue trackers for other decompilers, you will find a
+number of bugs we've found along the way. Very few to none of them are
+not fixed in the other decompilers.
+
+Another thing that makes this different from other CPython bytecode
+decompilers is the ability to deparse just fragments and give
+source-code information around a given bytecode offset.
 
 I use this to deparse fragments of code inside my trepan_
 debuggers_. For that, I need to record text fragments for all
@@ -39,12 +47,13 @@ location in more detail than just a line number.  It can be also used
 when source-code information does not exist and there is just bytecode
 information.
 
-Other parts of the library can be used inside Python for various
-bytecode-related tasks. For example you can read in bytecode,
-i.e. perform a version-independent `marshal.loads()`, and disassemble
-the bytecode using a version of Python different from the one used to
-compile the bytecode.
+Requirements
+------------
 
+This project requires Python 2.6 or later, PyPy 3-2.4, or PyPy-5.0.1.
+Python versions 2.4-2.7 are supported in the python-2.4 branch.
+The bytecode files it can read has been tested on Python bytecodes from
+versions 1.5, 2.1-2.7, and 3.0-3.6 and the above-mentioned PyPy versions.
 
 Installation
 ------------
@@ -53,13 +62,13 @@ This uses setup.py, so it follows the standard Python routine:
 
 ::
 
-    pip install -r requirements.txt
+    pip install -e .
     pip install -r requirements-dev.txt
     python setup.py install # may need sudo
     # or if you have pyenv:
     python setup.py develop
 
-A GNU makefile is also provided so `make install` (possibly as root or
+A GNU makefile is also provided so :code:`make install` (possibly as root or
 sudo) will do the steps above.
 
 Testing
@@ -73,7 +82,7 @@ A GNU makefile has been added to smooth over setting running the right
 command, and running tests from fastest to slowest.
 
 If you have remake_ installed, you can see the list of all tasks
-including tests via `remake --tasks`
+including tests via :code:`remake --tasks`
 
 
 Usage
@@ -83,33 +92,116 @@ Run
 
 ::
 
-     ./bin/uncompyle6 -h
-     ./bin/pydisassemble -h
+$ uncompyle6 *compiled-python-file-pyc-or-pyo*
 
-for usage help.
+For usage help:
+
+::
+
+   $ uncompyle6 -h
+
+If you want strong verification of the correctness of the
+decompilation process, add the `--verify` option. But there are
+situations where this will indicate a failure, although the generated
+program is semantically equivalent. Using option `--weak-verify` will
+tell you if there is something definitely wrong. Generally, large
+swaths of code are decompiled correctly, if not the entire program.
+
+You can also cross compare the results with pycdc_ . Since they work
+differently, bugs here often aren't in that, and vice versa.
 
 
 Known Bugs/Restrictions
 -----------------------
 
-Python 2 deparsing is probably as solid as the various versions of
-uncompyle2.  Python 3 deparsing is okay but not as solid. Python 3.5 is missing some of new opcodes added, but still often works.
+The biggest known and possibly fixable (but hard) problem has to do
+with handling control flow. (Python has probably the most diverse and
+screwy set of compound statements I've ever seen; a number of the
+usual ones like else clauses on loops and try blocks I suspect most
+programmers don't know aobut.)
+
+All of the Python decompilers I have looked at have the same
+problem. In some cases we can detect an erroneous decompilation and
+report that.
+
+*Verification* is the process of decompiling bytecode, compiling with
+a Python for that bytecode version, and then comparing the bytecode
+produced by the decompiled/compiled program. Some allowance is made
+for inessential differences. But other semantically equivalent
+differences are not caught. For example ``1 and 0`` is decompiled to
+the equivalent ``0``; remnants of the first true evaluation (1) is
+lost when Python compiles this. When Python next compiles ``0`` the
+resulting code is simpler.
+
+*Weak Verification*
+on the other hand doesn't check bytecode for equivalence but does
+check to see if the resulting decompiled source is a valid Python
+program by running the Python interpreter. Because the Python language
+has changed so much, for best results you should use the same Python
+Version in checking as used in the bytecode.
+
+Finally, we have automated running the standard Python tests after
+first compiling and decompiling the test program. Results here are a
+bit weak (if not better than most other Python decompilers). But over
+time this will probably get better.
+
+Python support is strongest in Python 2 for 2.7 and drops off as you
+get further away from that. Support is also probably pretty good for
+python 2.3-2.4 since a lot of the goodness of early the version of the
+decompiler from that era has been preserved (and Python compilation in
+that era was minimal)
+
+Later distributions average about 200 files. There is some work to do
+on the lower end Python versions which is more difficult for us to
+handle since we don't have a Python interpreter for versions 1.5, 1.6,
+and 2.0.
+
+In the Python 3 series, Python support is is strongest around 3.4 or
+3.3 and drops off as you move further away from those versions. Python
+3.6 changes things drastically by using word codes rather than byte
+codes. That has been addressed, but then it also changes function call
+opcodes and its semantics and has more problems with control flow than
+3.5 has. Between Python 3.5, 3.6 and 3.7 there have been major changes
+to the `MAKE_FUNCTION` and `CALL_FUNCTION` instructions. Those are
+not handled yet.
+
+Currently not all Python magic numbers are supported. Specifically in
+some versions of Python, notably Python 3.6, the magic number has
+changes several times within a version. We support only the released
+magic. There are also customized Python interpreters, notably Dropbox,
+which use their own magic and encrypt bytcode. With the exception of
+the Dropbox's old Python 2.5 interpreter this kind of thing is not
+handled.
+
+We also don't handle PJOrion_ obfuscated code. For that try: PJOrion
+Deobfuscator_ to unscramble the bytecode to get valid bytecode before
+trying this tool.
+
+Handling pathologically long lists of expressions or statements is
+slow.
+
+
+There is lots to do, so please dig in and help.
 
 See Also
 --------
 
-* https://github.com/zrax/pycdc : supports all versions of Python and is written in C++
-* https://code.google.com/archive/p/unpyc3/ : supports Python 3.2 only
+* https://github.com/zrax/pycdc : supports all versions of Python and is written in C++. Support for later Python 3 versions is a bit lacking though.
+* https://code.google.com/archive/p/unpyc3/ : supports Python 3.2 only. The above projects use a different decompiling technique than what is used here.
+* https://github.com/figment/unpyc3/ : fork of above, but supports Python 3.3 only. Includes some fixes like supporting function annotations
+* The HISTORY_ file.
+* `How to report a bug <https://github.com/rocky/python-uncompyle6/blob/master/HOW-TO-REPORT-A-BUG.md>`_
+* https://github.com/rocky/python-xdis : Cross Python version disassembler
+* https://github.com/rocky/python-xasm : Cross Python version assembler
 
-The above projects use a different decompiling technique what is used here.
 
-The HISTORY file.
-
-.. |downloads| image:: https://img.shields.io/pypi/dd/uncompyle6.svg
-.. _trepan: https://pypi.python.org/pypi/trepan
+.. _trepan: https://pypi.python.org/pypi/trepan2
+.. _HISTORY: https://github.com/rocky/python-uncompyle6/blob/master/HISTORY.md
 .. _debuggers: https://pypi.python.org/pypi/trepan3k
 .. _remake: https://bashdb.sf.net/remake
 .. _pycdc: https://github.com/zrax/pycdc
 .. _this: https://github.com/rocky/python-uncompyle6/wiki/Deparsing-technology-and-its-use-in-exact-location-reporting
 .. |buildstatus| image:: https://travis-ci.org/rocky/python-uncompyle6.svg
 		 :target: https://travis-ci.org/rocky/python-uncompyle6
+.. _PJOrion: http://www.koreanrandom.com/forum/topic/15280-pjorion-%D1%80%D0%B5%D0%B4%D0%B0%D0%BA%D1%82%D0%B8%D1%80%D0%BE%D0%B2%D0%B0%D0%BD%D0%B8%D0%B5-%D0%BA%D0%BE%D0%BC%D0%BF%D0%B8%D0%BB%D1%8F%D1%86%D0%B8%D1%8F-%D0%B4%D0%B5%D0%BA%D0%BE%D0%BC%D0%BF%D0%B8%D0%BB%D1%8F%D1%86%D0%B8%D1%8F-%D0%BE%D0%B1%D1%84
+.. _Deobfuscator: https://github.com/extremecoders-re/PjOrion-Deobfuscator

__pkginfo__.py

@@ -9,20 +9,23 @@
 
 # Things that change more often go here.
 copyright   = """
-Copyright (C) 2015, 2016 Rocky Bernstein <rb@dustyfeet.com>.
+Copyright (C) 2015-2017 Rocky Bernstein <rb@dustyfeet.com>.
 """
 
-classifiers =  ['Development Status :: 3 - Alpha',
+classifiers =  ['Development Status :: 5 - Production/Stable',
                 'Intended Audience :: Developers',
                 'Operating System :: OS Independent',
                 'Programming Language :: Python',
-                'Programming Language :: Python :: 2',
+                'Programming Language :: Python :: 2.4',
+                'Programming Language :: Python :: 2.5',
                 'Programming Language :: Python :: 2.6',
                 'Programming Language :: Python :: 2.7',
-                'Programming Language :: Python :: 3',
+                'Programming Language :: Python :: 3.1',
+                'Programming Language :: Python :: 3.2',
                 'Programming Language :: Python :: 3.3',
                 'Programming Language :: Python :: 3.4',
                 'Programming Language :: Python :: 3.5',
+                'Programming Language :: Python :: 3.6',
                 'Topic :: Software Development :: Debuggers',
                 'Topic :: Software Development :: Libraries :: Python Modules',
                 ]
@@ -30,33 +33,35 @@ classifiers =  ['Development Status :: 3 - Alpha',
 # The rest in alphabetic order
 author             = "Rocky Bernstein, Hartmut Goebel, John Aycock, and others"
 author_email       = "rb@dustyfeet.com"
+entry_points       = {
+    'console_scripts': [
+        'uncompyle6=uncompyle6.bin.uncompile:main_bin',
+        'pydisassemble=uncompyle6.bin.pydisassemble:main',
+    ]}
 ftp_url            = None
-install_requires   = ['python-spark >= 1.1.0']
-license            = 'GPL'
-
+install_requires   = ['spark-parser >= 1.8.0, < 1.9.0',
+                      'xdis >= 3.6.1, < 3.7.0', 'six']
 license            = 'MIT'
 mailing_list       = 'python-debugger@googlegroups.com'
 modname            = 'uncompyle6'
-packages           = ['uncompyle6', 'uncompyle6.opcodes', 'uncompyle6.semantics', 'uncompyle6.scanners', 'uncompyle6.parsers']
 py_modules         = None
-short_desc         = 'Python byte-code disassembler and source-code converter'
-scripts            = ['bin/uncompyle6', 'bin/pydisassemble']
-
-import os.path
-
-
-def get_srcdir():
-    filename = os.path.normcase(os.path.dirname(os.path.abspath(__file__)))
-    return os.path.realpath(filename)
-
-ns = {}
+short_desc         = 'Python cross-version byte-code deparser'
 web                = 'https://github.com/rocky/python-uncompyle6/'
 
 # tracebacks in zip files are funky and not debuggable
 zip_safe = True
 
 
-def read(*rnames):
-    return open(os.path.join(os.path.dirname(__file__), *rnames)).read()
+import os.path
+def get_srcdir():
+    filename = os.path.normcase(os.path.dirname(os.path.abspath(__file__)))
+    return os.path.realpath(filename)
 
+srcdir = get_srcdir()
+
+def read(*rnames):
+    return open(os.path.join(srcdir, *rnames)).read()
+
+# Get info from files; set: long_description and VERSION
 long_description   = ( read("README.rst") + '\n' )
+exec(read('uncompyle6/version.py'))

admin-tools/README.md

@@ -0,0 +1,11 @@
+Making a release is a somewhat tedious process so I've automated it a little
+
+
+Here are tools that I, rocky, use to check and build a distribution.
+
+They are customized to my environment:
+- I use pyenv to various Python versions installed
+- I have git repos for xdis, and spark parser at the same level as uncompyle6
+
+There may be other rocky-specific things that need customization.
+how-to-make-a-release.txt has overall how I make a release

admin-tools/check-newer-versions.sh

@@ -0,0 +1,26 @@
+#!/bin/bash
+function finish {
+  cd $owd
+}
+
+# FIXME put some of the below in a common routine
+owd=$(pwd)
+trap finish EXIT
+
+cd $(dirname ${BASH_SOURCE[0]})
+if ! source ./pyenv-newer-versions ; then
+    exit $?
+fi
+if ! source ./setup-master.sh ; then
+    exit $?
+fi
+cd ..
+for version in $PYVERSIONS; do
+    if ! pyenv local $version ; then
+	exit $?
+    fi
+    make clean && pip install -e .
+    if ! make check; then
+	exit $?
+    fi
+done

admin-tools/check-older-versions.sh

@@ -0,0 +1,27 @@
+#!/bin/bash
+function finish {
+  cd $owd
+}
+owd=$(pwd)
+trap finish EXIT
+
+cd $(dirname ${BASH_SOURCE[0]})
+if ! source ./pyenv-older-versions ; then
+    exit $?
+fi
+if ! source ./setup-python-2.4.sh ; then
+    exit $?
+fi
+
+PYVERSIONS='2.7.14 2.6.9 3.3.6 3.4.2 3.5.4 3.6.3'
+
+cd ..
+for version in $PYVERSIONS; do
+    if ! pyenv local $version ; then
+	exit $?
+    fi
+    make clean && python setup.py develop
+    if ! make check ; then
+	exit $?
+    fi
+done

admin-tools/how-to-make-a-release.md

@@ -0,0 +1,88 @@
+<!-- markdown-toc start - Don't edit this section. Run M-x markdown-toc-refresh-toc -->
+**Table of Contents**
+
+- [Get latest sources:](#get-latest-sources)
+- [Change version in uncompyle6/version.py. Then:](#change-version-in-uncompyle6versionpy-then)
+- [Update ChangeLog:](#update-changelog)
+- [Update NEWS from ChangeLog. Then:](#update-news-from-changelog-then)
+- [Make sure pyenv is running and check newer versions](#make-sure-pyenv-is-running-and-check-newer-versions)
+- [Switch to python-2.4, sync that up and build that first since it creates a tarball which we don't want.](#switch-to-python-24-sync-that-up-and-build-that-first-since-it-creates-a-tarball-which-we-dont-want)
+- [Update NEWS from master branch](#update-news-from-master-branch)
+- [Check against all versions](#check-against-all-versions)
+- [Make packages and tag](#make-packages-and-tag)
+- [Upload single package and look at Rst Formating](#upload-single-package-and-look-at-rst-formating)
+- [Upload rest of versions](#upload-rest-of-versions)
+- [Push tags:](#push-tags)
+
+<!-- markdown-toc end -->
+# Get latest sources:
+
+    $ . ./admin-tool/update-sources.sh
+
+# Change version in uncompyle6/version.py. Then:
+
+	$ emacs uncompyle6/version.py
+    $ source uncompyle6/version.py
+    $ echo $VERSION
+    $ git commit -m"Get ready for release $VERSION" .
+
+# Update ChangeLog:
+
+    $ make ChangeLog
+
+#  Update NEWS from ChangeLog. Then:
+
+	$ emacs NEWS
+    $ make check
+    $ git commit --amend .
+    $ git push   # get CI testing going early
+
+# Make sure pyenv is running and check newer versions
+
+    $ pyenv local && source admin-tools/check-newer-versions.sh
+
+# Switch to python-2.4, sync that up and build that first since it creates a tarball which we don't want.
+
+    $ source admin-tools/setup-python-2.4.sh
+    $ rm ChangeLog
+
+    # $ git merge master ?
+
+# Update NEWS from master branch
+
+    $ git commit -m"Get ready for release $VERSION" .
+
+# Check against all versions
+
+    $ source admin-tools/check-older-versions.sh
+    $ source admin-tools/check-newer-versions.sh
+
+# Make packages and tag
+
+    $ admin-tools/make-dist-older.sh
+    $ git tag release-python-2.4-$VERSION
+
+    $ admin-tools/make-dist-newer.sh
+    $ git tag release-$VERSION
+
+# Upload single package and look at Rst Formating
+
+    $ twine upload dist/uncompyle6-${VERSION}-py3.3.egg
+
+# Upload rest of versions
+
+    $ twine upload dist/uncompyle6-${VERSION}*
+
+# Push tags:
+
+    $ git push --tags
+
+# Check on a VM
+
+    $ cd /virtual/vagrant/virtual/vagrant/ubuntu-zesty
+	$ vagrant up
+	$ vagrant ssh
+	$ pyenv local 3.5.2
+	$ pip install --upgrade uncompyle6
+	$ exit
+	$ vagrant halt

admin-tools/make-dist-newer.sh

@@ -0,0 +1,38 @@
+#!/bin/bash
+PACKAGE=uncompyle6
+
+# FIXME put some of the below in a common routine
+function finish {
+  cd $owd
+}
+
+cd $(dirname ${BASH_SOURCE[0]})
+owd=$(pwd)
+trap finish EXIT
+
+if ! source ./pyenv-newer-versions ; then
+    exit $?
+fi
+if ! source ./setup-master.sh ; then
+    exit $?
+fi
+
+cd ..
+source $PACKAGE/version.py
+echo $VERSION
+
+for pyversion in $PYVERSIONS; do
+    if ! pyenv local $pyversion ; then
+	exit $?
+    fi
+    # pip bdist_egg create too-general wheels. So
+    # we narrow that by moving the generated wheel.
+
+    # Pick out first two number of version, e.g. 3.5.1 -> 35
+    first_two=$(echo $pyversion | cut -d'.' -f 1-2 | sed -e 's/\.//')
+    rm -fr build
+    python setup.py bdist_egg bdist_wheel
+    mv -v dist/${PACKAGE}-$VERSION-{py2.py3,py$first_two}-none-any.whl
+done
+
+python ./setup.py sdist

admin-tools/make-dist-older.sh

@@ -0,0 +1,39 @@
+#!/bin/bash
+PACKAGE=uncompyle6
+
+# FIXME put some of the below in a common routine
+function finish {
+  cd $owd
+}
+owd=$(pwd)
+trap finish EXIT
+
+cd $(dirname ${BASH_SOURCE[0]})
+if ! source ./pyenv-older-versions ; then
+    exit $?
+fi
+if ! source ./setup-python-2.4.sh ; then
+    exit $?
+fi
+
+cd ..
+source $PACKAGE/version.py
+echo $VERSION
+
+for pyversion in $PYVERSIONS; do
+    if ! pyenv local $pyversion ; then
+	exit $?
+    fi
+
+    rm -fr build
+    python setup.py bdist_egg
+done
+
+# Pypi can only have one source tarball.
+# Tarballs can get created from the above setup, so make sure to remove them since we want
+# the tarball from master.
+
+tarball=dist/${PACKAGE}-$VERSION-tar.gz
+if [[ -f $tarball ]]; then
+    rm -v dist/${PACKAGE}-$VERSION-tar.gz
+fi

admin-tools/pyenv-newer-versions

@@ -0,0 +1,6 @@
+# -*- shell-script -*-
+if [[ $0 == ${BASH_SOURCE[0]} ]] ; then
+    echo "This script should be *sourced* rather than run directly through bash"
+    exit 1
+fi
+export PYVERSIONS='3.5.2 3.6.2 2.6.9 3.3.6 2.7.14 3.4.2'

admin-tools/pyenv-older-versions

@@ -0,0 +1,6 @@
+# -*- shell-script -*-
+if [[ $0 == ${BASH_SOURCE[0]} ]] ; then
+    echo "This script should be *sourced* rather than run directly through bash"
+    exit 1
+fi
+export PYVERSIONS='2.4.6 2.5.6'

admin-tools/setup-master.sh

@@ -0,0 +1,22 @@
+#!/bin/bash
+PYTHON_VERSION=3.6.3
+
+# FIXME put some of the below in a common routine
+function finish {
+  cd $owd
+}
+
+export PATH=$HOME/.pyenv/bin/pyenv:$PATH
+owd=$(pwd)
+bs=${BASH_SOURCE[0]}
+if [[ $0 == $bs ]] ; then
+    echo "This script should be *sourced* rather than run directly through bash"
+    exit 1
+fi
+mydir=$(dirname $bs)
+fulldir=$(readlink -f $mydir)
+cd $fulldir/..
+(cd ../python-spark && git checkout master && pyenv local $PYTHON_VERSION) && git pull && \
+    (cd ../python-xdis && git checkout master && pyenv local $PYTHON_VERSION) && git pull && \
+    git checkout master && pyenv local $PYTHON_VERSION && git pull
+cd $owd

admin-tools/setup-python-2.4.sh

@@ -0,0 +1,16 @@
+#!/bin/bash
+PYTHON_VERSION=2.4.6
+
+owd=$(pwd)
+bs=${BASH_SOURCE[0]}
+if [[ $0 == $bs ]] ; then
+    echo "This script should be *sourced* rather than run directly through bash"
+    exit 1
+fi
+mydir=$(dirname $bs)
+fulldir=$(readlink -f $mydir)
+cd $fulldir/..
+(cd ../python-spark && git checkout python-2.4 && pyenv local $PYTHON_VERSION) && git pull && \
+    (cd ../python-xdis && git checkout python-2.4 && pyenv local $PYTHON_VERSION) && git pull && \
+    git checkout python-2.4 &&  pyenv local $PYTHON_VERSION && git pull
+cd $owd

appveyor.yml

@@ -0,0 +1,78 @@
+environment:
+  global:
+    # SDK v7.0 MSVC Express 2008's SetEnv.cmd script will fail if the
+    # /E:ON and /V:ON options are not enabled in the batch script intepreter
+    # See: http://stackoverflow.com/a/13751649/163740
+    CMD_IN_ENV: "cmd /E:ON /V:ON /C .\\appveyor\\run_with_env.cmd"
+
+  matrix:
+
+    # Pre-installed Python versions, which Appveyor may upgrade to
+    # a later point release.
+    # See: http://www.appveyor.com/docs/installed-software#python
+
+    # - PYTHON: "C:\\Python27"
+    #   PYTHON_VERSION: "2.7.x"
+    #   PYTHON_ARCH: "32"
+
+    - PYTHON: "C:\\Python27-x64"
+      PYTHON_VERSION: "2.7.x"
+      PYTHON_ARCH: "64"
+
+    # - PYTHON: "C:\\Python26"
+    #   PYTHON_VERSION: "2.6.x"
+    #   PYTHON_ARCH: "32"
+
+    # - PYTHON: "C:\\Python26-x64"
+    #   PYTHON_VERSION: "2.6.x"
+    #   PYTHON_ARCH: "64"
+
+install:
+  # We need wheel installed to build wheels
+  - "%PYTHON%\\python.exe -m pip install wheel"
+
+  # Install Python (from the official .msi of http://python.org) and pip when
+  # not already installed.
+  - ps: if (-not(Test-Path($env:PYTHON))) { & appveyor\install.ps1 }
+
+  # Prepend newly installed Python to the PATH of this build (this cannot be
+  # done from inside the powershell script as it would require to restart
+  # the parent CMD process).
+  - "SET PATH=%PYTHON%;%PYTHON%\\Scripts;%PATH%"
+  - "SET HOME=."
+
+  # Check that we have the expected version and architecture for Python
+  - "python --version"
+  - "python -c \"import struct; print(struct.calcsize('P') * 8)\""
+
+  # Upgrade to the latest version of pip to avoid it displaying warnings
+  # about it being out of date.
+  - "pip install --disable-pip-version-check --user --upgrade pip"
+
+  # Install the build dependencies of the project. If some dependencies contain
+  # compiled extensions and are not provided as pre-built wheel packages,
+  # pip will build them from source using the MSVC compiler matching the
+  # target Python version and architecture
+  - "%CMD_IN_ENV% pip install -r requirements.txt"
+
+build_script:
+  # Build the compiled extension
+  - "%CMD_IN_ENV% python setup.py build"
+
+test_script:
+  # Run the project tests
+  - "%CMD_IN_ENV% python test/test_pyenvlib.py --native --weak-verify"
+
+after_test:
+  # If tests are successful, create binary packages for the project.
+  - "%CMD_IN_ENV% python setup.py bdist_wininst"
+  - "%CMD_IN_ENV% python setup.py bdist_msi"
+  - ps: "ls dist"
+
+artifacts:
+  # Archive the generated packages in the ci.appveyor.com build report.
+  - path: dist\*
+
+#on_success:
+#  - TODO: upload the content of dist/*.whl to a public wheelhouse
+#

appveyor/install.ps1

@@ -0,0 +1,229 @@
+# Sample script to install Python and pip under Windows
+# Authors: Olivier Grisel, Jonathan Helmus, Kyle Kastner, and Alex Willmer
+# License: CC0 1.0 Universal: http://creativecommons.org/publicdomain/zero/1.0/
+
+$MINICONDA_URL = "http://repo.continuum.io/miniconda/"
+$BASE_URL = "https://www.python.org/ftp/python/"
+$GET_PIP_URL = "https://bootstrap.pypa.io/get-pip.py"
+$GET_PIP_PATH = "C:\get-pip.py"
+
+$PYTHON_PRERELEASE_REGEX = @"
+(?x)
+(?<major>\d+)
+\.
+(?<minor>\d+)
+\.
+(?<micro>\d+)
+(?<prerelease>[a-z]{1,2}\d+)
+"@
+
+
+function Download ($filename, $url) {
+    $webclient = New-Object System.Net.WebClient
+
+    $basedir = $pwd.Path + "\"
+    $filepath = $basedir + $filename
+    if (Test-Path $filename) {
+        Write-Host "Reusing" $filepath
+        return $filepath
+    }
+
+    # Download and retry up to 3 times in case of network transient errors.
+    Write-Host "Downloading" $filename "from" $url
+    $retry_attempts = 2
+    for ($i = 0; $i -lt $retry_attempts; $i++) {
+        try {
+            $webclient.DownloadFile($url, $filepath)
+            break
+        }
+        Catch [Exception]{
+            Start-Sleep 1
+        }
+    }
+    if (Test-Path $filepath) {
+        Write-Host "File saved at" $filepath
+    } else {
+        # Retry once to get the error message if any at the last try
+        $webclient.DownloadFile($url, $filepath)
+    }
+    return $filepath
+}
+
+
+function ParsePythonVersion ($python_version) {
+    if ($python_version -match $PYTHON_PRERELEASE_REGEX) {
+        return ([int]$matches.major, [int]$matches.minor, [int]$matches.micro,
+                $matches.prerelease)
+    }
+    $version_obj = [version]$python_version
+    return ($version_obj.major, $version_obj.minor, $version_obj.build, "")
+}
+
+
+function DownloadPython ($python_version, $platform_suffix) {
+    $major, $minor, $micro, $prerelease = ParsePythonVersion $python_version
+
+    if (($major -le 2 -and $micro -eq 0) `
+        -or ($major -eq 3 -and $minor -le 2 -and $micro -eq 0) `
+        ) {
+        $dir = "$major.$minor"
+        $python_version = "$major.$minor$prerelease"
+    } else {
+        $dir = "$major.$minor.$micro"
+    }
+
+    if ($prerelease) {
+        if (($major -le 2) `
+            -or ($major -eq 3 -and $minor -eq 1) `
+            -or ($major -eq 3 -and $minor -eq 2) `
+            -or ($major -eq 3 -and $minor -eq 3) `
+            ) {
+            $dir = "$dir/prev"
+        }
+    }
+
+    if (($major -le 2) -or ($major -le 3 -and $minor -le 4)) {
+        $ext = "msi"
+        if ($platform_suffix) {
+            $platform_suffix = ".$platform_suffix"
+        }
+    } else {
+        $ext = "exe"
+        if ($platform_suffix) {
+            $platform_suffix = "-$platform_suffix"
+        }
+    }
+
+    $filename = "python-$python_version$platform_suffix.$ext"
+    $url = "$BASE_URL$dir/$filename"
+    $filepath = Download $filename $url
+    return $filepath
+}
+
+
+function InstallPython ($python_version, $architecture, $python_home) {
+    Write-Host "Installing Python" $python_version "for" $architecture "bit architecture to" $python_home
+    if (Test-Path $python_home) {
+        Write-Host $python_home "already exists, skipping."
+        return $false
+    }
+    if ($architecture -eq "32") {
+        $platform_suffix = ""
+    } else {
+        $platform_suffix = "amd64"
+    }
+    $installer_path = DownloadPython $python_version $platform_suffix
+    $installer_ext = [System.IO.Path]::GetExtension($installer_path)
+    Write-Host "Installing $installer_path to $python_home"
+    $install_log = $python_home + ".log"
+    if ($installer_ext -eq '.msi') {
+        InstallPythonMSI $installer_path $python_home $install_log
+    } else {
+        InstallPythonEXE $installer_path $python_home $install_log
+    }
+    if (Test-Path $python_home) {
+        Write-Host "Python $python_version ($architecture) installation complete"
+    } else {
+        Write-Host "Failed to install Python in $python_home"
+        Get-Content -Path $install_log
+        Exit 1
+    }
+}
+
+
+function InstallPythonEXE ($exepath, $python_home, $install_log) {
+    $install_args = "/quiet InstallAllUsers=1 TargetDir=$python_home"
+    RunCommand $exepath $install_args
+}
+
+
+function InstallPythonMSI ($msipath, $python_home, $install_log) {
+    $install_args = "/qn /log $install_log /i $msipath TARGETDIR=$python_home"
+    $uninstall_args = "/qn /x $msipath"
+    RunCommand "msiexec.exe" $install_args
+    if (-not(Test-Path $python_home)) {
+        Write-Host "Python seems to be installed else-where, reinstalling."
+        RunCommand "msiexec.exe" $uninstall_args
+        RunCommand "msiexec.exe" $install_args
+    }
+}
+
+function RunCommand ($command, $command_args) {
+    Write-Host $command $command_args
+    Start-Process -FilePath $command -ArgumentList $command_args -Wait -Passthru
+}
+
+
+function InstallPip ($python_home) {
+    $pip_path = $python_home + "\Scripts\pip.exe"
+    $python_path = $python_home + "\python.exe"
+    if (-not(Test-Path $pip_path)) {
+        Write-Host "Installing pip..."
+        $webclient = New-Object System.Net.WebClient
+        $webclient.DownloadFile($GET_PIP_URL, $GET_PIP_PATH)
+        Write-Host "Executing:" $python_path $GET_PIP_PATH
+        & $python_path $GET_PIP_PATH
+    } else {
+        Write-Host "pip already installed."
+    }
+}
+
+
+function DownloadMiniconda ($python_version, $platform_suffix) {
+    if ($python_version -eq "3.4") {
+        $filename = "Miniconda3-3.5.5-Windows-" + $platform_suffix + ".exe"
+    } else {
+        $filename = "Miniconda-3.5.5-Windows-" + $platform_suffix + ".exe"
+    }
+    $url = $MINICONDA_URL + $filename
+    $filepath = Download $filename $url
+    return $filepath
+}
+
+
+function InstallMiniconda ($python_version, $architecture, $python_home) {
+    Write-Host "Installing Python" $python_version "for" $architecture "bit architecture to" $python_home
+    if (Test-Path $python_home) {
+        Write-Host $python_home "already exists, skipping."
+        return $false
+    }
+    if ($architecture -eq "32") {
+        $platform_suffix = "x86"
+    } else {
+        $platform_suffix = "x86_64"
+    }
+    $filepath = DownloadMiniconda $python_version $platform_suffix
+    Write-Host "Installing" $filepath "to" $python_home
+    $install_log = $python_home + ".log"
+    $args = "/S /D=$python_home"
+    Write-Host $filepath $args
+    Start-Process -FilePath $filepath -ArgumentList $args -Wait -Passthru
+    if (Test-Path $python_home) {
+        Write-Host "Python $python_version ($architecture) installation complete"
+    } else {
+        Write-Host "Failed to install Python in $python_home"
+        Get-Content -Path $install_log
+        Exit 1
+    }
+}
+
+
+function InstallMinicondaPip ($python_home) {
+    $pip_path = $python_home + "\Scripts\pip.exe"
+    $conda_path = $python_home + "\Scripts\conda.exe"
+    if (-not(Test-Path $pip_path)) {
+        Write-Host "Installing pip..."
+        $args = "install --yes pip"
+        Write-Host $conda_path $args
+        Start-Process -FilePath "$conda_path" -ArgumentList $args -Wait -Passthru
+    } else {
+        Write-Host "pip already installed."
+    }
+}
+
+function main () {
+    InstallPython $env:PYTHON_VERSION $env:PYTHON_ARCH $env:PYTHON
+    InstallPip $env:PYTHON
+}
+
+main

appveyor/run_with_env.cmd

@@ -0,0 +1,87 @@
+:: To build extensions for 64 bit Python 3, we need to configure environment
+:: variables to use the MSVC 2010 C++ compilers from GRMSDKX_EN_DVD.iso of:
+:: MS Windows SDK for Windows 7 and .NET Framework 4 (SDK v7.1)
+::
+:: To build extensions for 64 bit Python 2, we need to configure environment
+:: variables to use the MSVC 2008 C++ compilers from GRMSDKX_EN_DVD.iso of:
+:: MS Windows SDK for Windows 7 and .NET Framework 3.5 (SDK v7.0)
+::
+:: 32 bit builds, and 64-bit builds for 3.5 and beyond, do not require specific
+:: environment configurations.
+::
+:: Note: this script needs to be run with the /E:ON and /V:ON flags for the
+:: cmd interpreter, at least for (SDK v7.0)
+::
+:: More details at:
+:: https://github.com/cython/cython/wiki/64BitCythonExtensionsOnWindows
+:: http://stackoverflow.com/a/13751649/163740
+::
+:: Author: Olivier Grisel
+:: License: CC0 1.0 Universal: http://creativecommons.org/publicdomain/zero/1.0/
+::
+:: Notes about batch files for Python people:
+::
+:: Quotes in values are literally part of the values:
+::      SET FOO="bar"
+:: FOO is now five characters long: " b a r "
+:: If you don't want quotes, don't include them on the right-hand side.
+::
+:: The CALL lines at the end of this file look redundant, but if you move them
+:: outside of the IF clauses, they do not run properly in the SET_SDK_64==Y
+:: case, I don't know why.
+@ECHO OFF
+SET COMMAND_TO_RUN=%*
+SET WIN_SDK_ROOT=C:\Program Files\Microsoft SDKs\Windows
+SET WIN_WDK=c:\Program Files (x86)\Windows Kits\10\Include\wdf
+
+:: Extract the major and minor versions, and allow for the minor version to be
+:: more than 9.  This requires the version number to have two dots in it.
+SET MAJOR_PYTHON_VERSION=%PYTHON_VERSION:~0,1%
+IF "%PYTHON_VERSION:~3,1%" == "." (
+    SET MINOR_PYTHON_VERSION=%PYTHON_VERSION:~2,1%
+) ELSE (
+    SET MINOR_PYTHON_VERSION=%PYTHON_VERSION:~2,2%
+)
+
+:: Based on the Python version, determine what SDK version to use, and whether
+:: to set the SDK for 64-bit.
+IF %MAJOR_PYTHON_VERSION% == 2 (
+    SET WINDOWS_SDK_VERSION="v7.0"
+    SET SET_SDK_64=Y
+) ELSE (
+    IF %MAJOR_PYTHON_VERSION% == 3 (
+        SET WINDOWS_SDK_VERSION="v7.1"
+        IF %MINOR_PYTHON_VERSION% LEQ 4 (
+            SET SET_SDK_64=Y
+        ) ELSE (
+            SET SET_SDK_64=N
+            IF EXIST "%WIN_WDK%" (
+                :: See: https://connect.microsoft.com/VisualStudio/feedback/details/1610302/
+                REN "%WIN_WDK%" 0wdf
+            )
+        )
+    ) ELSE (
+        ECHO Unsupported Python version: "%MAJOR_PYTHON_VERSION%"
+        EXIT 1
+    )
+)
+
+IF %PYTHON_ARCH% == 64 (
+    IF %SET_SDK_64% == Y (
+        ECHO Configuring Windows SDK %WINDOWS_SDK_VERSION% for Python %MAJOR_PYTHON_VERSION% on a 64 bit architecture
+        SET DISTUTILS_USE_SDK=1
+        SET MSSdk=1
+        "%WIN_SDK_ROOT%\%WINDOWS_SDK_VERSION%\Setup\WindowsSdkVer.exe" -q -version:%WINDOWS_SDK_VERSION%
+        "%WIN_SDK_ROOT%\%WINDOWS_SDK_VERSION%\Bin\SetEnv.cmd" /x64 /release
+        ECHO Executing: %COMMAND_TO_RUN%
+        call %COMMAND_TO_RUN% || EXIT 1
+    ) ELSE (
+        ECHO Using default MSVC build environment for 64 bit architecture
+        ECHO Executing: %COMMAND_TO_RUN%
+        call %COMMAND_TO_RUN% || EXIT 1
+    )
+) ELSE (
+    ECHO Using default MSVC build environment for 32 bit architecture
+    ECHO Executing: %COMMAND_TO_RUN%
+    call %COMMAND_TO_RUN% || EXIT 1
+)

bin/pydisassemble

@@ -1,83 +1,3 @@
 #!/usr/bin/env python
-# Mode: -*- python -*-
-#
-# Copyright (c) 2015-2016 by Rocky Bernstein <rb@dustyfeet.com>
-#
-from __future__ import print_function
-import sys, os, getopt
-
-program =  os.path.basename(__file__)
-
-__doc__ = """
-Usage:
-  %s [OPTIONS]... FILE
-  %s [--help | -h | -V | --version]
-
-Examples:
-  %s foo.pyc
-  %s foo.py
-  %s -o foo.pydis foo.pyc
-  %s -o /tmp foo.pyc
-
-Options:
-  -o <path>     output decompiled files to this path:
-                if multiple input files are decompiled, the common prefix
-                is stripped from these names and the remainder appended to
-                <path>
-  --help        show this message
-
-""" % ((program,) * 6)
-
-
-Usage_short = \
-"%s [--help] [--verify] [--showasm] [--showast] [-o <path>] FILE|DIR..." % program
-
-from uncompyle6 import check_python_version
-from uncompyle6.disas import disassemble_files
-from uncompyle6.version import VERSION
-
-check_python_version(program)
-
-outfile = '-'
-out_base = None
-
-
-try:
-    opts, files = getopt.getopt(sys.argv[1:], 'hVo:', ['help', 'version'])
-except getopt.GetoptError as e:
-    print('%s: %s' % (os.path.basename(sys.argv[0]), e),  file=sys.stderr)
-    sys.exit(-1)
-
-for opt, val in opts:
-    if opt in ('-h', '--help'):
-        print(__doc__)
-        sys.exit(1)
-    elif opt in ('-V', '--version'):
-        print("%s %s" % (program, VERSION))
-        sys.exit(0)
-    elif opt == '-o':
-        outfile = val
-    else:
-        print(opt)
-        print(Usage_short)
-        sys.exit(1)
-
-# argl, commonprefix works on strings, not on path parts,
-# thus we must handle the case with files in 'some/classes'
-# and 'some/cmds'
-src_base = os.path.commonprefix(files)
-if src_base[-1:] != os.sep:
-    src_base = os.path.dirname(src_base)
-if src_base:
-    sb_len = len( os.path.join(src_base, '') )
-    files = [f[sb_len:] for f in files]
-    del sb_len
-
-if outfile == '-':
-    outfile = None # use stdout
-elif outfile and os.path.isdir(outfile):
-    out_base = outfile; outfile = None
-elif outfile and len(files) > 1:
-    out_base = outfile; outfile = None
-
-disassemble_files(src_base, out_base, files, outfile)
+from uncompyle6.bin.pydisassemble import main
+main()

bin/uncompyle6

@@ -1,222 +1,3 @@
 #!/usr/bin/env python
-# Mode: -*- python -*-
-#
-# Copyright (c) 2015-2016 by Rocky Bernstein
-# Copyright (c) 2000-2002 by hartmut Goebel <h.goebel@crazy-compilers.com>
-#
-from __future__ import print_function
-import sys, os, getopt, time
-
-program =  os.path.basename(__file__)
-
-__doc__ = """
-Usage:
-  %s [OPTIONS]... [ FILE | DIR]...
-  %s [--help | -h | --V | --version]
-
-Examples:
-  %s      foo.pyc bar.pyc       # decompile foo.pyc, bar.pyc to stdout
-  %s -o . foo.pyc bar.pyc       # decompile to ./foo.pyc_dis and ./bar.pyc_dis
-  %s -o /tmp /usr/lib/python1.5 # decompile whole library
-
-Options:
-  -o <path>     output decompiled files to this path:
-                if multiple input files are decompiled, the common prefix
-                is stripped from these names and the remainder appended to
-                <path>
-                  uncompyle6 -o /tmp bla/fasel.pyc bla/foo.pyc
-                    -> /tmp/fasel.pyc_dis, /tmp/foo.pyc_dis
-                  uncompyle6 -o /tmp bla/fasel.pyc bar/foo.pyc
-                    -> /tmp/bla/fasel.pyc_dis, /tmp/bar/foo.pyc_dis
-                  uncompyle6 -o /tmp /usr/lib/python1.5
-                    -> /tmp/smtplib.pyc_dis ... /tmp/lib-tk/FixTk.pyc_dis
-  -c <file>     attempts a disassembly after compiling <file>
-  -d            print timestamps
-  -p <integer>  use <integer> number of processes
-  -r            recurse directories looking for .pyc and .pyo files
-  --verify      compare generated source with input byte-code
-                (requires -o)
-  --help        show this message
-
-Debugging Options:
-  --asm     -a  include byte-code         (disables --verify)
-  --grammar -g  show matching grammar
-  --tree    -t  include syntax tree       (disables --verify)
-
-Extensions of generated files:
-  '.pyc_dis' '.pyo_dis'   successfully decompiled (and verified if --verify)
-    + '_unverified'       successfully decompile but --verify failed
-    + '_failed'           decompile failed (contact author for enhancement)
-""" % ((program,) * 5)
-
-program =  os.path.basename(__file__)
-
-from uncompyle6 import verify, check_python_version
-from uncompyle6.main import main, status_msg
-from uncompyle6.version import VERSION
-
-def usage():
-    print("""usage:
-   %s [--verify] [--asm] [--tree] [--grammar] [-o <path>] FILE|DIR...
-   %s [--help | -h | --version | -V]
-"""  % (program, program))
-    sys.exit(1)
-
-
-check_python_version(program)
-
-showasm = showast = do_verify = recurse_dirs = False
-numproc = 0
-outfile = '-'
-out_base = None
-codes = []
-timestamp = False
-timestampfmt = "# %Y.%m.%d %H:%M:%S %Z"
-
-try:
-    opts, files = getopt.getopt(sys.argv[1:], 'hagtdrVo:c:p:',
-                                'help asm grammar recurse timestamp tree verify version '
-                                'showgrammar'.split(' '))
-except getopt.GetoptError as e:
-    print('%s: %s' % (os.path.basename(sys.argv[0]), e),  file=sys.stderr)
-    sys.exit(-1)
-
-options = {}
-for opt, val in opts:
-    if opt in ('-h', '--help'):
-        print(__doc__)
-        sys.exit(0)
-    elif opt in ('-V', '--version'):
-        print("%s %s" % (program, VERSION))
-        sys.exit(0)
-    elif opt == '--verify':
-        options['do_verify'] = True
-    elif opt in ('--asm', '-a'):
-        options['showasm'] = True
-        options['do_verify'] = False
-    elif opt in ('--tree', '-t'):
-        options['showast'] = True
-        options['do_verify'] = False
-    elif opt in ('--grammar', '-g'):
-        options['showgrammar'] = True
-    elif opt == '-o':
-        outfile = val
-    elif opt in ('--timestamp', '-d'):
-        timestamp = True
-    elif opt == '-c':
-        codes.append(val)
-    elif opt == '-p':
-        numproc = int(val)
-    elif opt in ('--recurse', '-r'):
-        recurse_dirs = True
-    else:
-        print(opt, file=sys.stderr)
-        usage()
-
-# expand directory if specified
-if recurse_dirs:
-    expanded_files = []
-    for f in files:
-        if os.path.isdir(f):
-            for root, _, dir_files in os.walk(f):
-                for df in dir_files:
-                    if df.endswith('.pyc') or df.endswith('.pyo'):
-                        expanded_files.append(os.path.join(root, df))
-    files = expanded_files
-
-# argl, commonprefix works on strings, not on path parts,
-# thus we must handle the case with files in 'some/classes'
-# and 'some/cmds'
-src_base = os.path.commonprefix(files)
-if src_base[-1:] != os.sep:
-    src_base = os.path.dirname(src_base)
-if src_base:
-    sb_len = len( os.path.join(src_base, '') )
-    files = [f[sb_len:] for f in files]
-    del sb_len
-
-if not files:
-    print("No files given", file=sys.stderr)
-    usage()
-
-
-if outfile == '-':
-    outfile = None # use stdout
-elif outfile and os.path.isdir(outfile):
-    out_base = outfile; outfile = None
-elif outfile and len(files) > 1:
-    out_base = outfile; outfile = None
-
-if timestamp:
-    print(time.strftime(timestampfmt))
-
-if numproc <= 1:
-    try:
-        result = main(src_base, out_base, files, codes, outfile,
-                      **options)
-        if len(files) > 1:
-            mess = status_msg(do_verify, *result)
-            print('# ' + mess)
-            pass
-    except (KeyboardInterrupt):
-        pass
-    except verify.VerifyCmpError:
-        raise
-else:
-    from multiprocessing import Process, Queue
-
-    try:
-        from Queue import Empty
-    except ImportError:
-        from Queue import Empty
-
-    fqueue = Queue(len(files)+numproc)
-    for f in files:
-        fqueue.put(f)
-    for i in range(numproc):
-        fqueue.put(None)
-
-    rqueue = Queue(numproc)
-
-    def process_func():
-        try:
-            (tot_files, okay_files, failed_files, verify_failed_files) = (0, 0, 0, 0)
-            while 1:
-                f = fqueue.get()
-                if f is None:
-                    break
-                (t, o, f, v) = \
-                  main(src_base, out_base, [f], codes, outfile, **options)
-                tot_files += t
-                okay_files += o
-                failed_files += f
-                verify_failed_files += v
-        except (Empty, KeyboardInterrupt):
-            pass
-        rqueue.put((tot_files, okay_files, failed_files, verify_failed_files))
-        rqueue.close()
-
-    try:
-        procs = [Process(target=process_func) for i in range(numproc)]
-        for p in procs:
-            p.start()
-        for p in procs:
-            p.join()
-        try:
-            (tot_files, okay_files, failed_files, verify_failed_files) = (0, 0, 0, 0)
-            while True:
-                (t, o, f, v) = rqueue.get(False)
-                tot_files += t
-                okay_files += o
-                failed_files += f
-                verify_failed_files += v
-        except Empty:
-            pass
-        print('# decompiled %i files: %i okay, %i failed, %i verify failed' %
-              (tot_files, okay_files, failed_files, verify_failed_files))
-    except (KeyboardInterrupt, OSError):
-        pass
-
-
-if timestamp:
-    print(time.strftime(timestampfmt))
+from uncompyle6.bin.uncompile import main_bin
+main_bin()

circle.yml

@@ -1,12 +1,12 @@
 machine:
     python:
-        version: 2.7.8
+        version: 2.7.10
     environment:
         COMPILE: --compile
 
 dependencies:
     override:
-        - pip install -r requirements.txt
+        - pip install -e .
         - pip install -r requirements-dev.txt
 test:
     override:

pytest/.gitignore

@@ -1 +1,2 @@
+/.hypothesis
 /__pycache__

pytest/test_CALL_FUNCTION_KW.sh

@@ -0,0 +1,6 @@
+source ../.venv.3.6/bin/activate
+py.test -k test_CALL_FUNCTION_KW
+source ../.venv.3.5/bin/activate
+py.test -k test_CALL_FUNCTION_KW
+source ../.venv.2.7/bin/activate
+py.test -k test_CALL_FUNCTION_KW

pytest/test_basic.py

@@ -0,0 +1,11 @@
+from uncompyle6.scanner import get_scanner
+from uncompyle6.parser import get_python_parser
+
+def test_get_scanner():
+    # See that we can retrieve a scanner using a full version number
+    assert get_scanner('2.7.13')
+
+
+def test_get_parser():
+    # See that we can retrieve a sparser using a full version number
+    assert get_python_parser('2.7.13')

pytest/test_build_const_key_map.py

@@ -0,0 +1,21 @@
+import pytest
+# uncompyle6
+from uncompyle6 import PYTHON_VERSION
+from validate import validate_uncompyle
+
+
+@pytest.mark.skipif(PYTHON_VERSION < 3.6, reason='need at least python 3.6')
+@pytest.mark.parametrize('text', (
+    "{0.: 'a', -1: 'b'}",   # BUILD_MAP
+    "{'a':'b'}",            # BUILD_MAP
+    "{0: 1}",               # BUILD_MAP
+    "{b'0':1, b'2':3}",     # BUILD_CONST_KEY_MAP
+    "{0: 1, 2: 3}",         # BUILD_CONST_KEY_MAP
+    "{'a':'b','c':'d'}",    # BUILD_CONST_KEY_MAP
+    "{0: 1, 2: 3}",         # BUILD_CONST_KEY_MAP
+    "{'a': 1, 'b': 2}",     # BUILD_CONST_KEY_MAP
+    "{'a':'b','c':'d'}",    # BUILD_CONST_KEY_MAP
+    "{0.0:'b',0.1:'d'}",    # BUILD_CONST_KEY_MAP
+))
+def test_build_const_key_map(text):
+    validate_uncompyle(text)

pytest/test_deparse.py

@@ -20,12 +20,20 @@ def for_range_stmt():
     for i in range(2):
         i+1
 
+# # FIXME: add this test - but for Python 2.7+ only
+# def set_comp():
+#     {y for y in range(3)}
+
+# FIXME: add this test
+def list_comp():
+    [y for y in range(3)]
+
 def get_parsed_for_fn(fn):
     code = fn.__code__ if PYTHON3 else fn.func_code
     return deparse(PYTHON_VERSION, code)
 
 def check_expect(expect, parsed):
-    debug = True
+    debug = False
     i = 2
     max_expect = len(expect)
     for name, offset in sorted(parsed.offsets.keys()):
@@ -160,7 +168,7 @@ return (x, y)
 -------------
 """.split("\n")
     check_expect(expect, parsed)
-#     ########################################################
+    ########################################################
 #     # try
 
 #     expect = """
@@ -291,6 +299,12 @@ return
 Contained in...
     i + 1
     -----
+31
+return
+------
+Contained in...
+for i in range(2): ...
+------------------ ...
 34
 return
 ------

pytest/test_disasm.py

@@ -24,7 +24,7 @@ os.chdir(src_dir)
 def test_funcoutput(capfd, test_tuple, function_to_test):
 
     in_file , filename_expected = test_tuple
-    function_to_test(in_file)
+    function_to_test(in_file, native=False)
     resout, reserr = capfd.readouterr()
     expected = open(filename_expected, "r").read()
     if resout != expected:

pytest/test_docstring.py

@@ -0,0 +1,78 @@
+import sys
+from uncompyle6 import PYTHON3
+if PYTHON3:
+    from io import StringIO
+    minint = -sys.maxsize-1
+    maxint = sys.maxsize
+else:
+    from StringIO import StringIO
+    minint = -sys.maxint-1
+    maxint = sys.maxint
+from uncompyle6.semantics.helper import print_docstring
+
+class PrintFake():
+    def __init__(self):
+        self.pending_newlines = 0
+        self.f = StringIO()
+
+    def write(self, *data):
+        if (len(data) == 0) or (len(data) == 1 and data[0] == ''):
+            return
+        out = ''.join((str(j) for j in data))
+        n = 0
+        for i in out:
+            if i == '\n':
+                n += 1
+                if n == len(out):
+                    self.pending_newlines = max(self.pending_newlines, n)
+                    return
+            elif n:
+                self.pending_newlines = max(self.pending_newlines, n)
+                out = out[n:]
+                break
+            else:
+                break
+
+        if self.pending_newlines > 0:
+            self.f.write('\n'*self.pending_newlines)
+            self.pending_newlines = 0
+
+        for i in out[::-1]:
+            if i == '\n':
+                self.pending_newlines += 1
+            else:
+                break
+
+        if self.pending_newlines:
+            out = out[:-self.pending_newlines]
+        self.f.write(out)
+    def println(self, *data):
+        if data and not(len(data) == 1 and data[0] ==''):
+            self.write(*data)
+        self.pending_newlines = max(self.pending_newlines, 1)
+        return
+    pass
+
+def test_docstring():
+
+    for doc, expect in (
+            ("Now is the time",
+             '  """Now is the time"""'),
+           ("""
+Now is the time
+""",
+            '''  """
+  Now is the time
+  """''')
+
+            # (r'''func placeholder - ' and with ("""\nstring\n  """)''',
+            #  """  r'''func placeholder - ' and with (\"\"\"\nstring\n\"\"\")'''"""),
+            # (r"""func placeholder - ' and with ('''\nstring\n''') and \"\"\"\nstring\n\"\"\" """,
+            # """  r\"\"\"func placeholder - ' and with ('''\nstring\n''') and \"\"\"\nstring\n\"\"\" \"\"\"""")
+            ):
+
+        o = PrintFake()
+        # print(doc)
+        # print(expect)
+        print_docstring(o, '  ', doc)
+        assert expect == o.f.getvalue()

pytest/test_fjt.py

@@ -0,0 +1,77 @@
+#!/usr/bin/env python
+from uncompyle6 import PYTHON_VERSION, IS_PYPY
+from uncompyle6.scanner import get_scanner
+from xdis.bytecode import Bytecode
+from array import array
+def bug(state, slotstate):
+    if state:
+        if slotstate is not None:
+            for key, value in slotstate.items():
+                setattr(state, key, 2)
+
+# From 2.7 disassemble
+# Problem is not getting while, because
+# COME_FROM not added
+def bug_loop(disassemble, tb=None):
+    if tb:
+        try:
+            tb = 5
+        except AttributeError:
+            raise RuntimeError
+        while tb: tb = tb.tb_next
+    disassemble(tb)
+
+def test_if_in_for():
+    code = bug.__code__
+    scan = get_scanner(PYTHON_VERSION)
+    print(PYTHON_VERSION)
+    if 2.7 <= PYTHON_VERSION <= 3.0 and not IS_PYPY:
+        n = scan.setup_code(code)
+        scan.build_lines_data(code, n)
+        scan.build_prev_op(n)
+        fjt = scan.find_jump_targets(False)
+
+        ## FIXME: the data below is wrong.
+        ## we get different results currenty as well.
+        ## We need to probably fix both the code
+        ## and the test below
+        # assert {15: [3], 69: [66], 63: [18]} == fjt
+        # assert scan.structs == \
+        #   [{'start': 0, 'end': 72, 'type': 'root'},
+        #    {'start': 15, 'end': 66, 'type': 'if-then'},
+        #    {'start': 31, 'end': 59, 'type': 'for-loop'},
+        #    {'start': 62, 'end': 63, 'type': 'for-else'}]
+
+        code = bug_loop.__code__
+        n = scan.setup_code(code)
+        scan.build_lines_data(code, n)
+        scan.build_prev_op(n)
+        fjt = scan.find_jump_targets(False)
+        assert{64: [42], 67: [42, 42], 42: [16, 41], 19: [6]} == fjt
+        assert scan.structs == [
+            {'start': 0, 'end': 80, 'type': 'root'},
+            {'start': 3, 'end': 64, 'type': 'if-then'},
+            {'start': 6, 'end': 15, 'type': 'try'},
+            {'start': 19, 'end': 38, 'type': 'except'},
+            {'start': 45, 'end': 67, 'type': 'while-loop'},
+            {'start': 70, 'end': 64, 'type': 'while-else'},
+            # previous bug was not mistaking while-loop for if-then
+            {'start': 48, 'end': 67, 'type': 'while-loop'}]
+
+    elif 3.2 < PYTHON_VERSION <= 3.4:
+        bytecode = Bytecode(code, scan.opc)
+        scan.code = array('B', code.co_code)
+        scan.build_lines_data(code)
+        scan.build_prev_op()
+        scan.insts = list(bytecode)
+        fjt  = scan.find_jump_targets(False)
+        assert {69: [66], 63: [18]} == fjt
+        assert scan.structs == \
+          [{'end': 72, 'type': 'root', 'start': 0},
+           {'end': 66, 'type': 'if-then', 'start': 6},
+           {'end': 63, 'type': 'if-then', 'start': 18},
+           {'end': 59, 'type': 'for-loop', 'start': 31},
+           {'end': 63, 'type': 'for-else', 'start': 62}]
+    else:
+        assert True, "FIXME: should note fixed"
+    return

pytest/test_fstring.py

@@ -0,0 +1,150 @@
+# std
+import os
+# test
+import pytest
+import hypothesis
+from hypothesis import strategies as st
+# uncompyle6
+from uncompyle6 import PYTHON_VERSION, deparse_code
+
+
+@st.composite
+def expressions(draw):
+    # todo : would be nice to generate expressions using hypothesis however
+    # this is pretty involved so for now just use a corpus of expressions
+    # from which to select.
+    return draw(st.sampled_from((
+        'abc',
+        'len(items)',
+        'x + 1',
+        'lineno',
+        'container',
+        'self.attribute',
+        'self.method()',
+        # These expressions are failing, I think these are control
+        # flow problems rather than problems with FORMAT_VALUE,
+        # however I need to confirm this...
+        #'sorted(items, key=lambda x: x.name)',
+        #'func(*args, **kwargs)',
+        #'text or default',
+        #'43 if life_the_universe and everything else None'
+    )))
+
+
+@st.composite
+def format_specifiers(draw):
+    """
+    Generate a valid format specifier using the rules:
+
+    format_spec ::=  [[fill]align][sign][#][0][width][,][.precision][type]
+    fill        ::=  <any character>
+    align       ::=  "<" | ">" | "=" | "^"
+    sign        ::=  "+" | "-" | " "
+    width       ::=  integer
+    precision   ::=  integer
+    type        ::=  "b" | "c" | "d" | "e" | "E" | "f" | "F" | "g" | "G" | "n" | "o" | "s" | "x" | "X" | "%"
+
+    See https://docs.python.org/2/library/string.html
+
+    :param draw: Let hypothesis draw from other strategies.
+
+    :return: An example format_specifier.
+    """
+    alphabet_strategy = st.characters(min_codepoint=ord('a'), max_codepoint=ord('z'))
+    fill = draw(st.one_of(alphabet_strategy, st.none()))
+    align = draw(st.sampled_from(list('<>=^')))
+    fill_align = (fill + align or '') if fill else ''
+
+    type_ = draw(st.sampled_from('bcdeEfFgGnosxX%'))
+    can_have_sign = type_ in 'deEfFgGnoxX%'
+    can_have_comma = type_ in 'deEfFgG%'
+    can_have_precision = type_ in 'fFgG'
+    can_have_pound = type_ in 'boxX%'
+    can_have_zero = type_ in 'oxX'
+
+    sign = draw(st.sampled_from(list('+- ') + [''])) if can_have_sign else ''
+    pound = draw(st.sampled_from(('#', '',))) if can_have_pound else ''
+    zero = draw(st.sampled_from(('0', '',))) if can_have_zero else ''
+
+    int_strategy = st.integers(min_value=1, max_value=1000)
+
+    width = draw(st.one_of(int_strategy, st.none()))
+    width = str(width) if width is not None else ''
+
+    comma = draw(st.sampled_from((',', '',))) if can_have_comma else ''
+    if can_have_precision:
+        precision = draw(st.one_of(int_strategy, st.none()))
+        precision = '.' + str(precision) if precision else ''
+    else:
+        precision = ''
+
+    return ''.join((fill_align, sign, pound, zero, width, comma, precision, type_,))
+
+
+@st.composite
+def fstrings(draw):
+    """
+    Generate a valid f-string.
+    See https://www.python.org/dev/peps/pep-0498/#specification
+
+    :param draw: Let hypothsis draw from other strategies.
+
+    :return: A valid f-string.
+    """
+    character_strategy = st.characters(
+        blacklist_characters='\r\n\'\\s{}',
+        min_codepoint=1,
+        max_codepoint=1000,
+    )
+    is_raw = draw(st.booleans())
+    integer_strategy = st.integers(min_value=0, max_value=3)
+    expression_count = draw(integer_strategy)
+    content = []
+    for _ in range(expression_count):
+        expression = draw(expressions())
+        conversion = draw(st.sampled_from(('', '!s', '!r', '!a',)))
+        has_specifier = draw(st.booleans())
+        specifier = ':' + draw(format_specifiers()) if has_specifier else ''
+        content.append('{{{}{}}}'.format(expression, conversion, specifier))
+        content.append(draw(st.text(character_strategy)))
+    content = ''.join(content)
+    return "f{}'{}'".format('r' if is_raw else '', content)
+
+
+@pytest.mark.skipif(PYTHON_VERSION < 3.6, reason='need at least python 3.6')
+@hypothesis.given(format_specifiers())
+def test_format_specifiers(format_specifier):
+    """Verify that format_specifiers generates valid specifiers"""
+    try:
+        exec('"{:' + format_specifier + '}".format(0)')
+    except ValueError as e:
+        if 'Unknown format code' not in str(e):
+            raise
+
+
+def run_test(text):
+    hypothesis.assume(len(text))
+    hypothesis.assume("f'{" in text)
+    expr = text + '\n'
+    code = compile(expr, '<string>', 'single')
+    deparsed = deparse_code(PYTHON_VERSION, code, compile_mode='single')
+    recompiled = compile(deparsed.text, '<string>', 'single')
+    if recompiled != code:
+        assert 'dis(' + deparsed.text.strip('\n') + ')' == 'dis(' + expr.strip('\n') + ')'
+
+
+@pytest.mark.skipif(PYTHON_VERSION < 3.6, reason='need at least python 3.6')
+@hypothesis.given(fstrings())
+def test_uncompyle_fstring(fstring):
+    """Verify uncompyling fstring bytecode"""
+    run_test(fstring)
+
+
+@pytest.mark.skipif(PYTHON_VERSION < 3.6, reason='need at least python 3.6')
+@pytest.mark.parametrize('fstring', [
+    "f'{abc}{abc!s}'",
+    "f'{abc}0'",
+])
+def test_uncompyle_direct(fstring):
+    """useful for debugging"""
+    run_test(fstring)

pytest/test_function_call.py

@@ -0,0 +1,175 @@
+# std
+import string
+# 3rd party
+from hypothesis import given, assume, example, settings, strategies as st
+import pytest
+# uncompyle
+from validate import validate_uncompyle
+from test_fstring import expressions
+
+
+alpha = st.sampled_from(string.ascii_lowercase)
+numbers = st.sampled_from(string.digits)
+alphanum = st.sampled_from(string.ascii_lowercase + string.digits)
+
+
+@st.composite
+def function_calls(draw,
+                   min_keyword_args=0, max_keyword_args=5,
+                   min_positional_args=0, max_positional_args=5,
+                   min_star_args=0, max_star_args=1,
+                   min_double_star_args=0, max_double_star_args=1):
+    """
+    Strategy factory for generating function calls.
+
+    :param draw: Callable which draws examples from other strategies.
+
+    :return: The function call text.
+    """
+    st_positional_args = st.lists(
+        alpha,
+        min_size=min_positional_args,
+        max_size=max_positional_args
+    )
+    st_keyword_args = st.lists(
+        alpha,
+        min_size=min_keyword_args,
+        max_size=max_keyword_args
+    )
+    st_star_args = st.lists(
+        alpha,
+        min_size=min_star_args,
+        max_size=max_star_args
+    )
+    st_double_star_args = st.lists(
+        alpha,
+        min_size=min_double_star_args,
+        max_size=max_double_star_args
+    )
+
+    positional_args = draw(st_positional_args)
+    keyword_args = draw(st_keyword_args)
+    st_values = st.lists(
+        expressions(),
+        min_size=len(keyword_args),
+        max_size=len(keyword_args)
+    )
+    keyword_args = [
+        x + '=' + e
+        for x, e in
+        zip(keyword_args, draw(st_values))
+    ]
+    star_args = ['*' + x for x in draw(st_star_args)]
+    double_star_args = ['**' + x for x in draw(st_double_star_args)]
+
+    arguments = positional_args + keyword_args + star_args + double_star_args
+    draw(st.randoms()).shuffle(arguments)
+    arguments = ','.join(arguments)
+
+    function_call = 'fn({arguments})'.format(arguments=arguments)
+    try:
+        # TODO: Figure out the exact rules for ordering of positional, keyword,
+        # star args, double star args and in which versions the various
+        # types of arguments are supported so we don't need to check that the
+        # expression compiles like this.
+        compile(function_call, '<string>', 'single')
+    except:
+        assume(False)
+    return function_call
+
+
+def test_function_no_args():
+    validate_uncompyle("fn()")
+
+
+def isolated_function_calls(which):
+    """
+    Returns a strategy for generating function calls, but isolated to 
+    particular types of arguments, for example only positional arguments.
+
+    This can help reason about debugging errors in specific types of function
+    calls.
+
+    :param which: One of 'keyword', 'positional', 'star', 'double_star'
+
+    :return: Strategy for generating an function call isolated to specific
+             argument types.
+    """
+    kwargs = dict(
+        max_keyword_args=0,
+        max_positional_args=0,
+        max_star_args=0,
+        max_double_star_args=0,
+    )
+    kwargs['_'.join(('min', which, 'args'))] = 1
+    kwargs['_'.join(('max', which, 'args'))] = 5 if 'star' not in which else 1
+    return function_calls(**kwargs)
+
+
+with settings(max_examples=25):
+
+    @given(isolated_function_calls('positional'))
+    @example("fn(0)")
+    def test_function_positional_only(expr):
+        validate_uncompyle(expr)
+
+    @given(isolated_function_calls('keyword'))
+    @example("fn(a=0)")
+    def test_function_call_keyword_only(expr):
+        validate_uncompyle(expr)
+
+    @given(isolated_function_calls('star'))
+    @example("fn(*items)")
+    def test_function_call_star_only(expr):
+        validate_uncompyle(expr)
+
+    @given(isolated_function_calls('double_star'))
+    @example("fn(**{})")
+    def test_function_call_double_star_only(expr):
+        validate_uncompyle(expr)
+
+
+@pytest.mark.xfail()
+def test_BUILD_CONST_KEY_MAP_BUILD_MAP_UNPACK_WITH_CALL_BUILD_TUPLE_CALL_FUNCTION_EX():
+    validate_uncompyle("fn(w=0,m=0,**v)")
+
+
+@pytest.mark.xfail()
+def test_BUILD_MAP_BUILD_MAP_UNPACK_WITH_CALL_BUILD_TUPLE_CALL_FUNCTION_EX():
+    validate_uncompyle("fn(a=0,**g)")
+
+
+@pytest.mark.xfail()
+def test_CALL_FUNCTION_EX():
+    validate_uncompyle("fn(*g,**j)")
+
+
+@pytest.mark.xfail()
+def test_BUILD_MAP_CALL_FUNCTION_EX():
+    validate_uncompyle("fn(*z,u=0)")
+
+
+@pytest.mark.xfail()
+def test_BUILD_TUPLE_CALL_FUNCTION_EX():
+    validate_uncompyle("fn(**a)")
+
+
+@pytest.mark.xfail()
+def test_BUILD_MAP_BUILD_TUPLE_BUILD_TUPLE_UNPACK_WITH_CALL_CALL_FUNCTION_EX():
+    validate_uncompyle("fn(b,b,b=0,*a)")
+
+
+@pytest.mark.xfail()
+def test_BUILD_TUPLE_BUILD_TUPLE_UNPACK_WITH_CALL_CALL_FUNCTION_EX():
+    validate_uncompyle("fn(*c,v)")
+
+
+@pytest.mark.xfail()
+def test_BUILD_CONST_KEY_MAP_CALL_FUNCTION_EX():
+    validate_uncompyle("fn(i=0,y=0,*p)")
+
+
+@pytest.mark.skip(reason='skipping property based test until all individual tests are passing')
+@given(function_calls())
+def test_function_call(function_call):
+    validate_uncompyle(function_call)

pytest/test_grammar.py

@@ -0,0 +1,85 @@
+import re
+from uncompyle6 import PYTHON_VERSION, PYTHON3, IS_PYPY # , PYTHON_VERSION
+from uncompyle6.parser import get_python_parser, python_parser
+from uncompyle6.scanner import get_scanner
+
+def test_grammar():
+
+    def check_tokens(tokens, opcode_set):
+        remain_tokens = set(tokens) - opcode_set
+        remain_tokens = set([re.sub('_\d+$','', t) for t in remain_tokens])
+        remain_tokens = set([re.sub('_CONT$','', t) for t in remain_tokens])
+        remain_tokens = set(remain_tokens) - opcode_set
+        assert remain_tokens == set([]), \
+            "Remaining tokens %s\n====\n%s" % (remain_tokens, p.dump_grammar())
+
+    p = get_python_parser(PYTHON_VERSION, is_pypy=IS_PYPY)
+    (lhs, rhs, tokens,
+     right_recursive, dup_rhs) = p.check_sets()
+    expect_lhs = set(['expr1024', 'pos_arg'])
+    unused_rhs = set(['build_list', 'call_function', 'mkfunc',
+                      'mklambda',
+                      'unpack',])
+
+    expect_right_recursive = frozenset([('designList',
+                                         ('designator', 'DUP_TOP', 'designList'))])
+    if PYTHON3:
+        expect_lhs.add('load_genexpr')
+
+        unused_rhs = unused_rhs.union(set("""
+        except_pop_except genexpr classdefdeco2 listcomp
+        """.split()))
+        if 3.0 <= PYTHON_VERSION:
+            expect_lhs.add("annotate_arg")
+            expect_lhs.add("annotate_tuple")
+            unused_rhs.add("mkfunc_annotate")
+            pass
+    else:
+        expect_lhs.add('kwarg')
+
+    # FIXME: grammar remove_rule on
+    #  kv3 ::= expr expr STORE_MAP
+    # doesn't currently work in grammar, so we have this extraneous kv3 around.
+    if 3.3 <= PYTHON_VERSION <= 3.4:
+        expect_lhs.add('kv3')
+
+    assert expect_lhs == set(lhs)
+    assert unused_rhs == set(rhs)
+    assert expect_right_recursive == right_recursive
+
+    expect_dup_rhs = frozenset([('COME_FROM',), ('CONTINUE',), ('JUMP_ABSOLUTE',),
+                                ('LOAD_CONST',),
+                                ('JUMP_BACK',), ('JUMP_FORWARD',)])
+    reduced_dup_rhs = {k: dup_rhs[k] for k in dup_rhs if k not in expect_dup_rhs}
+    for k in reduced_dup_rhs:
+        print(k, reduced_dup_rhs[k])
+    # assert not reduced_dup_rhs, reduced_dup_rhs
+
+    s = get_scanner(PYTHON_VERSION, IS_PYPY)
+    ignore_set = set(
+            """
+            JUMP_BACK CONTINUE
+            COME_FROM COME_FROM_EXCEPT
+            COME_FROM_EXCEPT_CLAUSE
+            COME_FROM_LOOP COME_FROM_WITH
+            COME_FROM_FINALLY ELSE
+            LOAD_GENEXPR LOAD_ASSERT LOAD_SETCOMP LOAD_DICTCOMP
+            LAMBDA_MARKER
+            RETURN_END_IF RETURN_END_IF_LAMBDA RETURN_VALUE_LAMBDA RETURN_LAST
+            """.split())
+    if 2.6 <= PYTHON_VERSION <= 2.7:
+        opcode_set = set(s.opc.opname).union(ignore_set)
+        check_tokens(tokens, opcode_set)
+    elif PYTHON_VERSION == 3.4:
+        ignore_set.add('LOAD_CLASSNAME')
+        ignore_set.add('STORE_LOCALS')
+        opcode_set = set(s.opc.opname).union(ignore_set)
+        check_tokens(tokens, opcode_set)
+
+def test_dup_rule():
+    import inspect
+    python_parser(PYTHON_VERSION, inspect.currentframe().f_code,
+                  is_pypy=IS_PYPY,
+                  parser_debug={
+                      'dups': True, 'transition': False, 'reduce': False,
+                      'rules': False, 'errorstack': None, 'context': True})

pytest/test_load.py

@@ -1,10 +0,0 @@
-import sys
-from uncompyle6.load import load_file, check_object_path, load_module
-
-def test_load():
-    """Basic test of load_file, check_object_path and load_module"""
-    co = load_file(__file__)
-    obj_path = check_object_path(__file__)
-    version, timestamp, magic_int, co2 = load_module(obj_path)
-    assert sys.version[0:3] == str(version)
-    assert co == co2

pytest/test_pysource.py

@@ -0,0 +1,176 @@
+from uncompyle6 import PYTHON3
+from uncompyle6.semantics.consts import (
+    escape, NONE,
+    # RETURN_NONE, PASS, RETURN_LOCALS
+)
+
+if PYTHON3:
+    from io import StringIO
+    def iteritems(d):
+        return d.items()
+else:
+    from StringIO import StringIO
+    def iteritems(d):
+        return d.iteritems()
+
+from uncompyle6.semantics.pysource import SourceWalker as SourceWalker
+
+def test_template_engine():
+    s = StringIO()
+    sw = SourceWalker(2.7, s, None)
+    sw.ast = NONE
+    sw.template_engine(('--%c--', 0), NONE)
+    print(sw.f.getvalue())
+    assert sw.f.getvalue() == '--None--'
+    # FIXME: and so on...
+
+from uncompyle6.semantics.consts import (
+    TABLE_DIRECT, TABLE_R,
+    )
+
+from uncompyle6.semantics.fragments import (
+    TABLE_DIRECT_FRAGMENT,
+    )
+
+skip_for_now = "DELETE_DEREF".split()
+
+def test_tables():
+    for t, name, fragment in (
+            (TABLE_DIRECT, 'TABLE_DIRECT', False),
+            (TABLE_R, 'TABLE_R', False),
+            (TABLE_DIRECT_FRAGMENT, 'TABLE_DIRECT_FRAGMENT', True)):
+        for k, entry in iteritems(t):
+            if k in skip_for_now:
+                continue
+            fmt = entry[0]
+            arg = 1
+            i = 0
+            m = escape.search(fmt)
+            print("%s[%s]" % (name, k))
+            while m:
+                i = m.end()
+                typ = m.group('type') or '{'
+                if typ in frozenset(['%', '+', '-', '|', ',', '{']):
+                    # No args
+                    pass
+                elif typ in frozenset(['c', 'p', 'P', 'C', 'D']):
+                    # One arg - should be int or tuple of int
+                    if typ == 'c':
+                        item = entry[arg]
+                        if isinstance(item, tuple):
+                            assert isinstance(item[1], str), (
+                            "%s[%s][%d] kind %s is '%s' should be str but is %s. "
+                            "Full entry: %s" %
+                            (name, k, arg, typ, item[1], type(item[1]), entry)
+                            )
+                            item = item[0]
+                        assert isinstance(item, int), (
+                            "%s[%s][%d] kind %s is '%s' should be an int but is %s. "
+                            "Full entry: %s" %
+                            (name, k, arg, typ, item, type(item), entry)
+                            )
+                    elif typ in frozenset(['C', 'D']):
+                        tup = entry[arg]
+                        assert isinstance(tup, tuple), (
+                            "%s[%s][%d] type %s is %s should be an tuple but is %s. "
+                            "Full entry: %s" %
+                            (name, k, arg, typ, entry[arg], type(entry[arg]), entry)
+                            )
+                        assert len(tup) == 3
+                        for j, x in enumerate(tup[:-1]):
+                            assert isinstance(x, int), (
+                                "%s[%s][%d][%d] type %s is %s should be an tuple but is %s. "
+                                "Full entry: %s" %
+                                (name, k, arg, j, typ, x, type(x), entry)
+                                )
+                        assert isinstance(tup[-1], str) or tup[-1] is None, (
+                                "%s[%s][%d][%d] sep type %s is %s should be an string but is %s. "
+                                "Full entry: %s" %
+                                (name, k, arg, j, typ, tup[-1], type(x), entry)
+                                )
+
+                    elif typ == 'P':
+                        tup = entry[arg]
+                        assert isinstance(tup, tuple), (
+                            "%s[%s][%d] type %s is %s should be an tuple but is %s. "
+                            "Full entry: %s" %
+                            (name, k, arg, typ, entry[arg], type(entry[arg]), entry)
+                            )
+                        assert len(tup) == 4
+                        for j, x in enumerate(tup[:-2]):
+                            assert isinstance(x, int), (
+                                "%s[%s][%d][%d] type %s is '%s' should be an tuple but is %s. "
+                                "Full entry: %s" %
+                                (name, k, arg, j, typ, x, type(x), entry)
+                                )
+                        assert isinstance(tup[-2], str), (
+                                "%s[%s][%d][%d] sep type %s is '%s' should be an string but is %s. "
+                                "Full entry: %s" %
+                                (name, k, arg, j, typ, x, type(x), entry)
+                                )
+                        assert isinstance(tup[1], int), (
+                                "%s[%s][%d][%d] prec type %s is '%s' should be an int but is %s. "
+                                "Full entry: %s" %
+                                (name, k, arg, j, typ, x, type(x), entry)
+                                )
+
+                    else:
+                        # Should be a tuple which contains only ints
+                        tup = entry[arg]
+                        assert isinstance(tup, tuple), (
+                            "%s[%s][%d] type %s is '%s' should be an tuple but is %s. "
+                            "Full entry: %s" %
+                            (name, k, arg, typ, entry[arg], type(entry[arg]), entry)
+                            )
+                        assert len(tup) == 2
+                        for j, x in enumerate(tup):
+                            assert isinstance(x, int), (
+                                "%s[%s][%d][%d] type '%s' is '%s should be an int but is %s. Full entry: %s" %
+                                (name, k, arg, j, typ, x, type(x), entry)
+                                )
+                        pass
+                    arg += 1
+                elif typ in frozenset(['r']) and fragment:
+                    pass
+                elif typ == 'b' and fragment:
+                    assert isinstance(entry[arg], int), (
+                        "%s[%s][%d] type %s is '%s' should be an int but is %s. "
+                        "Full entry: %s" %
+                        (name, k, arg, typ, entry[arg], type(entry[arg]), entry)
+                        )
+                    arg += 1
+                elif typ == 'x' and fragment:
+                    tup = entry[arg]
+                    assert isinstance(tup, tuple), (
+                        "%s[%s][%d] type %s is '%s' should be an tuple but is %s. "
+                        "Full entry: %s" %
+                        (name, k, arg, typ, entry[arg], type(entry[arg]), entry)
+                            )
+                    assert len(tup) == 2
+                    assert isinstance(tup[0], int), (
+                        "%s[%s][%d] source type %s is '%s' should be an int but is %s. "
+                        "Full entry: %s" %
+                        (name, k, arg, typ, entry[arg], type(entry[arg]), entry)
+                            )
+                    assert isinstance(tup[1], tuple), (
+                        "%s[%s][%d] dest type %s is '%s' should be an tuple but is %s. "
+                        "Full entry: %s" %
+                        (name, k, arg, typ, entry[arg], type(entry[arg]), entry)
+                            )
+                    for j, x in enumerate(tup[1]):
+                        assert isinstance(x, int), (
+                            "%s[%s][%d][%d] type %s is %s should be an int but is %s. Full entry: %s" %
+                            (name, k, arg, j, typ, x, type(x), entry)
+                            )
+                    arg += 1
+                    pass
+                else:
+                    assert False, (
+                        "%s[%s][%d] type %s is not known. Full entry: %s" %
+                            (name, k, arg, typ, entry)
+                        )
+                m = escape.search(fmt, i)
+                pass
+            assert arg == len(entry), (
+                "%s[%s] arg %d should be length of entry %d. Full entry: %s" %
+                            (name, k, arg, len(entry), entry))

pytest/test_single_compile.py

@@ -9,8 +9,6 @@ def test_single_mode():
         'i = j % 4',
         'i = {}',
         'i = []',
-        'while i < 1 or stop:\n    i\n',
-        'while i < 1 or stop:\n    print%s\n' % ('(i)' if PYTHON3 else ' i'),
         'for i in range(10):\n    i\n',
         'for i in range(10):\n    for j in range(10):\n        i + j\n',
         'try:\n    i\nexcept Exception:\n    j\nelse:\n    k\n'

pytest/testdata/if-2.7.right

@@ -1,13 +1,12 @@
 # Python 2.7
 # Embedded file name: simple_source/branching/05_if.py
 
-   6       0	LOAD_NAME         'True'
-           3	POP_JUMP_IF_FALSE '15'
-
-   7       6	LOAD_NAME         'False'
-           9	STORE_NAME        'b'
-          12	JUMP_FORWARD      '15'
-        15_0	COME_FROM         '12'
-          15	LOAD_CONST        ''
-          18	RETURN_VALUE      ''
+   6       0  LOAD_NAME             0  'True'
+           3  POP_JUMP_IF_FALSE    15  'to 15'
 
+   7       6  LOAD_NAME             1  'False'
+           9  STORE_NAME            2  'b'
+          12  JUMP_FORWARD          0  'to 15'
+        15_0  COME_FROM                '12'
+          15  LOAD_CONST            0  ''
+          18  RETURN_VALUE     

pytest/testdata/ifelse-2.7.right

@@ -1,16 +1,15 @@
 # Python 2.7
 # Embedded file name: simple_source/branching/05_ifelse.py
 
-   3       0	LOAD_NAME         'True'
-           3	POP_JUMP_IF_FALSE '15'
+   3       0  LOAD_NAME             0  'True'
+           3  POP_JUMP_IF_FALSE    15  'to 15'
 
-   4       6	LOAD_CONST        1
-           9	STORE_NAME        'b'
-          12	JUMP_FORWARD      '21'
-
-   6      15	LOAD_CONST        2
-          18	STORE_NAME        'd'
-        21_0	COME_FROM         '12'
-          21	LOAD_CONST        ''
-          24	RETURN_VALUE      ''
+   4       6  LOAD_CONST            0  1
+           9  STORE_NAME            1  'b'
+          12  JUMP_FORWARD          6  'to 21'
 
+   6      15  LOAD_CONST            1  2
+          18  STORE_NAME            2  'd'
+        21_0  COME_FROM                '12'
+          21  LOAD_CONST            2  ''
+          24  RETURN_VALUE     

pytest/validate.py

@@ -0,0 +1,149 @@
+# future
+from __future__ import print_function
+# std
+import os
+import difflib
+import subprocess
+import tempfile
+import functools
+# compatability
+import six
+# uncompyle6 / xdis
+from uncompyle6 import PYTHON_VERSION, IS_PYPY, deparse_code
+# TODO : I think we can get xdis to support the dis api (python 3 version) by doing something like this there
+from xdis.bytecode import Bytecode
+from xdis.main import get_opcode
+opc = get_opcode(PYTHON_VERSION, IS_PYPY)
+Bytecode = functools.partial(Bytecode, opc=opc)
+
+
+def _dis_to_text(co):
+    return Bytecode(co).dis()
+
+
+def print_diff(original, uncompyled):
+    """
+    Try and display a pretty html line difference between the original and
+    uncompyled code and bytecode if elinks and BeautifulSoup are installed
+    otherwise just show the diff.
+
+    :param original: Text describing the original code object.
+    :param uncompyled: Text describing the uncompyled code object.
+    """
+    original_lines = original.split('\n')
+    uncompyled_lines = uncompyled.split('\n')
+    args = original_lines, uncompyled_lines, 'original', 'uncompyled'
+    try:
+        from bs4 import BeautifulSoup
+        diff = difflib.HtmlDiff().make_file(*args)
+        diff = BeautifulSoup(diff, "html.parser")
+        diff.select_one('table[summary="Legends"]').extract()
+    except ImportError:
+        print('\nTo display diff highlighting run:\n    pip install BeautifulSoup4')
+        diff = difflib.HtmlDiff().make_table(*args)
+
+    with tempfile.NamedTemporaryFile(delete=False) as f:
+        f.write(str(diff).encode('utf-8'))
+
+    try:
+        print()
+        html = subprocess.check_output([
+            'elinks',
+            '-dump',
+            '-no-references',
+            '-dump-color-mode',
+            '1',
+            f.name,
+        ]).decode('utf-8')
+        print(html)
+    except:
+        print('\nFor side by side diff install elinks')
+        diff = difflib.Differ().compare(original_lines, uncompyled_lines)
+        print('\n'.join(diff))
+    finally:
+        os.unlink(f.name)
+
+
+def are_instructions_equal(i1, i2):
+    """
+    Determine if two instructions are approximately equal,
+    ignoring certain fields which we allow to differ, namely:
+
+    * code objects are ignore (should probaby be checked) due to address
+    * line numbers
+
+    :param i1: left instruction to compare
+    :param i2: right instruction to compare
+
+    :return: True if the two instructions are approximately equal, otherwise False.
+    """
+    result = (1==1
+        and i1.opname == i2.opname
+        and i1.opcode == i2.opcode
+        and i1.arg == i2.arg
+        # ignore differences due to code objects
+        # TODO : Better way of ignoring address
+        and (i1.argval == i2.argval or '<code object' in str(i1.argval))
+        # TODO : Should probably recurse to check code objects
+        and (i1.argrepr == i2.argrepr or '<code object' in i1.argrepr)
+        and i1.offset == i2.offset
+        # ignore differences in line numbers
+        #and i1.starts_line
+        and i1.is_jump_target == i2.is_jump_target
+    )
+    return result
+
+
+def are_code_objects_equal(co1, co2):
+    """
+    Determine if two code objects are approximately equal,
+    see are_instructions_equal for more information.
+
+    :param i1: left code object to compare
+    :param i2: right code object to compare
+
+    :return: True if the two code objects are approximately equal, otherwise False.
+    """
+    instructions1 = Bytecode(co1)
+    instructions2 = Bytecode(co2)
+    for opcode1, opcode2 in zip(instructions1, instructions2):
+        if not are_instructions_equal(opcode1, opcode2):
+            return False
+    return True
+
+
+def validate_uncompyle(text, mode='exec'):
+    """
+    Validate decompilation of the given source code.
+
+    :param text: Source to validate decompilation of.
+    """
+    original_code = compile(text, '<string>', mode)
+    original_dis = _dis_to_text(original_code)
+    original_text = text
+
+    deparsed = deparse_code(PYTHON_VERSION, original_code,
+                            compile_mode=mode,
+                            out=six.StringIO(),
+                            is_pypy=IS_PYPY)
+    uncompyled_text = deparsed.text
+    uncompyled_code = compile(uncompyled_text, '<string>', 'exec')
+
+    if not are_code_objects_equal(uncompyled_code, original_code):
+
+        uncompyled_dis = _dis_to_text(uncompyled_text)
+
+        def output(text, dis):
+            width = 60
+            return '\n\n'.join([
+                ' SOURCE CODE '.center(width, '#'),
+                text.strip(),
+                ' BYTECODE '.center(width, '#'),
+                dis
+            ])
+
+        original = output(original_text, original_dis)
+        uncompyled = output(uncompyled_text, uncompyled_dis)
+        print_diff(original, uncompyled)
+
+        assert 'original' == 'uncompyled'

requirements-dev.txt

@@ -1,2 +1,3 @@
-pytest
+pytest>=3.0.0
 flake8
+hypothesis

requirements.txt

@@ -1 +1,2 @@
-spark_parser >= 1.1.0
+# Pick up stuff from setup.py
+-e .

setup.py

@@ -3,30 +3,27 @@
 """Setup script for the 'uncompyle6' distribution."""
 
 from __pkginfo__ import \
-    author,           author_email,                                  \
-    license,          long_description,                              \
-    modname,          packages,           py_modules,       scripts, \
-    short_desc,       web,                zip_safe
-
-__import__('pkg_resources')
-from setuptools import setup
-
-exec(open('uncompyle6/version.py').read())
+    author,           author_email,       install_requires,          \
+    license,          long_description,   classifiers,               \
+    entry_points,     modname,            py_modules,                \
+    short_desc,       VERSION,            web,                       \
+    zip_safe
 
+from setuptools import setup, find_packages
 setup(
        author             = author,
        author_email       = author_email,
-       # classifiers        = classifiers,
+       classifiers        = classifiers,
        description        = short_desc,
-       # install_requires   = install_requires,
+       entry_points       = entry_points,
+       install_requires   = install_requires,
        license            = license,
        long_description   = long_description,
-       py_modules         = py_modules,
        name               = modname,
-       packages           = packages,
+       packages           = find_packages(),
+       py_modules         = py_modules,
        test_suite         = 'nose.collector',
        url                = web,
-       setup_requires     = ['nose>=1.0'],
-       scripts            = scripts,
+       tests_require      = ['nose>=1.0'],
        version            = VERSION,
        zip_safe           = zip_safe)

test/Makefile

@@ -3,11 +3,12 @@ PHONY=check clean dist distclean test test-unit test-functional rmChangeLog clea
 GIT2CL ?= git2cl
 PYTHON ?= python
 
-PYTHON_VERSION = $(shell $(PYTHON) -V | cut -d ' ' -f 2 | cut -d'.' -f1,2)
+PYTHON_VERSION = $(shell $(PYTHON) -V 2>&1 | cut -d ' ' -f 2 | cut -d'.' -f1,2)
 NATIVE_CHECK = check-$(PYTHON_VERSION)
 
 # Set COMPILE='--compile' to force compilation before check
 COMPILE ?=
+COVER_DIR=../tmp/grammar-cover
 
 # Run short tests
 check-short:
@@ -16,52 +17,148 @@ check-short:
 
 # Run all tests
 check:
-	@$(PYTHON) -V && PYTHON_VERSION=`$(PYTHON) -V 2>&1 | cut -d ' ' -f 2 | cut -d'.' -f1,2`; \
-	$(MAKE) check-$$PYTHON_VERSION
+	$(MAKE) check-$(PYTHON_VERSION)
 
 #: Run working tests from Python 2.6 or 2.7
-check-2.6 check-2.7: check-bytecode-sans-3.5 check-2.7-ok
+check-2.6 check-2.7: check-bytecode-2 check-bytecode-3 check-bytecode-1 check-native-short
+
+#: Run working tests from Python 3.0
+check-3.0: check-bytecode
+	$(PYTHON) test_pythonlib.py --bytecode-3.0 --weak-verify $(COMPILE)
+
+#: Run working tests from Python 3.1
+check-3.1: check-bytecode
+	$(PYTHON) test_pythonlib.py --bytecode-3.1 --weak-verify $(COMPILE)
+
+#: Run working tests from Python 3.2
+check-3.2: check-bytecode
+	$(PYTHON) test_pythonlib.py --bytecode-3.2 --weak-verify $(COMPILE)
 
 #: Run working tests from Python 3.3
 check-3.3: check-bytecode
-	$(PYTHON) test_pythonlib.py --bytecode-3.3 --verify $(COMPILE)
-
-#: Run working tests from Python 3.5
-check-3.5: check-bytecode
+	$(PYTHON) test_pythonlib.py --bytecode-3.3 --weak-verify $(COMPILE)
 
 #: Run working tests from Python 3.4
 check-3.4: check-bytecode check-3.4-ok check-2.7-ok
-	$(PYTHON) test_pythonlib.py --bytecode-3.4 --verify $(COMPILE)
+	$(PYTHON) test_pythonlib.py --bytecode-3.4 --weak-verify $(COMPILE)
+
+#: Run working tests from Python 3.5
+check-3.5: check-bytecode
+	$(PYTHON) test_pythonlib.py --bytecode-3.5 --verify $(COMPILE)
+
+#: Run working tests from Python 3.6
+check-3.6: check-bytecode
+	$(PYTHON) test_pythonlib.py --bytecode-3.6 --weak-verify $(COMPILE)
+
+# FIXME
+#: this is called when running under pypy3.5-5.8.0 or pypy2-5.6.0
+5.8 5.6:
 
 #: Check deparsing only, but from a different Python version
 check-disasm:
 	$(PYTHON) dis-compare.py
 
-#: Check deparsing bytecode only
+#: Check deparsing bytecode 1.x only
+check-bytecode-1:
+	$(PYTHON) test_pythonlib.py --bytecode-1.5
+
+#: Check deparsing bytecode 2.x only
 check-bytecode-2:
-	$(PYTHON) test_pythonlib.py --bytecode-2.5 --bytecode-2.6 --bytecode-2.7
+	$(PYTHON) test_pythonlib.py \
+	--bytecode-2.1 --bytecode-2.2 --bytecode-2.3 --bytecode-2.4 \
+        --bytecode-2.5 --bytecode-2.6 --bytecode-2.7 --bytecode-pypy2.7
 
-#: Check deparsing bytecode only
-check-bytecode:
-	$(PYTHON) test_pythonlib.py --bytecode-2.5 --bytecode-2.6 --bytecode-2.7 \
-        --bytecode-3.2 --bytecode-3.3 --bytecode-3.4 --bytecode-3.5
+#: Check deparsing bytecode 3.x only
+check-bytecode-3:
+	$(PYTHON) test_pythonlib.py --bytecode-3.0 \
+       --bytecode-3.1 --bytecode-3.2 --bytecode-3.3 \
+       --bytecode-3.4 --bytecode-3.5 --bytecode-3.6 --bytecode-pypy3.2
 
-#: Check deparsing bytecode only
-check-bytecode-sans-3.5:
-	$(PYTHON) test_pythonlib.py --bytecode-2.5 --bytecode-2.6 --bytecode-2.7 \
-        --bytecode-3.2 --bytecode-3.3 --bytecode-3.4
+#: Check deparsing bytecode that works running Python 2 and Python 3
+check-bytecode: check-bytecode-3
+	$(PYTHON) test_pythonlib.py \
+	--bytecode-2.1 --bytecode-2.2 --bytecode-2.3 --bytecode-2.4 \
+        --bytecode-2.5 --bytecode-2.6 --bytecode-2.7 \
+        --bytecode-pypy2.7 --bytecode-1
+
+
+#: Check deparsing Python 2.1
+check-bytecode-2.1:
+	$(PYTHON) test_pythonlib.py --bytecode-2.1
+
+#: Check deparsing Python 2.2
+check-bytecode-2.2:
+	$(PYTHON) test_pythonlib.py --bytecode-2.2
+
+#: Check deparsing Python 2.3
+check-bytecode-2.3:
+	$(PYTHON) test_pythonlib.py --bytecode-2.3
+
+#: Check deparsing Python 2.4
+check-bytecode-2.4:
+	$(PYTHON) test_pythonlib.py --bytecode-2.4
 
 #: Check deparsing Python 2.5
 check-bytecode-2.5:
 	$(PYTHON) test_pythonlib.py --bytecode-2.5
 
+#: Get grammar coverage for Python 2.5
+grammar-coverage-2.5:
+	-rm $(COVER_DIR)/spark-grammar-25.cover
+	SPARK_PARSER_COVERAGE=$(COVER_DIR)/spark-grammar-25.cover $(PYTHON) test_pythonlib.py --bytecode-2.5
+	SPARK_PARSER_COVERAGE=$(COVER_DIR)/spark-grammar-25.cover $(PYTHON) test_pyenvlib.py --2.5.6
+
+#: Get grammar coverage for Python 2.6
+grammar-coverage-2.6:
+	-rm $(COVER_DIR)/spark-grammar-26.cover
+	SPARK_PARSER_COVERAGE=$(COVER_DIR)/spark-grammar-26.cover $(PYTHON) test_pythonlib.py --bytecode-2.6
+	SPARK_PARSER_COVERAGE=$(COVER_DIR)/spark-grammar-26.cover $(PYTHON) test_pyenvlib.py --2.6.9
+
+#: Get grammar coverage for Python 2.7
+grammar-coverage-2.7:
+	-rm $(COVER_DIR)/spark-grammar-27.cover
+	SPARK_PARSER_COVERAGE=$(COVER_DIR)/spark-grammar-27.cover $(PYTHON) test_pythonlib.py --bytecode-2.7
+	SPARK_PARSER_COVERAGE=$(COVER_DIR)/spark-grammar-27.cover $(PYTHON) test_pyenvlib.py --2.7.13
+
+#: Get grammar coverage for Python 3.2
+grammar-coverage-3.2:
+	-rm $(COVER_DIR)/spark-grammar-32.cover
+	SPARK_PARSER_COVERAGE=$(COVER_DIR)/spark-grammar-32.cover $(PYTHON) test_pythonlib.py --bytecode-3.2
+	SPARK_PARSER_COVERAGE=$(COVER_DIR)/spark-grammar-32.cover $(PYTHON) test_pyenvlib.py --3.2.6
+
+#: Get grammar coverage for Python 3.3
+grammar-coverage-3.3:
+	-rm $(COVER_DIR)/spark-grammar-33.cover
+	SPARK_PARSER_COVERAGE=$(COVER_DIR)/spark-grammar-33.cover $(PYTHON) test_pythonlib.py --bytecode-3.3
+	SPARK_PARSER_COVERAGE=$(COVER_DIR)/spark-grammar-33.cover $(PYTHON) test_pyenvlib.py --3.3.6
+
+##: Get grammar coverage for Python 3.4
+grammar-coverage-3.4:
+	-rm $(COVER_DIR)/spark-grammar-34.cover
+	SPARK_PARSER_COVERAGE=$(COVER_DIR)/spark-grammar-34.cover $(PYTHON) test_pythonlib.py --bytecode-3.4
+	SPARK_PARSER_COVERAGE=$(COVER_DIR)/spark-grammar-34.cover $(PYTHON) test_pyenvlib.py --3.4.2
+
+##: Get grammar coverage for Python 3.5
+grammar-coverage-3.5:
+	rm $(COVER_DIR)/spark-grammar-35.cover || /bin/true
+	SPARK_PARSER_COVERAGE=$(COVER_DIR)/spark-grammar-35.cover $(PYTHON) test_pythonlib.py --bytecode-3.5
+	SPARK_PARSER_COVERAGE=$(COVER_DIR)/spark-grammar-35.cover $(PYTHON) test_pyenvlib.py --3.5.2
+
 #: Check deparsing Python 2.6
-check-bytecode-2.6:
-	$(PYTHON) test_pythonlib.py --bytecode-2.6
+pcheck-bytecode-2.6:
+	$(PYTHON) test_pythonlib.py --bytecode-2.6 --weak-verify
 
 #: Check deparsing Python 2.7
 check-bytecode-2.7:
-	$(PYTHON) test_pythonlib.py --bytecode-2.7
+	$(PYTHON) test_pythonlib.py --bytecode-2.7 --weak-verify
+
+#: Check deparsing Python 3.0
+check-bytecode-3.0:
+	$(PYTHON) test_pythonlib.py --bytecode-3.0
+
+#: Check deparsing Python 3.1
+check-bytecode-3.1:
+	$(PYTHON) test_pythonlib.py --bytecode-3.1
 
 #: Check deparsing Python 3.2
 check-bytecode-3.2:
@@ -73,15 +170,23 @@ check-bytecode-3.3:
 
 #: Check deparsing Python 3.4
 check-bytecode-3.4:
-	$(PYTHON) test_pythonlib.py --bytecode-3.4
+	$(PYTHON) test_pythonlib.py --bytecode-3.4 --weak-verify
 
 #: Check deparsing Python 3.5
 check-bytecode-3.5:
 	$(PYTHON) test_pythonlib.py --bytecode-3.5
 
+#: Check deparsing Python 3.6
+check-bytecode-3.6:
+	$(PYTHON) test_pythonlib.py --bytecode-3.6
+
 #: short tests for bytecodes only for this version of Python
 check-native-short:
-	$(PYTHON) test_pythonlib.py --bytecode-$(PYTHON_VERSION) --verify $(COMPILE)
+	$(PYTHON) test_pythonlib.py --bytecode-$(PYTHON_VERSION) --weak-verify $(COMPILE)
+
+#: Run longer Python 2.6's lib files known to be okay
+check-2.6-ok:
+	$(PYTHON) test_pythonlib.py --ok-2.6 --weak-verify $(COMPILE)
 
 #: Run longer Python 2.7's lib files known to be okay
 check-2.7-ok:
@@ -95,6 +200,18 @@ check-3.2-ok:
 check-3.4-ok:
 	$(PYTHON) test_pythonlib.py --ok-3.4 --verify $(COMPILE)
 
+#: PyPy of some sort. E.g. [PyPy 5.0.1 with GCC 4.8.4]
+# Skip for now
+2.6:
+
+#: PyPy 5.0.x with Python 2.7 ...
+pypy-2.7 5.0 5.3:
+	$(PYTHON) test_pythonlib.py --bytecode-pypy2.7 --verify
+
+#: PyPy 2.4.x with Python 3.2 ...
+pypy-3.2 2.4:
+	$(PYTHON) test_pythonlib.py --bytecode-pypy3.2 --verify
+
 clean: clean-py-dis clean-dis clean-unverified
 
 clean-dis:
@@ -103,6 +220,6 @@ clean-dis:
 clean-unverified:
 	find . -name '*_unverified' -exec rm -v '{}' ';'
 
-#: Clean temporary compile/decompile/verify direcotries in /tmp
+#: Clean temporary compile/decompile/verify directories in /tmp
 clean-py-dis:
 	rm -fr /tmp/py-dis-* || true

test/add-test.py

@@ -2,11 +2,15 @@
 """ Trivial helper program to bytecompile and run an uncompile
 """
 import os, sys, py_compile
-assert len(sys.argv) == 2
-path = sys.argv[1]
-short = os.path.basename(path)
+assert len(sys.argv) >= 2
 version = sys.version[0:3]
-cfile =  "bytecode_%s/%s" % (version, short) + 'c'
-print("byte-compiling %s to %s" % (path, cfile))
-py_compile.compile(path, cfile)
-os.system("../bin/uncompyle6 -a -t %s" % cfile)
+for path in sys.argv[1:]:
+    short = os.path.basename(path)
+    if hasattr(sys, 'pypy_version_info'):
+        cfile =  "bytecode_pypy%s/%s" % (version, short) + 'c'
+    else:
+        cfile =  "bytecode_%s/%s" % (version, short) + 'c'
+    print("byte-compiling %s to %s" % (path, cfile))
+    py_compile.compile(path, cfile)
+    if isinstance(version, str) or version >= (2, 6, 0):
+        os.system("../bin/uncompyle6 -a -t %s" % cfile)