Dependencies stay within 2nd semantic level

Start to localize make_function routines by Python version

.gitignore

@@ -1,5 +1,7 @@
+*.pyc
 *_dis
 *~
+*.pyc
 /.cache
 /.eggs
 /.python-version
@@ -8,6 +10,7 @@
 /__pkginfo__.pyc
 /dist
 /how-to-make-a-release.txt
+/nose-*.egg
 /tmp
 /uncompyle6.egg-info
 __pycache__

.travis.yml

@@ -3,10 +3,11 @@ language: python
 sudo: false
 
 python:
-  - '2.6'
-  - '2.7'
-  - '3.4'
   - '3.5'
+  - '2.7.12'
+  - '2.6'
+  - '3.3'
+  - '3.4'
 
 install:
 - pip install -r requirements.txt
@@ -14,3 +15,8 @@ install:
 
 script:
 - python ./setup.py develop && COMPILE='--compile' make check
+
+# blacklist
+branches:
+  except:
+    - data-driven-pytest

DECOMPYLE-2.4-CHANGELOG.txt

@@ -0,0 +1,131 @@
+This is the changelog from *decompyle*'s release 2.4 and before
+passed on by Dan Pascu
+
+
+release 2.4 (Dan Pascu)
+  - Replaced the way code structures are identified by the parser.
+    Previously, the scanner introduced some COME_FROM entries in the
+    dissasembly output to mark all the destinations of jump instructions.
+    Using these COME_FROM labels the parser was then able to identify the
+    code structures (if tests, while loops, etc). Up to python-2.3 this was
+    possible because the code structures were clearly defined and jump
+    targets were always to the same points in a given strcuture making it
+    easy to identify the structure. Python 2.3 however introduced optimized
+    jumps to increase code performance. In the previous version of decompyle
+    (2.3) we used a technique to identify the code structures and then used
+    these structures to determine where the jump targets would have been if
+    not optimized. Using this information we then added COME_FROM labels at
+    the points where they would have been if not optimized, thus emulating
+    the way decompyle worked with versions before python 2.3. However with
+    the introduction of even more optimizations in python 2.4 this technique
+    no longer works. Not only the jump targets are no longer an effective
+    mean for the parser to identify the code structures, but also trying to
+    emulate the old way things were solved when it clearly no longer works
+    is not the right solution. To solve this issue, the code to identify the
+    structures that we had developed in version 2.3, was used to add real
+    start/end points for strcuture identification, instead of the COME_FROM
+    labels. Now these new start/end labels are used by the parser to more
+    precisely identify the structures and the COME_FROM labels were removed
+    completely. The scanner is responsible to identify these code structures
+    and use any knowledge of optimizations that python applies to determine
+    the start/end points of any structure and then mark them with certain
+    keywords that are understood by the parser.
+  - Correctly identify certain `while 1' structures that were not
+    recognized in the previous version.
+  - Added support for new byte code constructs used by python 2.4
+
+release 2.3.2
+  - tidied up copyright and changelog information for releases 2.3 and later
+
+release 2.3.1 (Dan Pascu)
+  - implemented a structure detection technique that fixes problems with
+    optimised jumps in Python >= 2.3. In the previous release (decompyle 2.3),
+    these problems meant that some files were incorrectly decompiled and
+    others could not be decompiled at all. With this new structure detection
+    technique, thorough testing over the standard python libraries suggests
+    that decompyle 2.3.1 can handle everything that decompyle 2.2beta1 could,
+    plus new Python 2.3 bytecodes and constructs.
+
+release 2.3 (Dan Pascu)
+  - support for Python 2.3 added
+  - use the marshal and disassembly code from their respective python
+    versions, so that decompyle can manipulate bytecode independently
+    of the interpreter that runs decompyle itself (for example it can
+    decompile python2.3 bytecode even when running under python2.2)
+
+——————————————————
+
+release 2.2beta1 (hartmut Goebel)
+  - support for Python 1.5 up to Python 2.2
+  - no longer requires to be run with the Python interpreter version
+    which generated the byte-code.
+  - requires Python 2.2
+  - pretty-prints docstrings, hashes, lists and tuples
+  - decompyle is now a script and a package
+  - added emacs mode-hint and tab-width for each file output
+  - enhanced test suite: more test patterns, .pyc/.pyo included
+  - avoids unnecessary 'global' statements
+  - still untested: EXTENDED_ARG
+
+  internal changes:
+  - major code overhoul: splitted into several modules, clean-ups
+  - use a list of valid magics instead of the single one from imp.py
+  - uses copies of 'dis.py' for every supported version. This ensures
+    correct disassemling of the byte-code.
+  - use a single Walker and a single Parser, thus saving time and memory
+  - use augmented assign and 'print >>' internally
+  - optimized 'Walker.engine', the main part of code generation
+release 0.6.0: (hartmut Goebel)
+  - extended print (Python 2.0)
+  - extended import (Python 2.0) (may not cover all cases)
+  - augmented assign (Python 2.0) (may not cover all cases)
+  - list comprehensions (Python 2.0)
+  - equivalent for 'apply' (Python 1.6)
+  - if .. elif .. else are now nested as expected
+  - assert test, data
+  - unpack list corrected (was the same as unpack tuple)
+  - fixed unpack tuple (trailing semicolon was missing)
+  - major speed up :-)
+  - reduced memory usage (pre-alpha-0.5 has increased it a lot)
+  - still missing: EXTENDED_ARG
+
+pre-alpha-0.5: (hartmut Goebel)
+  - *args, **kwargs
+  - global
+  - formal tuple parameters (eg. def a(self, (x,y,z)) )
+  - actual lambda parameters (eg.  X(lambda z: z**2) )
+  - remove last 'return None' in procedures
+  - remove last 'return locals()' in class definitions
+  - docstrings
+
+pre-alpha-0.4: (hartmut Goebel)
+  - assert
+  - try/except/finally
+  - parentheses in expressions
+  - nested expressions
+  - extracted dissassemble() from module dis and
+    removed ugly redirect of stdout, thus saved a lot of
+    ugly code and a lot of memory
+
+pre-alpha-0.3: (hartmut Goebel)
+  - keyword arguments
+  - some boolean expressions
+  - and/or
+  - complex conditions in if/while
+  - read byte-code from .pyc without importing
+  - access to the body of classes and modules
+  - class and function definitions
+  - a = b = c = xxx
+
+pre-alpha-0.1 -> pre-alpha-0.2:
+  -  SET_LINENO filtered out in lexer now
+  -  added support for subscripts (just for Christian Tismer :-)
+  -  fixed bug with handling of BUILD_{LIST,TUPLE} & CALL_FUNCTION
+  -  dict-building support
+  -  comparison support
+  -  exec support
+  -  del support
+  -  pass support
+  -  slice support
+  -  no more extraneous (albeit legal) commas
+  -  finally, it excepts try [sic] but not all 42 variations of it

HISTORY.md

@@ -4,7 +4,8 @@ There have been a number of people who have worked on this. I am awed
 by the amount of work, number of people who have contributed to this,
 and the cleverness in the code.
 
-The below is an annotated history from my reading of the sources cited.
+The below is an annotated history from talking to participants
+involved and my reading of the code and sources cited.
 
 In 1998, John Aycock first wrote a grammar parser in Python,
 eventually called SPARK, that was usable inside a Python program. This
@@ -23,10 +24,14 @@ working on his thesis, John realized SPARK could be used to deparse
 Python bytecode. In the fall of 1999, he started writing the Python
 program, "decompyle", to do this.
 
-This code introduced another clever idea: using table-driven
-semantics routines, using format specifiers.
+To help with control structure deparsing the instruction sequence was
+augmented with pseudo instruction COME_FROM. This code introduced
+another clever idea: using table-driven semantics routines, using
+format specifiers.
 
-The last mention of a release of SPARK from John is around 2002.
+The last mention of a release of SPARK from John is around 2002. As
+released, although the Early Algorithm parser was in good shape, this
+code was woefully lacking as serious Python deparser.
 
 In the fall of 2000, Hartmut Goebel
 [took over maintaining the code](https://groups.google.com/forum/#!searchin/comp.lang.python/hartmut$20goebel/comp.lang.python/35s3mp4-nuY/UZALti6ujnQJ). The
@@ -36,7 +41,7 @@ first subsequent public release announcement that I can find is
 From the CHANGES file found in
 [the tarball for that release](http://old-releases.ubuntu.com/ubuntu/pool/universe/d/decompyle2.2/decompyle2.2_2.2beta1.orig.tar.gz),
 it appears that Hartmut did most of the work to get this code to
-accept the full Python language. He added precidence to the table
+accept the full Python language. He added precedence to the table
 specifiers, support for multiple versions of Python, the
 pretty-printing of docstrings, lists, and hashes. He also wrote test and verification routines of
 deparsed bytecode, and used this in an extensive set of tests that he also wrote. He could verify against the entire Python library.
@@ -55,32 +60,72 @@ it doesn't look like he's done anything compiler-wise since SPARK). So
 I hope people will use the crazy-compilers service. I wish them the
 success that his good work deserves.
 
-Next we get to
-["uncompyle" and PyPI](https://pypi.python.org/pypi/uncompyle/1.1) and
-the era of git repositories. In contrast to decompyle, this now runs
-only on Python 2.7 although it accepts bytecode back to Python
+Dan Pascu did a bit of work from late 2004 to early 2006 to get this
+code to handle first Python 2.3 and then 2.4 bytecodes. Because of
+jump optimization introduced in the CPython bytecode compiler at that
+time, various JUMP instructions were classifed as going backwards, and
+COME FROM instructions were reintroduced.  See
+RELEASE-2.4-CHANGELOG.txt for more details here. There wasn't a public
+release of RELEASE-2.4 and bytecodes other than Python 2.4 weren't
+supported. Dan says the Python 2.3 version could verify the entire
+python library.
+
+Next we get to ["uncompyle" and
+PyPI](https://pypi.python.org/pypi/uncompyle/1.1) and the era of
+public version control. (Dan's code although not public used
+[darcs](http://darcs.net/) for version control.)
+
+In contrast to _decompyle_, _uncompyle_ at least in its final versions,
+runs only on Python 2.7. However it accepts bytecode back to Python
 2.5. Thomas Grainger is the package owner of this, although Hartmut is
-listed as the author.
+still listed as the author.
 
 The project exists not only on
 [github](https://github.com/gstarnberger/uncompyle) but also on
-[bitbucket](https://bitbucket.org/gstarnberger/uncompyle) where the
-git history goes back to 2009. Somewhere in there the name was changed
-from "decompyle" to "uncompyle".
+[bitbucket](https://bitbucket.org/gstarnberger/uncompyle) and later
+the defunct [google
+code](https://code.google.com/archive/p/unpyc/). The git/svn history
+goes back to 2009. Somewhere in there the name was changed from
+"decompyle" to "unpyc" by Keknehv, and then to "uncompyle" by Guenther Starnberger.
 
 The name Thomas Grainger isn't found in (m)any of the commits in the
-several years of active development. Guenther Starnberger, Keknehv,
-hamled, and Eike Siewertsen are principle committers here.
+several years of active development. First Keknehv worked on this up
+to Python 2.5 or so while acceping Python bytecode back to 2.0 or
+so. Then hamled made a few commits earler on, while Eike Siewertsen
+made a few commits later on. But mostly wibiti, and Guenther
+Starnberger got the code to where uncompyle2 was around 2012.
+
+In uncompyle2 decompilation of python bytecode 2.5 & 2.6 is done by
+transforming the byte code into a a pseudo 2.7 python bytecode and is
+based on code from Eloi Vanderbeken.
+
+This project, uncompyle6, abandons that approach for various
+reasons. However the main reason is that we need offsets in fragment
+deparsing to be exactly the same, and the transformation process can
+remove instructions.  Adding instructions with psuedo_offsets is
+however okay.
+
+Uncompyle6, however owes its existence to the fork of uncompyle2 by
+Myst herie (Mysterie) whose first commit picks up at
+2012. I chose this since it seemed to have been at that time the most
+actively, if briefly, worked on. Also starting around 2012 is Dark
+Fenx's uncompyle3 which I used for inspiration for Python3 support.
+
+I started working on this late 2015, mostly to add fragment support.
+In that, I decided to make this runnable on Python 3.2+ and Python 2.6+
+while, handling Python bytecodes from Python versions 2.5+ and
+3.2+. In doing so, it has been expedient to separate this into three
+projects: load loading and disassembly (xdis), parsing and tree
+building (spark_parser), and grammar and semantic actions for
+decompiling (uncompyle6).
 
-This project, uncompyle6, however owes its existence to uncompyle2 by
-Myst herie (Mysterie) whose first commit seems to goes back to 2012;
-it is also based on Hartmut's code. I chose this as it seems had been
-the most actively worked on most recently.
 
 Over the many years, code styles and Python features have
 changed. However brilliant the code was and still is, it hasn't really
 had a single public active maintainer. And there have been many forks
-of the code.
+of the code.  I have spent a great deal of time trying to organize and
+modularize the code so that it can handle more Python versions more
+gracefully (with still only moderate success).
 
 That it has been in need of an overhaul has been recognized by the
 Hartmut a decade an a half ago:
@@ -90,12 +135,20 @@ Hartmut a decade an a half ago:
     NB. This is not a masterpiece of software, but became more like a hack.
     Probably a complete rewrite would be sensefull. hG/2000-12-27
 
-Lastly, I should mention [unpyc](https://code.google.com/p/unpyc3/)
-and most especially [pycdc](https://github.com/zrax/pycdc), largely by
-Michael Hansen and Darryl Pogue. If they supported getting source-code
-fragments and I could call it from Python, I'd probably ditch this and
-use that. From what I've seen, the code runs blindingly fast and spans
-all versions of Python.
+This project deparses using an Early-algorithm parse with lots of
+massaging of tokens and the grammar in the scanner
+phase. Early-algorithm parsers are context free and tend to be linear
+if the grammar is LR or left recursive.
+
+Another approach that doesn't use grammars is to do something like
+simulate execution symbolically and build expression trees off of
+stack results. The two important projects that work this way are
+[unpyc3](https://code.google.com/p/unpyc3/) and most especially
+[pycdc](https://github.com/zrax/pycdc) The latter project is largely
+by Michael Hansen and Darryl Pogue. If they supported getting
+source-code fragments and I could call it from Python, I'd probably
+ditch this and use that. From what I've seen, the code runs blindingly
+fast and spans all versions of Python.
 
 Tests for the project have been, or are being, culled from all of the
 projects mentioned.

LICENSE

@@ -1,6 +1,6 @@
-Copyright (c) 1998-2002 John Aycock
-Copyright (c) 2000 by hartmut Goebel <h.goebel@crazy-compilers.com>
 Copyright (c) 2015 by Rocky Bernstein
+Copyright (c) 2000 by hartmut Goebel <h.goebel@crazy-compilers.com>
+Copyright (c) 1998-2002 John Aycock
 
   Permission is hereby granted, free of charge, to any person obtaining
   a copy of this software and associated documentation files (the

MANIFEST.in

@@ -1,9 +1,11 @@
 include README.rst
-include HISTORY.md
 include ChangeLog
+include HISTORY.md
+include LICENSE
+include DECOMPYLE-2.4-CHANGELOG.txt
 include __pkginfo__.py
 recursive-include uncompyle6 *.py
 include bin/uncompyle6
 include bin/pydisassemble
-recursive-include test *.py
+recursive-include test *.py *.pyc
 recursive-include pytest *.py

Makefile

@@ -23,18 +23,31 @@ check:
 	@PYTHON_VERSION=`$(PYTHON) -V 2>&1 | cut -d ' ' -f 2 | cut -d'.' -f1,2`; \
 	$(MAKE) check-$$PYTHON_VERSION
 
+# Run all quick tests
+check-short: pytest
+	$(MAKE) -C test check-short
+
 #: Tests for Python 2.7, 3.3 and 3.4
 check-2.7 check-3.3 check-3.4: pytest
 	$(MAKE) -C test $@
 
-#: Tests for Python 3.5 - pytest doesn't work here
-check-3.5:
+#: Tests for Python 3.2 and 3.5 - pytest doesn't work here
+# Or rather 3.5 doesn't work not on Travis
+check-3.1 check-3.2 check-3.5 check-3.6:
 	$(MAKE) -C test $@
 
 #:Tests for Python 2.6 (doesn't have pytest)
 check-2.6:
 	$(MAKE) -C test $@
 
+#:PyPy 2.6.1 or PyPy 5.0.1
+# Skip for now
+2.6 5.0 5.3:
+
+#:PyPy pypy3-2.4.0 Python 3:
+pypy-3.2 2.4:
+	$(MAKE) -C test $@
+
 #: Run py.test tests
 pytest:
 	$(MAKE) -C pytest check
@@ -44,9 +57,9 @@ clean: clean_pyc
 	$(PYTHON) ./setup.py $@
 	(cd test && $(MAKE) clean)
 
-#: Create source (tarball) and binary (egg) distribution
+#: Create source (tarball) and wheel distribution
 dist:
-	$(PYTHON) ./setup.py sdist bdist_egg
+	$(PYTHON) ./setup.py sdist bdist_wheel
 
 #: Remove .pyc files
 clean_pyc:
@@ -73,6 +86,11 @@ bdist_egg:
 	$(PYTHON) ./setup.py bdist_egg
 
 
+#: Create binary wheel distribution
+bdist_wheel:
+	$(PYTHON) ./setup.py bdist_wheel
+
+
 # It is too much work to figure out how to add a new command to distutils
 # to do the following. I'm sure distutils will someday get there.
 DISTCLEAN_FILES = build dist *.pyc

NEWS

@@ -1,3 +1,189 @@
+uncompyle6 2.9.3 2016-10-26
+
+Release forced by incompatiblity change in xdis 3.2.0.
+
+- Python 3.1 bugs:
+  * handle "with ... as"
+  * handle "with"
+  * Start handling def (...) -> yy (has bugs still)
+
+- DRY Python 3.x via inheritance
+- Python 3.6 work (from Daniel Bradburn)
+  * Handle 3.6 buildstring
+  * Handle 3.6 handle single and multiple fstring better
+
+
+uncompyle6 2.9.2 2016-10-15
+
+- use source-code line breaks to assist in where to break
+  in tuples and maps
+- Fix Python 1.5 decompyle bugs
+- Fix some Python 2.6 and below bugs
+- DRY fragments.py code a little
+
+uncompyle6 2.9.1 2016-10-09
+
+- Improved Python 1.5 decompiling
+- Handle old-style pre Python 2.2 classes
+
+uncompyle6 2.9.0 2016-10-09
+
+- Use xdis 3.0.0 protocol load_module.
+  this Forces change in requirements.txt and _pkg_info_.py
+- Start Python 1.5 decompiling; another round of work is needed to
+  remove bugs
+- Simpify python 2.1 grammar
+- Fix bug with -t ...  Wasn't showing source text when -t option was given
+- Fix 2.1-2.6 bug in list comprehension
+
+uncompyle6 2.8.4 2016-10-08
+
+- Python 3 disassembly bug fixes
+- Python 3.6 fstring bug fixes (from moagstar)
+- Python 2.1 disassembly
+- COME_FROM suffixes added in Python3
+- use .py extension in verification disassembly
+
+uncompyle6 2.8.3 2016-09-11 live from NYC!
+
+NOTE: this is possibly the last release before a major reworking of
+control-flow structure detection is done.
+
+- Lots of bug fixes in decompilation:
+  * 3.0 .. 3.4 whileTrue bug
+  * 3.x function declaration deparsing:
+    . 3.0 .. 3.2 *args processing
+    . 3.0 .. 3.2 call name and kwargs bug
+    . 3.0 .. getting parameter of *
+    . 3.0 .. handling varible number of args
+    . 3.0 .. "if" structure bugs
+  * 3.5+ if/else bugs
+  * 2.2-2.6 bugs
+    . try/except control flow
+    . a == b == c -like detection
+    . generator detection
+    . "while .. and" statement bugs
+    . handle "except <cond>, <var>"
+    . use older raise format in 2.x
+- scanner "disassemble" is now "ingest". True disassembly is done by xdis
+- Start accepting Python 3.1 bytecode
+- Add --weak-verify option on test_pyenvlib and test_pythonlib. This
+  catches more bugs more easily
+- bump xdis requirement so we can deparse dropbox 2.5 code
+- Added H. Goebel's changes before 2.4 in DECOMPYLE-2.4-CHANGELOG.txt
+
+uncompyle6 2.8.2 2016-08-29
+
+- Handle Python 3.6 format string conversions !r, !s, !a
+- Start to handle 3.1 bytecode
+- Fix some PyPy translation bugs
+- We now only handle 3.6.0a3+ since that is incompatible with 3.6 before that
+
+uncompyle6 2.8.1 2016-08-20
+
+- Add Python 2.2 decompilation
+
+- Fix bugs
+ * PyPy LOOKUP_METHOD bug
+ * Python 3.6 FORMAT_VALUE handles expressions now
+
+uncompyle6 2.8.0 2016-08-03
+
+- Start Python 3.6 support (moagstar)
+  more work on PEP 498 needed
+- tidy bytecode/word output
+- numerous decompiling bugs fixed
+- grammar testing started
+- show magic number in deparsed output
+- better grammar and semantic action segregation based
+  on python bytecode version
+
+uncompyle6 2.7.1 2016-07-26
+
+- PyPy bytecodes for 2.7 and 3.2 added
+- Instruction formatting improved slightly
+- 2.7 bytecode "continue" bug fixed
+
+uncompyle6 2.7.0 2016-07-15
+
+- Many Syntax and verifification bugs removed
+  tested on standard libraries from 2.3.7 to 3.5.1
+  and they all decompile and verify fine.
+  I'm sure there are more bugs though.
+
+uncompyle6 2.6.2 2016-07-11 Manhattenhenge
+
+- Extend bytecodes back to 2.3
+- Fix bugs:
+  * 3.x and 2.7 set comprehensions,
+  * while1 loops
+  * continue statements
+- DRY and segregate grammar more
+
+uncompyle6 2.6.1 2016-07-08
+
+- Go over Python 2.5 bytecode deparsing
+  all library programs now deparse
+- Fix a couple bugs in 2.6 deparsing
+
+uncompyle6 2.6.0 2016-07-07
+
+- Improve Python 2.6 bytecode deparsing:
+  stdlib now will deparse something
+- Better <2.6 vs. 2.7 grammar separation
+- Fix some 2.7 deparsing bugs
+- Fix bug in installing uncompyle6 script
+- Doc improvments
+
+uncompyle6 2.5.0 2016-06-22  Summer Solstace
+
+- Much better Python 3.2-3.5 coverage.
+  3.4.6 is probably the best;3.2 and 3.5 are weaker
+- Better AST printing with -t
+- Better error reporting
+- Better fragment offset tracking
+- Some (much-needed) code refactoring
+
+uncompyle6 2.4.0 2016-05-18 (in memory of Lewis Bernstein)
+
+- Many Python 3 bugs fixed:
+  * Python 3.2 to 3.5 libaries largely
+    uncompyle and most verify
+- pydisassembler:
+  * disassembles all code objects in a file
+  * can select showing bytecode before
+    or after uncompyle mangling, option -U
+- DRY scanner code (but more is desired)
+- Some code cleanup (but more is desired)
+- Misc Bugs fixed:
+  * handle complex number unmarshaling
+  * Running on Python 2 to works on Python 3.5 bytecodes now
+
+uncompyle6 2.3.5 and 2.3.6 2016-05-14
+
+- Python 2 class decorator fix (thanks to Tey)
+- Fix fragment parsing bugs
+- Fix some Python 3 parsing bugs:
+  * Handling single in * parameter
+  * "while True"
+  * escape from for inside if
+  * yield expressions
+- Correct history based on info from Dan Pascu
+- Fix up pip packaging, ugh.
+
+uncompyle6 2.3.4 2016-05-5
+
+- More Python 3.5 parsing bugs addressed
+- decompiling Python 3.5 from other Python versions works
+- test from Python 3.2
+- remove "__module__ = __name__" in 3.0 <= Python 3.2
+
+uncompyle6 2.3.3 2016-05-3
+
+- Fix bug in running uncompyle6 script on Python 3
+- Speed up performance on deparsing long lists by grouping in chunks of 32 and 256 items
+- DRY Python expressions between Python 2 and 3
+
 uncompyle6 2.3.2 2016-05-1
 
 - Add --version option standalone scripts

PKG-INFO

@@ -5,6 +5,6 @@ Summary: Python byte-code to source-code converter
 Home-page: http://github.com/rocky/python-uncompyle6
 Author: Rocky
 Author-email: rb@dustyfeet.com
-License: GPLv3
+License: MIT
 Description: UNKNOWN
 Platform: UNKNOWN

README.rst

@@ -3,22 +3,28 @@
 uncompyle6
 ==========
 
-A native Python bytecode Disassembler, Decompiler, Fragment Decompiler
-and bytecode library
+A native Python cross-version Decompiler and Fragment Decompiler.
+Follows in the tradition of decompyle, uncompyle, and uncompyle2.
 
 
 Introduction
 ------------
 
 *uncompyle6* translates Python bytecode back into equivalent Python
-source code. It accepts bytecodes from Python version 2.5 to 3.4 or
-so and has been tested on Python running versions 2.6, 2.7, 3.3,
-3.4 and 3.5.
+source code. It accepts bytecodes from Python version 2.1 to 3.6 or
+so, including PyPy bytecode and Dropbox's Python 2.5 bytecode.
 
 Why this?
 ---------
 
-What makes this different from other CPython bytecode decompilers?  Its
+There were a number of decompyle, uncompile, uncompyle2, uncompyle3
+forks around. All of them came basically from the same code base, and
+almost all of them no were no longer actively maintained. Only one
+handled Python 3, and even there, only 3.2. This code pulls these
+together and moves forward. It also addresses a number of open issues
+in the previous forks.
+
+What makes this different from other CPython bytecode decompilers?: its
 ability to deparse just fragments and give source-code information
 around a given bytecode offset.
 
@@ -34,12 +40,12 @@ location in more detail than just a line number.  It can be also used
 when source-code information does not exist and there is just bytecode
 information.
 
-Other parts of the library can be used inside Python for various
-bytecode-related tasks. For example you can read in bytecode,
-i.e. perform a version-independent `marshal.loads()`, and disassemble
-the bytecode using a version of Python different from the one used to
-compile the bytecode.
+Requirements
+------------
 
+This project requires Python 2.6 or later, PyPy 3-2.4, or PyPy-5.0.1.
+The bytecode files it can read has been tested on Python bytecodes from
+versions 2.1-2.7, and 3.2-3.6 and the above-mentioned PyPy versions.
 
 Installation
 ------------
@@ -54,7 +60,7 @@ This uses setup.py, so it follows the standard Python routine:
     # or if you have pyenv:
     python setup.py develop
 
-A GNU makefile is also provided so `make install` (possibly as root or
+A GNU makefile is also provided so :code:`make install` (possibly as root or
 sudo) will do the steps above.
 
 Testing
@@ -68,7 +74,7 @@ A GNU makefile has been added to smooth over setting running the right
 command, and running tests from fastest to slowest.
 
 If you have remake_ installed, you can see the list of all tasks
-including tests via `remake --tasks`
+including tests via :code:`remake --tasks`
 
 
 Usage
@@ -78,28 +84,59 @@ Run
 
 ::
 
-     ./bin/uncompyle6 -h
-     ./bin/pydisassemble -h
+$ uncompyle6 *compiled-python-file-pyc-or-pyo*
 
-for usage help
+For usage help:
+
+::
+
+   $ uncompyle6 -h
 
 
 Known Bugs/Restrictions
 -----------------------
 
-Python 2 deparsing is probably as solid as the various versions of
-uncompyle2.  Python 3 deparsing is okay but not as solid.
+About 90% of the decompilation verifies from Python 2.3.7 to Python
+3.4.2 on the standard library packages I have on my system.
+
+(Verification is the process of decompiling bytecode, compiling with a
+Python for that byecode version, and then comparing the byetcode
+produced by the decompiled/compiled program. Some allowance is made
+for inessential differences, but other semantically equivalent
+differences are not caught.)
+
+Later distributions average about 200 files. At this point, 2.7
+decompilation is definitely better than uncompyle2. A number of bugs
+have been fixed. We now handle more Python bytecodes than the old
+decompyle program that handled Python bytecodes ranging from 1.5 to
+2.4.  There is some work do do on the lower end which is more
+difficult for us since we don't have a Python interpreter for versions
+1.5, 1.6 or 2.0.
+
+Python 3.5 largely works, but still has some bugs in it.
+Python 3.6 changes things drastically by using word codes rather than
+byte codes, and that needs to be addressed.
+
+Currently not all Python magic numbers are supported. Specifically in
+some versions of Python, notably Python 3.6, the magic number has
+changes several times within a version. We support only the released
+magic. There are also customized Python interpreters, notably Dropbox,
+which use their own magic and encrypt bytcode. With the exception of
+the Dropbox's old Python 2.5 interpreter this kind of thing is not
+handled.
+
+There is lots to do, so please dig in and help.
 
 See Also
 --------
 
-* https://github.com/zrax/pycdc
-* https://code.google.com/p/unpyc3/
-
-The HISTORY file.
+* https://github.com/zrax/pycdc : supports all versions of Python and is written in C++
+* https://code.google.com/archive/p/unpyc3/ : supports Python 3.2 only. The above projects use a different decompiling technique what is used here.
+* The HISTORY_ file.
 
 .. |downloads| image:: https://img.shields.io/pypi/dd/uncompyle6.svg
 .. _trepan: https://pypi.python.org/pypi/trepan
+.. _HISTORY: https://github.com/rocky/python-uncompyle6/blob/master/HISTORY.md
 .. _debuggers: https://pypi.python.org/pypi/trepan3k
 .. _remake: https://bashdb.sf.net/remake
 .. _pycdc: https://github.com/zrax/pycdc

__pkginfo__.py

@@ -12,7 +12,7 @@ copyright   = """
 Copyright (C) 2015, 2016 Rocky Bernstein <rb@dustyfeet.com>.
 """
 
-classifiers =  ['Development Status :: 3 - Alpha',
+classifiers =  ['Development Status :: 4 - Beta',
                 'Intended Audience :: Developers',
                 'Operating System :: OS Independent',
                 'Programming Language :: Python',
@@ -23,6 +23,7 @@ classifiers =  ['Development Status :: 3 - Alpha',
                 'Programming Language :: Python :: 3.3',
                 'Programming Language :: Python :: 3.4',
                 'Programming Language :: Python :: 3.5',
+                'Programming Language :: Python :: 3.6',
                 'Topic :: Software Development :: Debuggers',
                 'Topic :: Software Development :: Libraries :: Python Modules',
                 ]
@@ -30,33 +31,35 @@ classifiers =  ['Development Status :: 3 - Alpha',
 # The rest in alphabetic order
 author             = "Rocky Bernstein, Hartmut Goebel, John Aycock, and others"
 author_email       = "rb@dustyfeet.com"
+entry_points={
+    'console_scripts': [
+        'uncompyle6=uncompyle6.bin.uncompile:main_bin',
+        'pydisassemble=uncompyle6.bin.pydisassemble:main',
+    ]}
 ftp_url            = None
-install_requires   = ['python-spark >= 1.1.0']
-license            = 'GPL'
-
+install_requires   = ['spark-parser >= 1.4.0, < 1.5.0',
+                      'xdis >= 3.2.0, < 3.3.0']
 license            = 'MIT'
 mailing_list       = 'python-debugger@googlegroups.com'
 modname            = 'uncompyle6'
-packages           = ['uncompyle6', 'uncompyle6.opcodes', 'uncompyle6.semantics', 'uncompyle6.scanners', 'uncompyle6.parsers']
 py_modules         = None
-short_desc         = 'Python byte-code disassembler and source-code converter'
-scripts            = ['bin/uncompyle6', 'bin/pydisassemble']
-
-import os.path
-
-
-def get_srcdir():
-    filename = os.path.normcase(os.path.dirname(os.path.abspath(__file__)))
-    return os.path.realpath(filename)
-
-ns = {}
+short_desc         = 'Python cross-version byte-code deparser'
 web                = 'https://github.com/rocky/python-uncompyle6/'
 
 # tracebacks in zip files are funky and not debuggable
 zip_safe = True
 
 
-def read(*rnames):
-    return open(os.path.join(os.path.dirname(__file__), *rnames)).read()
+import os.path
+def get_srcdir():
+    filename = os.path.normcase(os.path.dirname(os.path.abspath(__file__)))
+    return os.path.realpath(filename)
 
+srcdir = get_srcdir()
+
+def read(*rnames):
+    return open(os.path.join(srcdir, *rnames)).read()
+
+# Get info from files; set: long_description and VERSION
 long_description   = ( read("README.rst") + '\n' )
+exec(read('uncompyle6/version.py'))

bin/pydisassemble

@@ -1,83 +1,3 @@
 #!/usr/bin/env python
-# Mode: -*- python -*-
-#
-# Copyright (c) 2015-2016 by Rocky Bernstein <rb@dustyfeet.com>
-#
-from __future__ import print_function
-import sys, os, getopt
-
-program =  os.path.basename(__file__)
-
-__doc__ = """
-Usage:
-  %s [OPTIONS]... FILE
-  %s [--help | -h | -V | --version]
-
-Examples:
-  %s foo.pyc
-  %s foo.py
-  %s -o foo.pydis foo.pyc
-  %s -o /tmp foo.pyc
-
-Options:
-  -o <path>     output decompiled files to this path:
-                if multiple input files are decompiled, the common prefix
-                is stripped from these names and the remainder appended to
-                <path>
-  --help        show this message
-
-""" % ((program,) * 6)
-
-
-Usage_short = \
-"%s [--help] [--verify] [--showasm] [--showast] [-o <path>] FILE|DIR..." % program
-
-from uncompyle6 import check_python_version
-from uncompyle6.disas import disassemble_files
-from uncompyle6.version import VERSION
-
-check_python_version(program)
-
-outfile = '-'
-out_base = None
-
-
-try:
-    opts, files = getopt.getopt(sys.argv[1:], 'hVo:', ['help', 'version'])
-except getopt.GetoptError as e:
-    print('%s: %s' % (os.path.basename(sys.argv[0]), e),  file=sys.stderr)
-    sys.exit(-1)
-
-for opt, val in opts:
-    if opt in ('-h', '--help'):
-        print(__doc__)
-        sys.exit(1)
-    elif opt in ('-V', '--version'):
-        print("%s %s" % (program, VERSION))
-        sys.exit(0)
-    elif opt == '-o':
-        outfile = val
-    else:
-        print(opt)
-        print(Usage_short)
-        sys.exit(1)
-
-# argl, commonprefix works on strings, not on path parts,
-# thus we must handle the case with files in 'some/classes'
-# and 'some/cmds'
-src_base = os.path.commonprefix(files)
-if src_base[-1:] != os.sep:
-    src_base = os.path.dirname(src_base)
-if src_base:
-    sb_len = len( os.path.join(src_base, '') )
-    files = [f[sb_len:] for f in files]
-    del sb_len
-
-if outfile == '-':
-    outfile = None # use stdout
-elif outfile and os.path.isdir(outfile):
-    out_base = outfile; outfile = None
-elif outfile and len(files) > 1:
-    out_base = outfile; outfile = None
-
-disassemble_files(src_base, out_base, files, outfile)
+from uncompyle6.bin.pydisassemble import main
+main()

bin/uncompyle6

@@ -1,222 +1,3 @@
 #!/usr/bin/env python
-# Mode: -*- python -*-
-#
-# Copyright (c) 2015-2016 by Rocky Bernstein
-# Copyright (c) 2000-2002 by hartmut Goebel <h.goebel@crazy-compilers.com>
-#
-from __future__ import print_function
-import sys, os, getopt, time
-
-program =  os.path.basename(__file__)
-
-__doc__ = """
-Usage:
-  %s [OPTIONS]... [ FILE | DIR]...
-  %s [--help | -h | --V | --version]
-
-Examples:
-  %s      foo.pyc bar.pyc       # decompile foo.pyc, bar.pyc to stdout
-  %s -o . foo.pyc bar.pyc       # decompile to ./foo.pyc_dis and ./bar.pyc_dis
-  %s -o /tmp /usr/lib/python1.5 # decompile whole library
-
-Options:
-  -o <path>     output decompiled files to this path:
-                if multiple input files are decompiled, the common prefix
-                is stripped from these names and the remainder appended to
-                <path>
-                  uncompyle6 -o /tmp bla/fasel.pyc bla/foo.pyc
-                    -> /tmp/fasel.pyc_dis, /tmp/foo.pyc_dis
-                  uncompyle6 -o /tmp bla/fasel.pyc bar/foo.pyc
-                    -> /tmp/bla/fasel.pyc_dis, /tmp/bar/foo.pyc_dis
-                  uncompyle6 -o /tmp /usr/lib/python1.5
-                    -> /tmp/smtplib.pyc_dis ... /tmp/lib-tk/FixTk.pyc_dis
-  -c <file>     attempts a disassembly after compiling <file>
-  -d            print timestamps
-  -p <integer>  use <integer> number of processes
-  -r            recurse directories looking for .pyc and .pyo files
-  --verify      compare generated source with input byte-code
-                (requires -o)
-  --help        show this message
-
-Debugging Options:
-  --asm     -a  include byte-code         (disables --verify)
-  --grammar -g  show matching grammar
-  --tree    -t  include syntax tree       (disables --verify)
-
-Extensions of generated files:
-  '.pyc_dis' '.pyo_dis'   successfully decompiled (and verified if --verify)
-    + '_unverified'       successfully decompile but --verify failed
-    + '_failed'           decompile failed (contact author for enhancement)
-""" % ((program,) * 6
-
-program =  os.path.basename(__file__)
-
-from uncompyle6 import verify, check_python_version
-from uncompyle6.main import main, status_msg
-from uncompyle6.version import VERSION
-
-def usage():
-    print("""usage:
-   %s [--verify] [--asm] [--tree] [--grammar] [-o <path>] FILE|DIR...
-   %s [--help | -h | --version | -V]
-"""  % (program, program))
-    sys.exit(1)
-
-
-check_python_version(program)
-
-showasm = showast = do_verify = recurse_dirs = False
-numproc = 0
-outfile = '-'
-out_base = None
-codes = []
-timestamp = False
-timestampfmt = "# %Y.%m.%d %H:%M:%S %Z"
-
-try:
-    opts, files = getopt.getopt(sys.argv[1:], 'hagtdrVo:c:p:',
-                                'help asm grammar recurse timestamp tree verify version '
-                                'showgrammar'.split(' '))
-except getopt.GetoptError as e:
-    print('%s: %s' % (os.path.basename(sys.argv[0]), e),  file=sys.stderr)
-    sys.exit(-1)
-
-options = {}
-for opt, val in opts:
-    if opt in ('-h', '--help'):
-        print(__doc__)
-        sys.exit(0)
-    elif opt in ('-V', '--version'):
-        print("%s %s" % (program, VERSION))
-        sys.exit(0)
-    elif opt == '--verify':
-        options['do_verify'] = True
-    elif opt in ('--asm', '-a'):
-        options['showasm'] = True
-        options['do_verify'] = False
-    elif opt in ('--tree', '-t'):
-        options['showast'] = True
-        options['do_verify'] = False
-    elif opt in ('--grammar', '-g'):
-        options['showgrammar'] = True
-    elif opt == '-o':
-        outfile = val
-    elif opt in ('--timestamp', '-d'):
-        timestamp = True
-    elif opt == '-c':
-        codes.append(val)
-    elif opt == '-p':
-        numproc = int(val)
-    elif opt in ('--recurse', '-r'):
-        recurse_dirs = True
-    else:
-        print(opt, file=sys.stderr)
-        usage()
-
-# expand directory if specified
-if recurse_dirs:
-    expanded_files = []
-    for f in files:
-        if os.path.isdir(f):
-            for root, _, dir_files in os.walk(f):
-                for df in dir_files:
-                    if df.endswith('.pyc') or df.endswith('.pyo'):
-                        expanded_files.append(os.path.join(root, df))
-    files = expanded_files
-
-# argl, commonprefix works on strings, not on path parts,
-# thus we must handle the case with files in 'some/classes'
-# and 'some/cmds'
-src_base = os.path.commonprefix(files)
-if src_base[-1:] != os.sep:
-    src_base = os.path.dirname(src_base)
-if src_base:
-    sb_len = len( os.path.join(src_base, '') )
-    files = [f[sb_len:] for f in files]
-    del sb_len
-
-if not files:
-    print("No files given", file=sys.stderr)
-    usage()
-
-
-if outfile == '-':
-    outfile = None # use stdout
-elif outfile and os.path.isdir(outfile):
-    out_base = outfile; outfile = None
-elif outfile and len(files) > 1:
-    out_base = outfile; outfile = None
-
-if timestamp:
-    print(time.strftime(timestampfmt))
-
-if numproc <= 1:
-    try:
-        result = main(src_base, out_base, files, codes, outfile,
-                      **options)
-        if len(files) > 1:
-            mess = status_msg(do_verify, *result)
-            print('# ' + mess)
-            pass
-    except (KeyboardInterrupt):
-        pass
-    except verify.VerifyCmpError:
-        raise
-else:
-    from multiprocessing import Process, Queue
-
-    try:
-        from Queue import Empty
-    except ImportError:
-        from Queue import Empty
-
-    fqueue = Queue(len(files)+numproc)
-    for f in files:
-        fqueue.put(f)
-    for i in range(numproc):
-        fqueue.put(None)
-
-    rqueue = Queue(numproc)
-
-    def process_func():
-        try:
-            (tot_files, okay_files, failed_files, verify_failed_files) = (0, 0, 0, 0)
-            while 1:
-                f = fqueue.get()
-                if f is None:
-                    break
-                (t, o, f, v) = \
-                  main(src_base, out_base, [f], codes, outfile, **options)
-                tot_files += t
-                okay_files += o
-                failed_files += f
-                verify_failed_files += v
-        except (Empty, KeyboardInterrupt):
-            pass
-        rqueue.put((tot_files, okay_files, failed_files, verify_failed_files))
-        rqueue.close()
-
-    try:
-        procs = [Process(target=process_func) for i in range(numproc)]
-        for p in procs:
-            p.start()
-        for p in procs:
-            p.join()
-        try:
-            (tot_files, okay_files, failed_files, verify_failed_files) = (0, 0, 0, 0)
-            while True:
-                (t, o, f, v) = rqueue.get(False)
-                tot_files += t
-                okay_files += o
-                failed_files += f
-                verify_failed_files += v
-        except Empty:
-            pass
-        print('# decompiled %i files: %i okay, %i failed, %i verify failed' %
-              (tot_files, okay_files, failed_files, verify_failed_files))
-    except (KeyboardInterrupt, OSError):
-        pass
-
-
-if timestamp:
-    print(time.strftime(timestampfmt))
+from uncompyle6.bin.uncompile import main_bin
+main_bin()

pytest/test_deparse.py

@@ -20,12 +20,20 @@ def for_range_stmt():
     for i in range(2):
         i+1
 
+# # FIXME: add this test - but for Python 2.7+ only
+# def set_comp():
+#     {y for y in range(3)}
+
+# FIXME: add this test
+def list_comp():
+    [y for y in range(3)]
+
 def get_parsed_for_fn(fn):
     code = fn.__code__ if PYTHON3 else fn.func_code
     return deparse(PYTHON_VERSION, code)
 
 def check_expect(expect, parsed):
-    debug = True
+    debug = False
     i = 2
     max_expect = len(expect)
     for name, offset in sorted(parsed.offsets.keys()):
@@ -160,7 +168,7 @@ return (x, y)
 -------------
 """.split("\n")
     check_expect(expect, parsed)
-#     ########################################################
+    ########################################################
 #     # try
 
 #     expect = """
@@ -291,6 +299,12 @@ return
 Contained in...
     i + 1
     -----
+31
+return
+------
+Contained in...
+for i in range(2): ...
+------------------ ...
 34
 return
 ------

pytest/test_disasm.py

@@ -24,7 +24,7 @@ os.chdir(src_dir)
 def test_funcoutput(capfd, test_tuple, function_to_test):
 
     in_file , filename_expected = test_tuple
-    function_to_test(in_file)
+    function_to_test(in_file, native=False)
     resout, reserr = capfd.readouterr()
     expected = open(filename_expected, "r").read()
     if resout != expected:

pytest/test_fjt.py

@@ -0,0 +1,40 @@
+#!/usr/bin/env python
+from uncompyle6 import PYTHON_VERSION, IS_PYPY
+from uncompyle6.scanner import get_scanner
+from array import array
+def bug(state, slotstate):
+    if state:
+        if slotstate is not None:
+            for key, value in slotstate.items():
+                setattr(state, key, 2)
+
+def test_if_in_for():
+    code = bug.__code__
+    scan = get_scanner(PYTHON_VERSION)
+    print(PYTHON_VERSION)
+    if 2.7 <= PYTHON_VERSION <= 3.0 and not IS_PYPY:
+        n = scan.setup_code(code)
+        scan.build_lines_data(code, n)
+        scan.build_prev_op(n)
+        fjt = scan.find_jump_targets()
+        assert {15: [3], 69: [66], 63: [18]} == fjt
+        assert scan.structs == \
+          [{'start': 0, 'end': 72, 'type': 'root'},
+           {'start': 18, 'end': 66, 'type': 'if-then'},
+           {'start': 31, 'end': 59, 'type': 'for-loop'},
+           {'start': 62, 'end': 63, 'type': 'for-else'}]
+    elif 3.2 < PYTHON_VERSION <= 3.4:
+        scan.code = array('B', code.co_code)
+        scan.build_lines_data(code)
+        scan.build_prev_op()
+        fjt  = scan.find_jump_targets()
+        assert {69: [66], 63: [18]} == fjt
+        assert scan.structs == \
+          [{'end': 72, 'type': 'root', 'start': 0},
+           {'end': 66, 'type': 'if-then', 'start': 6},
+           {'end': 63, 'type': 'if-then', 'start': 18},
+           {'end': 59, 'type': 'for-loop', 'start': 31},
+           {'end': 63, 'type': 'for-else', 'start': 62}]
+    else:
+        assert True, "FIXME: should note fixed"
+    return

pytest/test_fstring.py

@@ -0,0 +1,150 @@
+# std
+import os
+# test
+import pytest
+import hypothesis
+from hypothesis import strategies as st
+# uncompyle6
+from uncompyle6 import PYTHON_VERSION, deparse_code
+
+
+@st.composite
+def expressions(draw):
+    # todo : would be nice to generate expressions using hypothesis however
+    # this is pretty involved so for now just use a corpus of expressions
+    # from which to select.
+    return draw(st.sampled_from((
+        'abc',
+        'len(items)',
+        'x + 1',
+        'lineno',
+        'container',
+        'self.attribute',
+        'self.method()',
+        # These expressions are failing, I think these are control
+        # flow problems rather than problems with FORMAT_VALUE,
+        # however I need to confirm this...
+        #'sorted(items, key=lambda x: x.name)',
+        #'func(*args, **kwargs)',
+        #'text or default',
+        #'43 if life_the_universe and everything else None'
+    )))
+
+
+@st.composite
+def format_specifiers(draw):
+    """
+    Generate a valid format specifier using the rules:
+
+    format_spec ::=  [[fill]align][sign][#][0][width][,][.precision][type]
+    fill        ::=  <any character>
+    align       ::=  "<" | ">" | "=" | "^"
+    sign        ::=  "+" | "-" | " "
+    width       ::=  integer
+    precision   ::=  integer
+    type        ::=  "b" | "c" | "d" | "e" | "E" | "f" | "F" | "g" | "G" | "n" | "o" | "s" | "x" | "X" | "%"
+
+    See https://docs.python.org/2/library/string.html
+
+    :param draw: Let hypothesis draw from other strategies.
+
+    :return: An example format_specifier.
+    """
+    alphabet_strategy = st.characters(min_codepoint=ord('a'), max_codepoint=ord('z'))
+    fill = draw(st.one_of(alphabet_strategy, st.none()))
+    align = draw(st.sampled_from(list('<>=^')))
+    fill_align = (fill + align or '') if fill else ''
+
+    type_ = draw(st.sampled_from('bcdeEfFgGnosxX%'))
+    can_have_sign = type_ in 'deEfFgGnoxX%'
+    can_have_comma = type_ in 'deEfFgG%'
+    can_have_precision = type_ in 'fFgG'
+    can_have_pound = type_ in 'boxX%'
+    can_have_zero = type_ in 'oxX'
+
+    sign = draw(st.sampled_from(list('+- ') + [''])) if can_have_sign else ''
+    pound = draw(st.sampled_from(('#', '',))) if can_have_pound else ''
+    zero = draw(st.sampled_from(('0', '',))) if can_have_zero else ''
+
+    int_strategy = st.integers(min_value=1, max_value=1000)
+
+    width = draw(st.one_of(int_strategy, st.none()))
+    width = str(width) if width is not None else ''
+
+    comma = draw(st.sampled_from((',', '',))) if can_have_comma else ''
+    if can_have_precision:
+        precision = draw(st.one_of(int_strategy, st.none()))
+        precision = '.' + str(precision) if precision else ''
+    else:
+        precision = ''
+
+    return ''.join((fill_align, sign, pound, zero, width, comma, precision, type_,))
+
+
+@st.composite
+def fstrings(draw):
+    """
+    Generate a valid f-string.
+    See https://www.python.org/dev/peps/pep-0498/#specification
+
+    :param draw: Let hypothsis draw from other strategies.
+
+    :return: A valid f-string.
+    """
+    character_strategy = st.characters(
+        blacklist_characters='\r\n\'\\s{}',
+        min_codepoint=1,
+        max_codepoint=1000,
+    )
+    is_raw = draw(st.booleans())
+    integer_strategy = st.integers(min_value=0, max_value=3)
+    expression_count = draw(integer_strategy)
+    content = []
+    for _ in range(expression_count):
+        expression = draw(expressions())
+        conversion = draw(st.sampled_from(('', '!s', '!r', '!a',)))
+        has_specifier = draw(st.booleans())
+        specifier = ':' + draw(format_specifiers()) if has_specifier else ''
+        content.append('{{{}{}}}'.format(expression, conversion, specifier))
+        content.append(draw(st.text(character_strategy)))
+    content = ''.join(content)
+    return "f{}'{}'".format('r' if is_raw else '', content)
+
+
+@pytest.mark.skipif(PYTHON_VERSION < 3.6, reason='need at least python 3.6')
+@hypothesis.given(format_specifiers())
+def test_format_specifiers(format_specifier):
+    """Verify that format_specifiers generates valid specifiers"""
+    try:
+        exec('"{:' + format_specifier + '}".format(0)')
+    except ValueError as e:
+        if 'Unknown format code' not in str(e):
+            raise
+
+
+def run_test(text):
+    hypothesis.assume(len(text))
+    hypothesis.assume("f'{" in text)
+    expr = text + '\n'
+    code = compile(expr, '<string>', 'single')
+    deparsed = deparse_code(PYTHON_VERSION, code, compile_mode='single')
+    recompiled = compile(deparsed.text, '<string>', 'single')
+    if recompiled != code:
+        assert 'dis(' + deparsed.text.strip('\n') + ')' == 'dis(' + expr.strip('\n') + ')'
+
+
+@pytest.mark.skipif(PYTHON_VERSION < 3.6, reason='need at least python 3.6')
+@hypothesis.given(fstrings())
+def test_uncompyle_fstring(fstring):
+    """Verify uncompyling fstring bytecode"""
+    run_test(fstring)
+
+
+@pytest.mark.skipif(PYTHON_VERSION < 3.6, reason='need at least python 3.6')
+@pytest.mark.parametrize('fstring', [
+    "f'{abc}{abc!s}'",
+    "f'{abc}0'",
+])
+def test_uncompyle_direct(fstring):
+    """useful for debugging"""
+    run_test(fstring)

pytest/test_grammar.py

@@ -0,0 +1,47 @@
+import pytest, re
+from uncompyle6 import PYTHON_VERSION, PYTHON3, IS_PYPY # , PYTHON_VERSION
+from uncompyle6.parser import get_python_parser
+from uncompyle6.scanner import get_scanner
+
+def test_grammar():
+
+    def check_tokens(tokens, opcode_set):
+        remain_tokens = set(tokens) - opcode_set
+        remain_tokens = set([re.sub('_\d+$','', t) for t in remain_tokens])
+        remain_tokens = set([re.sub('_CONT$','', t) for t in remain_tokens])
+        remain_tokens = set(remain_tokens) - opcode_set
+        assert remain_tokens == set([]), \
+            "Remaining tokens %s\n====\n%s" % (remain_tokens, p.dumpGrammar())
+
+    p = get_python_parser(PYTHON_VERSION, is_pypy=IS_PYPY)
+    lhs, rhs, tokens, right_recursive = p.checkSets()
+    expect_lhs = set(['expr1024', 'pos_arg'])
+    unused_rhs = set(['build_list', 'call_function', 'mkfunc', 'mklambda',
+                      'unpack', 'unpack_list'])
+    expect_right_recursive = [['designList', ('designator', 'DUP_TOP', 'designList')]]
+    if PYTHON3:
+        expect_lhs.add('load_genexpr')
+        unused_rhs = unused_rhs.union(set("""
+        except_pop_except genexpr classdefdeco2 listcomp
+        """.split()))
+    else:
+        expect_lhs.add('kwarg')
+    assert expect_lhs == set(lhs)
+    assert unused_rhs == set(rhs)
+    assert expect_right_recursive == right_recursive
+    s = get_scanner(PYTHON_VERSION, IS_PYPY)
+    ignore_set = set(
+            """
+            JUMP_BACK CONTINUE RETURN_END_IF
+            COME_FROM COME_FROM_EXCEPT COME_FROM_LOOP COME_FROM_WITH
+            COME_FROM_FINALLY
+            LOAD_GENEXPR LOAD_ASSERT LOAD_SETCOMP LOAD_DICTCOMP
+            LAMBDA_MARKER RETURN_LAST
+            """.split())
+    if 2.6 <= PYTHON_VERSION <= 2.7:
+        opcode_set = set(s.opc.opname).union(ignore_set)
+        check_tokens(tokens, opcode_set)
+    elif PYTHON_VERSION == 3.4:
+        ignore_set.add('LOAD_CLASSNAME')
+        opcode_set = set(s.opc.opname).union(ignore_set)
+        check_tokens(tokens, opcode_set)

pytest/test_load.py

@@ -1,10 +0,0 @@
-import sys
-from uncompyle6.load import load_file, check_object_path, load_module
-
-def test_load():
-    """Basic test of load_file, check_object_path and load_module"""
-    co = load_file(__file__)
-    obj_path = check_object_path(__file__)
-    version, timestamp, magic_int, co2 = load_module(obj_path)
-    assert sys.version[0:3] == str(version)
-    assert co == co2

pytest/test_single_compile.py

@@ -9,8 +9,6 @@ def test_single_mode():
         'i = j % 4',
         'i = {}',
         'i = []',
-        'while i < 1 or stop:\n    i\n',
-        'while i < 1 or stop:\n    print%s\n' % ('(i)' if PYTHON3 else ' i'),
         'for i in range(10):\n    i\n',
         'for i in range(10):\n    for j in range(10):\n        i + j\n',
         'try:\n    i\nexcept Exception:\n    j\nelse:\n    k\n'

pytest/testdata/if-2.7.right

@@ -1,13 +1,12 @@
 # Python 2.7
 # Embedded file name: simple_source/branching/05_if.py
 
-   6       0	LOAD_NAME         'True'
-           3	POP_JUMP_IF_FALSE '15'
-
-   7       6	LOAD_NAME         'False'
-           9	STORE_NAME        'b'
-          12	JUMP_FORWARD      '15'
-        15_0	COME_FROM         '12'
-          15	LOAD_CONST        ''
-          18	RETURN_VALUE      ''
+   6       0  LOAD_NAME             0  'True'
+           3  POP_JUMP_IF_FALSE    15  'to 15'
 
+   7       6  LOAD_NAME             1  'False'
+           9  STORE_NAME            2  'b'
+          12  JUMP_FORWARD          0  'to 15'
+        15_0  COME_FROM                '12'
+          15  LOAD_CONST            0  ''
+          18  RETURN_VALUE     

pytest/testdata/ifelse-2.7.right

@@ -1,16 +1,15 @@
 # Python 2.7
 # Embedded file name: simple_source/branching/05_ifelse.py
 
-   3       0	LOAD_NAME         'True'
-           3	POP_JUMP_IF_FALSE '15'
+   3       0  LOAD_NAME             0  'True'
+           3  POP_JUMP_IF_FALSE    15  'to 15'
 
-   4       6	LOAD_CONST        1
-           9	STORE_NAME        'b'
-          12	JUMP_FORWARD      '21'
-
-   6      15	LOAD_CONST        2
-          18	STORE_NAME        'd'
-        21_0	COME_FROM         '12'
-          21	LOAD_CONST        ''
-          24	RETURN_VALUE      ''
+   4       6  LOAD_CONST            0  1
+           9  STORE_NAME            1  'b'
+          12  JUMP_FORWARD          6  'to 21'
 
+   6      15  LOAD_CONST            1  2
+          18  STORE_NAME            2  'd'
+        21_0  COME_FROM                '12'
+          21  LOAD_CONST            2  ''
+          24  RETURN_VALUE     

requirements-dev.txt

@@ -1,2 +1,3 @@
 pytest
 flake8
+hypothesis

requirements.txt

@@ -1 +1,2 @@
-spark_parser >= 1.1.0
+# Pick up stuff from setup.py
+-e .

setup.py

@@ -3,30 +3,27 @@
 """Setup script for the 'uncompyle6' distribution."""
 
 from __pkginfo__ import \
-    author,           author_email,                                  \
-    license,          long_description,                              \
-    modname,          packages,           py_modules,       scripts, \
-    short_desc,       web,                zip_safe
-
-__import__('pkg_resources')
-from setuptools import setup
-
-exec(open('uncompyle6/version.py').read())
+    author,           author_email,       install_requires,          \
+    license,          long_description,   classifiers,               \
+    entry_points,     modname,            py_modules,                \
+    short_desc,       VERSION,            web,                       \
+    zip_safe
 
+from setuptools import setup, find_packages
 setup(
        author             = author,
        author_email       = author_email,
-       # classifiers        = classifiers,
+       classifiers        = classifiers,
        description        = short_desc,
-       # install_requires   = install_requires,
+       entry_points       = entry_points,
+       install_requires   = install_requires,
        license            = license,
        long_description   = long_description,
-       py_modules         = py_modules,
        name               = modname,
-       packages           = packages,
+       packages           = find_packages(),
+       py_modules         = py_modules,
        test_suite         = 'nose.collector',
        url                = web,
        setup_requires     = ['nose>=1.0'],
-       scripts            = scripts,
        version            = VERSION,
        zip_safe           = zip_safe)

test/Makefile

@@ -20,36 +20,74 @@ check:
 	$(MAKE) check-$$PYTHON_VERSION
 
 #: Run working tests from Python 2.6 or 2.7
-check-2.6 check-2.7: check-bytecode-sans-3.5 check-2.7-ok
+check-2.6 check-2.7: check-bytecode-2 check-bytecode-3 check-bytecode-1  check-2.7-ok
+
+#: Run working tests from Python 3.1
+check-3.1: check-bytecode
+	$(PYTHON) test_pythonlib.py --bytecode-3.1 --weak-verify $(COMPILE)
+
+#: Run working tests from Python 3.2
+check-3.2: check-bytecode
+	$(PYTHON) test_pythonlib.py --bytecode-3.2 --weak-verify $(COMPILE)
 
 #: Run working tests from Python 3.3
 check-3.3: check-bytecode
-	$(PYTHON) test_pythonlib.py --bytecode-3.3 --verify $(COMPILE)
-
-#: Run working tests from Python 3.5
-check-3.5: check-bytecode
+	$(PYTHON) test_pythonlib.py --bytecode-3.3 --weak-verify $(COMPILE)
 
 #: Run working tests from Python 3.4
 check-3.4: check-bytecode check-3.4-ok check-2.7-ok
-	$(PYTHON) test_pythonlib.py --bytecode-3.4 --verify $(COMPILE)
+	$(PYTHON) test_pythonlib.py --bytecode-3.4 --weak-verify $(COMPILE)
+
+#: Run working tests from Python 3.5
+check-3.5: check-bytecode
+	$(PYTHON) test_pythonlib.py --bytecode-3.5 --verify $(COMPILE)
+
+#: Run working tests from Python 3.6
+check-3.6: check-bytecode
+	$(PYTHON) test_pythonlib.py --bytecode-3.6 --verify $(COMPILE)
 
 #: Check deparsing only, but from a different Python version
 check-disasm:
 	$(PYTHON) dis-compare.py
 
-#: Check deparsing bytecode only
+#: Check deparsing bytecode 1.x only
+check-bytecode-1:
+	$(PYTHON) test_pythonlib.py --bytecode-1.5
+
+#: Check deparsing bytecode 2.x only
 check-bytecode-2:
-	$(PYTHON) test_pythonlib.py --bytecode-2.5 --bytecode-2.6 --bytecode-2.7
+	$(PYTHON) test_pythonlib.py \
+	--bytecode-2.1 --bytecode-2.2 --bytecode-2.3 --bytecode-2.4 \
+        --bytecode-2.5 --bytecode-2.6 --bytecode-2.7 --bytecode-pypy2.7
 
-#: Check deparsing bytecode only
-check-bytecode:
-	$(PYTHON) test_pythonlib.py --bytecode-2.5 --bytecode-2.6 --bytecode-2.7 \
-        --bytecode-3.2 --bytecode-3.3 --bytecode-3.4 --bytecode-3.5
+#: Check deparsing bytecode 3.x only
+check-bytecode-3:
+	$(PYTHON) test_pythonlib.py --bytecode-3.1 --bytecode-3.2 --bytecode-3.3 \
+       --bytecode-3.4 --bytecode-3.5 --bytecode-pypy3.2
 
-#: Check deparsing bytecode only
-check-bytecode-sans-3.5:
-	$(PYTHON) test_pythonlib.py --bytecode-2.5 --bytecode-2.6 --bytecode-2.7 \
-        --bytecode-3.2 --bytecode-3.3 --bytecode-3.4
+#: Check deparsing bytecode that works running Python 2 and Python 3
+check-bytecode: check-bytecode-3
+	$(PYTHON) test_pythonlib.py \
+	--bytecode-2.1 --bytecode-2.2 --bytecode-2.3 --bytecode-2.4 \
+        --bytecode-2.5 --bytecode-2.6 --bytecode-2.7 \
+        --bytecode-pypy2.7 --bytecode-1
+
+
+#: Check deparsing Python 2.1
+check-bytecode-2.1:
+	$(PYTHON) test_pythonlib.py --bytecode-2.1
+
+#: Check deparsing Python 2.2
+check-bytecode-2.2:
+	$(PYTHON) test_pythonlib.py --bytecode-2.2
+
+#: Check deparsing Python 2.3
+check-bytecode-2.3:
+	$(PYTHON) test_pythonlib.py --bytecode-2.3
+
+#: Check deparsing Python 2.4
+check-bytecode-2.4:
+	$(PYTHON) test_pythonlib.py --bytecode-2.4
 
 #: Check deparsing Python 2.5
 check-bytecode-2.5:
@@ -63,6 +101,10 @@ check-bytecode-2.6:
 check-bytecode-2.7:
 	$(PYTHON) test_pythonlib.py --bytecode-2.7
 
+#: Check deparsing Python 3.1
+check-bytecode-3.1:
+	$(PYTHON) test_pythonlib.py --bytecode-3.1
+
 #: Check deparsing Python 3.2
 check-bytecode-3.2:
 	$(PYTHON) test_pythonlib.py --bytecode-3.2
@@ -79,10 +121,18 @@ check-bytecode-3.4:
 check-bytecode-3.5:
 	$(PYTHON) test_pythonlib.py --bytecode-3.5
 
+#: Check deparsing Python 3.6
+check-bytecode-3.6:
+	$(PYTHON) test_pythonlib.py --bytecode-3.6
+
 #: short tests for bytecodes only for this version of Python
 check-native-short:
 	$(PYTHON) test_pythonlib.py --bytecode-$(PYTHON_VERSION) --verify $(COMPILE)
 
+#: Run longer Python 2.6's lib files known to be okay
+check-2.6-ok:
+	$(PYTHON) test_pythonlib.py --ok-2.6 --verify $(COMPILE)
+
 #: Run longer Python 2.7's lib files known to be okay
 check-2.7-ok:
 	$(PYTHON) test_pythonlib.py --ok-2.7 --verify $(COMPILE)
@@ -95,6 +145,18 @@ check-3.2-ok:
 check-3.4-ok:
 	$(PYTHON) test_pythonlib.py --ok-3.4 --verify $(COMPILE)
 
+#: PyPy of some sort. E.g. [PyPy 5.0.1 with GCC 4.8.4]
+# Skip for now
+2.6:
+
+#: PyPy 5.0.x with Python 2.7 ...
+pypy-2.7 5.0 5.3:
+	$(PYTHON) test_pythonlib.py --bytecode-pypy2.7 --verify
+
+#: PyPy 2.4.x with Python 3.2 ...
+pypy-3.2 2.4:
+	$(PYTHON) test_pythonlib.py --bytecode-pypy3.2 --verify
+
 clean: clean-py-dis clean-dis clean-unverified
 
 clean-dis:
@@ -103,6 +165,6 @@ clean-dis:
 clean-unverified:
 	find . -name '*_unverified' -exec rm -v '{}' ';'
 
-#: Clean temporary compile/decompile/verify direcotries in /tmp
+#: Clean temporary compile/decompile/verify directories in /tmp
 clean-py-dis:
 	rm -fr /tmp/py-dis-* || true

test/add-test.py

@@ -2,11 +2,15 @@
 """ Trivial helper program to bytecompile and run an uncompile
 """
 import os, sys, py_compile
-assert len(sys.argv) == 2
-path = sys.argv[1]
-short = os.path.basename(path)
+assert len(sys.argv) >= 2
 version = sys.version[0:3]
-cfile =  "bytecode_%s/%s" % (version, short) + 'c'
-print("byte-compiling %s to %s" % (path, cfile))
-py_compile.compile(path, cfile)
-os.system("../bin/uncompyle6 -a -t %s" % cfile)
+for path in sys.argv[1:]:
+    short = os.path.basename(path)
+    if hasattr(sys, 'pypy_version_info'):
+        cfile =  "bytecode_pypy%s/%s" % (version, short) + 'c'
+    else:
+        cfile =  "bytecode_%s/%s" % (version, short) + 'c'
+    print("byte-compiling %s to %s" % (path, cfile))
+    py_compile.compile(path, cfile)
+    if isinstance(version, str) or version >= (2, 6, 0):
+        os.system("../bin/uncompyle6 -a -t %s" % cfile)