Merge conflicts

uncompile -> uncompyle6

.gitignore

@@ -17,4 +17,5 @@
 __pycache__
 build
 /.venv*
-/.idea
+/.idea
+/.hypothesis

.travis.yml

@@ -3,13 +3,7 @@ language: python
 sudo: false
 
 python:
-  - '3.5'
-  - '2.7.12'
-  - '2.6'
-  - '3.3'
-  - '3.4'
-  - '3.2'
-  - '3.6'
+  - '2.7' # this is a cheat here because travis doesn't do 2.4-2.6
 
 install:
 - pip install -e .

ChangeLog

@@ -1,8 +1,254 @@
+2017-10-11  rocky <rb@dustyfeet.com>
+
+	* admin-tools/check-newer-versions.sh,
+	admin-tools/check-older-versions.sh,
+	admin-tools/how-to-make-a-release.txt,
+	admin-tools/make-dist-newer.sh, admin-tools/make-dist-older.sh,
+	admin-tools/pyenv-newer-versions, admin-tools/pyenv-older-versions,
+	uncompyle6/parsers/parse37.py, uncompyle6/scanners/scanner37.py,
+	uncompyle6/version.py: More administrivia
+
+2017-10-11  rocky <rb@dustyfeet.com>
+
+	* admin-tools/README.md, admin-tools/check-newer-versions.sh,
+	admin-tools/check-older-versions.sh,
+	admin-tools/how-to-make-a-release.txt,
+	admin-tools/make-dist-newer.sh, admin-tools/make-dist-older.sh,
+	admin-tools/pyenv-newer-versions, admin-tools/pyenv-older-versions,
+	admin-tools/setup-master.sh, admin-tools/setup-python-2.4.sh: Some
+	admin tools I use
+
+2017-10-11  rocky <rb@dustyfeet.com>
+
+	* uncompyle6/parser.py: Remove creaping Python 2.6ism
+
+2017-10-10  rocky <rb@dustyfeet.com>
+
+	* pytest/test_grammar.py: Sync with master
+
+2017-10-10  rocky <rb@dustyfeet.com>
+
+	* pytest/test_grammar.py, uncompyle6/parsers/parse26.py,
+	uncompyle6/parsers/parse27.py, uncompyle6/parsers/parse34.py,
+	uncompyle6/parsers/parse35.py, uncompyle6/parsers/parse36.py,
+	uncompyle6/parsers/parse37.py: Sync with master
+
+2017-10-10  rocky <rb@dustyfeet.com>
+
+	* uncompyle6/semantics/fragments.py: Sync with master
+
+2017-10-10  rocky <rb@dustyfeet.com>
+
+	* NEWS, __pkginfo__.py, pytest/test_grammar.py,
+	uncompyle6/parser.py, uncompyle6/parsers/parse15.py,
+	uncompyle6/parsers/parse2.py, uncompyle6/parsers/parse21.py,
+	uncompyle6/parsers/parse22.py, uncompyle6/parsers/parse23.py,
+	uncompyle6/parsers/parse24.py, uncompyle6/parsers/parse25.py,
+	uncompyle6/parsers/parse26.py, uncompyle6/parsers/parse27.py,
+	uncompyle6/parsers/parse3.py, uncompyle6/parsers/parse32.py,
+	uncompyle6/parsers/parse34.py, uncompyle6/parsers/parse35.py,
+	uncompyle6/parsers/parse36.py, uncompyle6/parsers/parse37.py,
+	uncompyle6/scanners/scanner22.py, uncompyle6/scanners/scanner26.py,
+	uncompyle6/scanners/scanner27.py, uncompyle6/scanners/scanner3.py,
+	uncompyle6/scanners/scanner36.py, uncompyle6/scanners/tok.py,
+	uncompyle6/semantics/check_ast.py,
+	uncompyle6/semantics/make_function.py,
+	uncompyle6/semantics/pysource.py, uncompyle6/verify.py,
+	uncompyle6/version.py: Sync with master
+
+2017-10-10  rocky <rb@dustyfeet.com>
+
+	* : Merge commit '1d7a3c6444eab5a02d899f789f2a57cfdcbc5a84' into
+	python-2.4
+
+2017-10-05  rocky <rb@dustyfeet.com>
+
+	* test/simple_source/branching/02_ifelse_lambda.py: One more test
+
+2017-10-05  rocky <rb@dustyfeet.com>
+
+	* .gitignore, pytest/test_grammar.py, uncompyle6/parser.py,
+	uncompyle6/parsers/parse26.py, uncompyle6/parsers/parse27.py,
+	uncompyle6/parsers/parse3.py, uncompyle6/semantics/consts.py,
+	uncompyle6/semantics/pysource.py: Sync with master
+
+2017-10-03  rocky <rb@dustyfeet.com>
+
+	* uncompyle6/parsers/parse2.py, uncompyle6/parsers/parse24.py,
+	uncompyle6/parsers/parse26.py: handle newer parser reduction
+	behavior
+
+2017-10-03  rocky <rb@dustyfeet.com>
+
+	* uncompyle6/semantics/fragments.py,
+	uncompyle6/semantics/pysource.py: Go over table-semantics
+	description yet again
+
+2017-10-02  rocky <rb@dustyfeet.com>
+
+	* uncompyle6/parsers/parse2.py, uncompyle6/parsers/parse3.py: Sync
+	with master
+
+2017-09-30  rocky <rb@dustyfeet.com>
+
+	* uncompyle6/parser.py, uncompyle6/scanners/scanner2.py,
+	uncompyle6/scanners/scanner3.py: Document hacky customize arg count
+	better.
+
+2017-09-26  rocky <rb@dustyfeet.com>
+
+	* README.rst: Word hacking
+
+2017-09-26  rocky <rb@dustyfeet.com>
+
+	* ChangeLog, NEWS: Get ready for release 2.12.0
+
+2017-09-26  rocky <rb@dustyfeet.com>
+
+	* uncompyle6/parsers/parse3.py: Annotation field can be unicode...  When deparsing Python 3.x from Python 2.
+
+2017-09-26  rocky <rb@dustyfeet.com>
+
+	* uncompyle6/parsers/parse3.py: No unicode in Python3.  but we need it in Python2. The bug was probably introduced as a
+	result of recent Python code type unteroperability canonicalization
+
+2017-09-26  rocky <rb@dustyfeet.com>
+
+	* uncompyle6/parsers/parse3.py: Pyton 3.1 Annotation args can be
+	unicode?
+
+2017-09-25  rocky <rb@dustyfeet.com>
+
+	* __pkginfo__.py: Require xdis 3.6.0 or greater
+
+2017-09-25  rocky <rb@dustyfeet.com>
+
+	* : commit 0654aed6c823d0bb20abdc866481ca5950db72f7 Author: rocky
+	<rb@dustyfeet.com> Date:   Thu Sep 21 11:29:17 2017 -0400
+
+2017-09-25  rocky <rb@dustyfeet.com>
+
+	* : Adjust for xdis opcode JUMP_OPS. release 2.12.0
+
+2017-09-21  rocky <rb@dustyfeet.com>
+
+	* pytest/test_pysource.py: Python 3 compatibility
+
+2017-09-21  rocky <rb@dustyfeet.com>
+
+	* pytest/test_pysource.py, uncompyle6/semantics/consts.py,
+	uncompyle6/semantics/fragments.py, uncompyle6/semantics/pysource.py: 
+	Unit test for format-specifiers
+
+2017-09-21  rocky <rb@dustyfeet.com>
+
+	* pytest/test_pysource.py, uncompyle6/semantics/consts.py,
+	uncompyle6/semantics/fragments.py, uncompyle6/semantics/pysource.py: 
+	Unit test for format-specifiers And in the process we catch some small bugs
+
+2017-09-20  rocky <rb@dustyfeet.com>
+
+	* uncompyle6/semantics/fragments.py,
+	uncompyle6/semantics/pysource.py: Tidy pysource and fragments
+
+2017-09-20  rocky <rb@dustyfeet.com>
+
+	* uncompyle6/semantics/fragments.py,
+	uncompyle6/semantics/pysource.py: Tidy pysource and fragments a
+	little more
+
+2017-09-20  rocky <rb@dustyfeet.com>
+
+	* uncompyle6/semantics/consts.py: Tidy/regularize table entry
+	formatting
+
+2017-09-20  rocky <rb@dustyfeet.com>
+
+	* uncompyle6/semantics/consts.py: Tidy/regularize table entry
+	formatting
+
+2017-09-20  rocky <rb@dustyfeet.com>
+
+	* test/test_pythonlib.py, uncompyle6/semantics/pysource.py: Small
+	fixes test_pyenvlib.py: it is sys.exit(), not exit() pysource.py:
+	reinstate nod type of async_func_call
+
+2017-09-20  rocky <rb@dustyfeet.com>
+
+	* test/test_pythonlib.py, uncompyle6/semantics/pysource.py: small
+	fixes...  test_pythonlib.py: it is sys.exit not exit pysource.py: restore node
+	type on async_call function
+
+2017-09-20  rocky <rb@dustyfeet.com>
+
+	* uncompyle6/semantics/consts.py, uncompyle6/semantics/pysource.py: 
+	More small doc changes
+
+2017-09-20  rocky <rb@dustyfeet.com>
+
+	* pytest/test_pysource.py, uncompyle6/semantics/pysource.py: Start
+	pysource unit test
+
+2017-09-20  rocky <rb@dustyfeet.com>
+
+	* pytest/test_pysource.py, uncompyle6/semantics/pysource.py: Update
+	Table-driven info...  Start a pysource unit test.
+
+2017-09-17  rocky <rb@dustyfeet.com>
+
+	* uncompyle6/semantics/fragments.py,
+	uncompyle6/semantics/pysource.py: emgine -> template_engine
+
+2017-09-17  rocky <rb@dustyfeet.com>
+
+	* uncompyle6/semantics/fragments.py,
+	uncompyle6/semantics/pysource.py: engine -> template_engine
+
+2017-09-13  rocky <rb@dustyfeet.com>
+
+	* test/Makefile: Need weak-verification on 3.4 for now
+
+2017-09-10  rocky <rb@dustyfeet.com>
+
+	* uncompyle6/semantics/fragments.py: Revert one of the changes
+	pending a better fix
+
+2017-09-10  rocky <rb@dustyfeet.com>
+
+	* uncompyle6/semantics/fragments.py,
+	uncompyle6/semantics/pysource.py: More semantic action cleanup
+
+2017-09-10  rocky <rb@dustyfeet.com>
+
+	* uncompyle6/scanners/scanner3.py, uncompyle6/scanners/tok.py: Match
+	Python 3.4's terms a little names better
+
+2017-09-09  rocky <rb@dustyfeet.com>
+
+	* uncompyle6/scanners/tok.py: Revert last revert
+
+2017-09-09  rocky <rb@dustyfeet.com>
+
+	* uncompyle6/scanners/tok.py: Revert last change
+
+2017-09-09  rocky <rb@dustyfeet.com>
+
+	* uncompyle6/scanners/tok.py: New-style Python classes only, please.
+
 2017-08-31  rocky <rb@dustyfeet.com>
 
-	* NEWS, README.rst, uncompyle6/parsers/parse37.py,
-	uncompyle6/semantics/make_function.py, uncompyle6/version.py: Get
-	ready for release 2.11.5
+	* uncompyle6/scanner.py, uncompyle6/scanners/scanner37.py: Skeletal
+	support for Python 3.7 Largely failing though.
+
+2017-08-31  rocky <rb@dustyfeet.com>
+
+	* : commit 356ea6c7705a557cb3e725d1aca8589dd62b5cdf Author: rocky
+	<rb@dustyfeet.com> Date:   Thu Aug 31 09:50:48 2017 -0400
+
+2017-08-31  rocky <rb@dustyfeet.com>
+
+	* : commit 4d5843851543bfb3c97fc3c49036f1a971fc1d66 Author: rocky
+	<rb@dustyfeet.com> Date:   Thu Aug 31 08:53:58 2017 -0400
 
 2017-08-15  rocky <rb@dustyfeet.com>
 
@@ -10,8 +256,8 @@
 
 2017-08-15  rocky <rb@dustyfeet.com>
 
-	* ChangeLog, NEWS, uncompyle6/version.py: Get ready for release
-	2.11.4
+	* : commit c54a47b15f85be50d2278aa79fd514eb08580e65 Author: rocky
+	<rb@dustyfeet.com> Date:   Tue Aug 15 10:47:12 2017 -0400
 
 2017-08-15  rocky <rb@dustyfeet.com>
 
@@ -19,6 +265,12 @@
 	uncompyle6/scanner.py: Misc cleanups...  remove code now in xdis require at least xdis 3.5.4 PyPy tolerance
 	in validate testing
 
+2017-08-13  rocky <rb@dustyfeet.com>
+
+	* ChangeLog, README.rst, __pkginfo__.py, pytest/test_basic.py,
+	uncompyle6/parser.py, uncompyle6/scanner.py: Allow version to be
+	string...  in get_python_parser and get_scanner
+
 2017-08-13  rocky <rb@dustyfeet.com>
 
 	* pytest/test_basic.py, uncompyle6/parser.py, uncompyle6/scanner.py: 
@@ -27,7 +279,8 @@
 
 2017-08-10  rocky <rb@dustyfeet.com>
 
-	* README.rst: Link typo Name is trepan2 now not trepan
+	* : commit 503039ab51f004cca27a9da43ff22b031cc486dc Author: rocky
+	<rb@dustyfeet.com> Date:   Thu Aug 10 09:41:48 2017 -0400
 
 2017-08-09  rocky <rb@dustyfeet.com>
 
@@ -36,6 +289,34 @@
 	release 2.11.3 need xdis 3.5.1 for now. Adjust for xdis "is-not" which we need as
 	"is not"
 
+2017-08-09  rocky <rb@dustyfeet.com>
+
+	* uncompyle6/semantics/consts.py: xdis "is not" is now "is-not"
+
+2017-08-09  rocky <rb@dustyfeet.com>
+
+	* ChangeLog: Get ready for release 2.11.3
+
+2017-08-09  rocky <rb@dustyfeet.com>
+
+	* : commit dc627d13b8455ded4bf708a596bb466f9df9bf7b Author: rocky
+	<rb@dustyfeet.com> Date:   Wed Aug 9 21:19:30 2017 -0400
+
+2017-08-03  rocky <rb@dustyfeet.com>
+
+	* pytest/test_deparse.py, pytest/test_docstring.py,
+	pytest/test_fjt.py, pytest/test_single_compile.py,
+	pytest/validate.py, uncompyle6/scanners/scanner3.py,
+	uncompyle6/scanners/scanner30.py: Python 2.4 comptiability and ...  exception match -> exception-match
+
+2017-08-02  rocky <rb@dustyfeet.com>
+
+	* __pkginfo__.py: Bump xdis
+
+2017-08-02  rocky <rb@dustyfeet.com>
+
+	* __pkginfo__.py: Remove six from python 2.4/2.5
+
 2017-08-02  rocky <rb@dustyfeet.com>
 
 	* __pkginfo__.py: Revert commit to wrong branch
@@ -44,6 +325,20 @@
 
 	* __pkginfo__.py: Remove six from Python-2.4/2.5 package
 
+2017-08-02  rocky <rb@dustyfeet.com>
+
+	* uncompyle6/scanners/scanner3.py: in xdis "exception match" is now
+	"exception-match"
+
+2017-08-02  rocky <rb@dustyfeet.com>
+
+	* __pkginfo__.py: Python 2.4 doesn't do six
+
+2017-08-01  rocky <rb@dustyfeet.com>
+
+	* pytest/validate.py, test/dis-compare.py,
+	test/simple-uncompyle-code-test.py: Python 2.4 compatibility
+
 2017-07-17  rocky <rb@dustyfeet.com>
 
 	* __pkginfo__.py, uncompyle6/scanners/scanner2.py,
@@ -108,10 +403,8 @@
 
 2017-06-25  rocky <rb@dustyfeet.com>
 
-	* ChangeLog, NEWS, test/simple_source/bug31/04_def_annotate.py,
-	uncompyle6/semantics/make_function.py,
-	uncompyle6/semantics/pysource.py: 3.x funciton and annotation bug
-	fixes
+	* : commit 9c072a6a423d8379712296dbcd499c772ba7ef59 Author: rocky
+	<rb@dustyfeet.com> Date:   Sun Jun 25 18:44:50 2017 -0400
 
 2017-06-25  rocky <rb@dustyfeet.com>
 
@@ -136,8 +429,14 @@
 
 2017-06-18  rocky <rb@dustyfeet.com>
 
-	* ChangeLog, NEWS, uncompyle6/version.py: Get ready for release
-	2.11.0
+	* uncompyle6/scanners/scanner2.py, uncompyle6/scanners/scanner3.py,
+	uncompyle6/semantics/fragments.py,
+	uncompyle6/semantics/make_function.py: More merge fixups from master
+
+2017-06-18  rocky <rb@dustyfeet.com>
+
+	* : commit af10f99776b142c44fb4507033fb3220b5f57910 Author: rocky
+	<rb@dustyfeet.com> Date:   Sun Jun 18 15:22:27 2017 -0400
 
 2017-06-13  rocky <rb@dustyfeet.com>
 
@@ -232,7 +531,20 @@
 
 2017-06-03  rocky <rb@dustyfeet.com>
 
-	* .travis.yml: Streamline .travis.yml a little bit
+	* pytest/validate.py: 2.4 doesn't do six
+
+2017-06-03  rocky <rb@dustyfeet.com>
+
+	* appveyor.yml: Nope it (appveyor) doesn't.
+
+2017-06-03  rocky <rb@dustyfeet.com>
+
+	* __pkginfo__.py, appveyor.yml: Administrivia See if appveyor will handle 2.5
+
+2017-06-03  rocky <rb@dustyfeet.com>
+
+	* : commit 7c299fbf3777c452d6a10075964961783f510699 Author: rocky
+	<rb@dustyfeet.com> Date:   Sat Jun 3 05:38:05 2017 -0400
 
 2017-06-03  rocky <rb@dustyfeet.com>
 
@@ -259,8 +571,12 @@
 
 2017-05-30  rocky <rb@dustyfeet.com>
 
-	* ChangeLog, NEWS, uncompyle6/version.py: Get ready for release
-	2.10.0
+	* pytest/test_function_call.py: No decorators in Python < 2.6
+
+2017-05-30  rocky <rb@dustyfeet.com>
+
+	* : commit ad98fae3d4b0b83f65b15da8201e33c0ee6dab17 Author: rocky
+	<rb@dustyfeet.com> Date:   Tue May 30 01:26:52 2017 -0400
 
 2017-05-30  rocky <rb@dustyfeet.com>
 
@@ -499,15 +815,25 @@
 	* ChangeLog, NEWS, uncompyle6/version.py: Get ready for release
 	2.9.11
 
+2017-05-06  rocky <rb@dustyfeet.com>
+
+	* ChangeLog, NEWS, uncompyle6/version.py: Get ready for release
+	2.9.11
+
 2017-05-06  rocky <rb@dustyfeet.com>
 
 	* test/Makefile: fix PYTHON variable setting in test/Makefile
 
 2017-05-06  rocky <rb@dustyfeet.com>
 
-	* test/simple_source/bug32/01_try_except_raise.py,
-	test/simple_source/bug32/03_if.py, uncompyle6/parsers/parse32.py,
-	uncompyle6/parsers/parse33.py: Fix more Python3.2 parser errors
+	* test/Makefile, uncompyle6/scanners/scanner2.py,
+	uncompyle6/scanners/scanner3.py, uncompyle6/semantics/fragments.py: 
+	Sync with master
+
+2017-05-06  rocky <rb@dustyfeet.com>
+
+	* : commit 4a4782290490187ac2fcaaecd3ca808f933722b2 Author: rocky
+	<rb@dustyfeet.com> Date:   Sat May 6 05:25:56 2017 -0400
 
 2017-05-05  rocky <rb@dustyfeet.com>
 
@@ -695,6 +1021,11 @@
 	haphazard way using real flow-control analysis. Hopefully that's on
 	the way.  In the meantime we have this hack.
 
+2017-04-14  rocky <rb@dustyfeet.com>
+
+	* : commit 7e8f7ba67431725fceec08344934c929a517efc5 Author: rocky
+	<rb@dustyfeet.com> Date:   Fri Apr 14 05:42:44 2017 -0400
+
 2017-04-14  rocky <rb@dustyfeet.com>
 
 	* test/simple_source/bug27+/03_if_1_else.py,
@@ -759,9 +1090,10 @@
 	uncompyle6/parsers/parse2.py, uncompyle6/parsers/parse3.py,
 	uncompyle6/parsers/parse35.py: Add more while1else grammar rules Towards addressing issue #93
 
-2017-04-09  rocky <rb@dustyfeet.com>
+2017-04-10  rocky <rb@dustyfeet.com>
 
-	* : One more FUNCTION_VAR test for 3.3
+	* : commit b9703cf6b41138b717c282fc791c08d807692b07 Author: rocky
+	<rb@dustyfeet.com> Date:   Sun Apr 9 06:58:41 2017 -0400
 
 2017-04-09  rocky <rb@dustyfeet.com>
 
@@ -872,11 +1204,8 @@
 
 2017-03-01  rocky <rb@dustyfeet.com>
 
-	* uncompyle6/scanner.py, uncompyle6/scanners/scanner2.py,
-	uncompyle6/scanners/scanner3.py, uncompyle6/verify.py: COME_FROM for
-	3.x POP_EXCEPT, DRY with op_name() ...  Start adding COME_FROMs for POP_EXCEPT in preparation for getting
-	tryelse blocks correct.  Simpler opname access functions:   - self.op_name(op) is self.opc.opname[op]   - self.op_name_from_offset(offset) is
-	  self.opc.opname[self.code[offset]] verify.py: not all offsets are ints
+	* : commit 160ec0d9cc5fe347f6e8bdb69515a28c76cfb368 Author: rocky
+	<rb@dustyfeet.com> Date:   Wed Mar 1 05:50:31 2017 -0500
 
 2017-02-28  rocky <rb@dustyfeet.com>
 
@@ -900,13 +1229,17 @@
 
 2017-02-25  rocky <rb@dustyfeet.com>
 
-	* ChangeLog, NEWS, __pkginfo__.py, uncompyle6/version.py: Get ready
-	for release 2.9.10
+	* : commit 1e3ea60055027dfd3f098661ac4f5979c5c48f7e Author: rocky
+	<rb@dustyfeet.com> Date:   Sat Feb 25 20:18:03 2017 -0500
 
 2017-02-25  rocky <rb@dustyfeet.com>
 
-	* uncompyle6/parser.py, uncompyle6/parsers/parse26.py: Python 2.6
-	parsing bugs ..  and some parser list nonterminal cleanup
+	* uncompyle6/parser.py: Python <= 2.6 grammar fixes
+
+2017-02-25  rocky <rb@dustyfeet.com>
+
+	* : commit 2fbbc728b10f0d3a754165708584bd80d33bc7f9 Author: rocky
+	<rb@dustyfeet.com> Date:   Sat Feb 25 04:45:10 2017 -0500
 
 2017-02-24  rocky <rb@dustyfeet.com>
 
@@ -920,9 +1253,15 @@
 	uncompyle6/parsers/parse25.py: Python 2.5 wasn't handling tryelse
 	properly
 
-2017-02-20  rocky <rb@dustyfeet.com>
+2017-02-22  rocky <rb@dustyfeet.com>
 
-	* : New test doesn't --verify correctly. Sigh.
+	* test/Makefile, test/simple_source/bug25/02_try_else.py,
+	uncompyle6/parsers/parse25.py: Python 2.5 was missing try else stmt
+
+2017-02-22  rocky <rb@dustyfeet.com>
+
+	* : commit b043f6bafc9b9ae26e64dc0f1441d7abae894c37 Author: rocky
+	<rb@dustyfeet.com> Date:   Mon Feb 20 09:22:01 2017 -0500
 
 2017-02-20  rocky <rb@dustyfeet.com>
 
@@ -961,6 +1300,10 @@
 	* test/simple_source/bug22/01_ops.py, test/test_pythonlib.py: Beef
 	up grammar coverage
 
+2017-02-10  rocky <rb@dustyfeet.com>
+
+	* test/Makefile: Group coverage Makefile targets
+
 2017-01-29  rocky <rb@dustyfeet.com>
 
 	* test/Makefile, test/simple_source/bug22/01_ops.py,
@@ -968,9 +1311,24 @@
 	uncompyle6/semantics/pysource.py: Changes based on grammar coverage
 	info
 
-2017-01-29  R. Bernstein <rocky@users.noreply.github.com>
+2017-01-29  rocky <rb@dustyfeet.com>
 
-	* : Merge pull request #83 from rocky/coverage Coverage
+	* test/Makefile, test/simple_source/bug22/01_ops.py,
+	uncompyle6/parsers/parse25.py, uncompyle6/semantics/consts.py,
+	uncompyle6/semantics/pysource.py: Changes based on coverage
+	information
+
+2017-01-29  rocky <rb@dustyfeet.com>
+
+	* : commit 9348411056cbe809e07c4ef341effa17bca90e2f Merge: 3dc766d
+	e71dd01 Author: R. Bernstein <rocky@users.noreply.github.com> Date:
+	Sun Jan 29 21:54:45 2017 -0500
+
+2017-01-29  rocky <rb@dustyfeet.com>
+
+	* test/Makefile, test/simple_source/bug22/01_ops.py,
+	test/test_pyenvlib.py, test/test_pythonlib.py,
+	uncompyle6/semantics/consts.py: Simplfy getting coverage consts.py: notes on versions use which ops
 
 2017-01-29  rocky <rb@dustyfeet.com>
 
@@ -1074,6 +1432,10 @@
 	* uncompyle6/__init__.py: sys.recursionlimit is optional, not
 	essential
 
+2017-01-11  rocky <rb@dustyfeet.com>
+
+	* NEWS, uncompyle6/version.py: Get ready for release 2.9.9
+
 2017-01-11  rocky <rb@dustyfeet.com>
 
 	* : commit b131c20e99514d3a969a51e841d3a823017f1beb Author: rocky
@@ -1083,9 +1445,22 @@
 
 	* ChangeLog, NEWS: Get ready for release 2.10.9
 
+2017-01-11  rocky <rb@dustyfeet.com>
+
+	* uncompyle6/parsers/parse3.py, uncompyle6/scanner.py,
+	uncompyle6/semantics/make_function.py, uncompyle6/version.py: Merge
+	changes ...  * str() in Python 2.4 doesn't detect unicode.  * index() doesn't work on tuples * ifelse change
+
+2017-01-11  rocky <rb@dustyfeet.com>
+
+	* : commit 7ece296f7638e71fad1117b940f7ffddbe095b1f Merge: 78a5b62
+	5035d54 Author: R. Bernstein <rocky@users.noreply.github.com> Date:
+	Wed Jan 11 07:10:23 2017 -0500
+
 2017-01-11  R. Bernstein <rocky@users.noreply.github.com>
 
-	* : Merge pull request #79 from rocky/revert-78-patch-1 Revert "fix bug : not generate all files when use "-ro""
+	* uncompyle6/main.py: Revert "fix bug : not generate all files when
+	use "-ro""
 
 2017-01-11  R. Bernstein <rocky@users.noreply.github.com>
 
@@ -1184,6 +1559,16 @@
 	* uncompyle6/parsers/parse35.py, uncompyle6/scanners/scanner3.py: 
 	Python 3.5 continue detection bug
 
+2017-01-02  rocky <rb@dustyfeet.com>
+
+	* uncompyle6/verify.py: 2.4 verify hacks
+
+2017-01-02  rocky <rb@dustyfeet.com>
+
+	* : commit a7d93e88b4e0dfd6876a7a31bd201a0e40f24bea Merge: 9891494
+	136f42a Author: rocky <rb@dustyfeet.com> Date:   Mon Jan 2 05:39:13
+	2017 -0500
+
 2017-01-01  rocky <rb@dustyfeet.com>
 
 	* uncompyle6/scanners/scanner3.py: add come_from for setup_finally
@@ -1198,6 +1583,14 @@
 
 	* README.rst: Note how to verify correctness ...  with --verify, --weak-verify and cross checking with pycdc
 
+2016-12-31  rocky <rb@dustyfeet.com>
+
+	* uncompyle6/version.py: We are version 2.9.9
+
+2016-12-31  rocky <rb@dustyfeet.com>
+
+	* uncompyle6/main.py: 2.7->2.4 conversion
+
 2016-12-31  rocky <rb@dustyfeet.com>
 
 	* ChangeLog, NEWS, uncompyle6/version.py: Get ready for release
@@ -1207,6 +1600,13 @@
 
 	* uncompyle6/parsers/parse26.py: 2.x list_if may have a THEN in it
 
+2016-12-31  rocky <rb@dustyfeet.com>
+
+	* test/Makefile, uncompyle6/main.py, uncompyle6/parsers/parse26.py,
+	uncompyle6/scanners/scanner2.py, uncompyle6/scanners/scanner26.py,
+	uncompyle6/scanners/scanner3.py, uncompyle6/semantics/pysource.py,
+	uncompyle6/verify.py: Merge master branche Handle 2.2 list_if
+
 2016-12-31  rocky <rb@dustyfeet.com>
 
 	* uncompyle6/scanners/scanner3.py: Towards fixing a Python 3.3
@@ -1235,10 +1635,9 @@
 
 	* : Merge pull request #73 from rocky/then-crap Add THEN token to improve Python 2.2-2.6 control flow detection
 
-2016-12-28  rocky <rb@dustyfeet.com>
+2016-12-29  R. Bernstein <rocky@users.noreply.github.com>
 
-	* uncompyle6/parsers/parse3.py, uncompyle6/scanners/tok.py: Misc
-	bugs
+	* : Merge pull request #72 from rocky/master THEN psuedo-ops for Python 2.x
 
 2016-12-28  rocky <rb@dustyfeet.com>
 
@@ -1265,8 +1664,7 @@
 
 2016-12-27  rocky <rb@dustyfeet.com>
 
-	* uncompyle6/scanners/scanner2.py: Make 2.6 and 2.7 ingest more
-	alike
+	* : Merge commit '9b1dd0f' into python-2.4
 
 2016-12-26  rocky <rb@dustyfeet.com>
 
@@ -1286,6 +1684,11 @@
 	* uncompyle6/parsers/parse25.py: fix bug in using python2 AST rules
 	in python 2.5
 
+2016-12-26  rocky <rb@dustyfeet.com>
+
+	* uncompyle6/parsers/parse25.py: Bug in using python2 ast checking
+	in python 2.5
+
 2016-12-26  rocky <rb@dustyfeet.com>
 
 	* : commit f1a947f106b231fb1480ba301b15e3ceaf78c94f Author: rocky
@@ -1298,15 +1701,19 @@
 	uncompyle6/verify.py: Scanner call fixes. NAME_MODULE removal for
 	<=2.4
 
-2016-12-24  rocky <rb@dustyfeet.com>
+2016-12-25  rocky <rb@dustyfeet.com>
 
-	* uncompyle6/parsers/astnode.py, uncompyle6/parsers/parse2.py,
-	uncompyle6/parsers/parse26.py, uncompyle6/parsers/parse3.py,
-	uncompyle6/parsers/parse36.py, uncompyle6/scanners/scanner15.py,
-	uncompyle6/scanners/scanner2.py, uncompyle6/scanners/scanner21.py,
-	uncompyle6/scanners/scanner22.py,
-	uncompyle6/semantics/fragments.py, uncompyle6/semantics/pysource.py: 
-	Lint
+	* uncompyle6/main.py, uncompyle6/scanners/scanner15.py,
+	uncompyle6/scanners/scanner21.py, uncompyle6/scanners/scanner22.py,
+	uncompyle6/scanners/scanner23.py, uncompyle6/scanners/scanner24.py,
+	uncompyle6/semantics/pysource.py, uncompyle6/verify.py: Removing
+	NAME_MODULE, lint and bug fixes scanner*.py: show_asm param is optional verify.py: call correct
+	scanners main.py, verify.py: Use older Python print statements
+
+2016-12-25  rocky <rb@dustyfeet.com>
+
+	* : commit e3f4beeb74e33d5b404094765cc63040f62a0b41 Author: rocky
+	<rb@dustyfeet.com> Date:   Sat Dec 24 07:45:02 2016 -0500
 
 2016-12-24  rocky <rb@dustyfeet.com>
 
@@ -1327,22 +1734,34 @@
 
 2016-12-18  rocky <rb@dustyfeet.com>
 
-	* pytest/.gitignore, test/simple_source/bug25/02_try_else.py,
-	uncompyle6/parsers/parse25.py, uncompyle6/parsers/parse26.py: Python
-	2.5 mistaken try/else
+	* : commit c7c0a989829a9a625333665516387c1177c611c2 Author: rocky
+	<rb@dustyfeet.com> Date:   Sun Dec 18 00:56:07 2016 -0500
 
 2016-12-17  rocky <rb@dustyfeet.com>
 
 	* uncompyle6/scanners/scanner2.py, uncompyle6/scanners/scanner25.py: 
 	show-asm on python2.5 is optional make scanner2 look a little more like scanner3
 
+2016-12-17  rocky <rb@dustyfeet.com>
+
+	* uncompyle6/scanners/scanner2.py, uncompyle6/scanners/scanner25.py: 
+	show-asm on python2.5 is optional Make scanner2 a little more like scanner3.
+
+2016-12-17  rocky <rb@dustyfeet.com>
+
+	* uncompyle6/parser.py, uncompyle6/scanner.py,
+	uncompyle6/scanners/scanner2.py, uncompyle6/scanners/scanner3.py,
+	uncompyle6/semantics/fragments.py: Python 2.6/2.7 tolerance in
+	Python 2.4 branch
+
 2016-12-16  rocky <rb@dustyfeet.com>
 
 	* NEWS: Release 2.9.8 news
 
 2016-12-16  rocky <rb@dustyfeet.com>
 
-	* __pkginfo__.py, uncompyle6/version.py: Get ready for release 2.9.8
+	* : commit 13d5cd1a588b7f4f2c233c436ce6b0b39db9950e Author: rocky
+	<rb@dustyfeet.com> Date:   Fri Dec 16 22:42:46 2016 -0500
 
 2016-12-16  rocky <rb@dustyfeet.com>
 
@@ -1410,15 +1829,12 @@
 
 2016-12-04  rocky <rb@dustyfeet.com>
 
-	* ChangeLog, NEWS, uncompyle6/main.py, uncompyle6/parser.py,
-	uncompyle6/parsers/parse26.py, uncompyle6/parsers/parse3.py,
-	uncompyle6/parsers/parse34.py, uncompyle6/parsers/parse35.py,
-	uncompyle6/scanner.py, uncompyle6/scanners/scanner2.py,
-	uncompyle6/scanners/scanner23.py, uncompyle6/scanners/scanner24.py,
-	uncompyle6/scanners/scanner3.py, uncompyle6/scanners/tok.py,
-	uncompyle6/semantics/make_function.py,
-	uncompyle6/semantics/pysource.py, uncompyle6/verify.py,
-	uncompyle6/version.py: Get ready for release 2.9.7 Some of the many lint things. Linting is kind of stupid though.
+	* ChangeLog: Get ready for release 2.9.7
+
+2016-12-04  rocky <rb@dustyfeet.com>
+
+	* : commit d22931cb49f0e28a0fbe48a7c1526b1f170a5b52 Author: rocky
+	<rb@dustyfeet.com> Date:   Sun Dec 4 07:31:34 2016 -0500
 
 2016-11-28  rocky <rb@dustyfeet.com>
 
@@ -1469,11 +1885,34 @@
 	* uncompyle6/semantics/pysource.py: Better line number tracking Indent Python 2 list comprehensions, albeit badly.  DRY code a
 	little via indent_if_source_nl
 
+2016-11-24  rocky <rb@dustyfeet.com>
+
+	* circle.yml: CircleCI build
+
+2016-11-24  rocky <rb@dustyfeet.com>
+
+	* uncompyle6/parsers/parse3.py: Remove dup Python 3 grammar rule
+
 2016-11-24  rocky <rb@dustyfeet.com>
 
 	* uncompyle6/parsers/parse3.py, uncompyle6/scanners/scanner2.py: 
 	<2.7 "if" detection and dup Python 3 grammar rule
 
+2016-11-24  rocky <rb@dustyfeet.com>
+
+	* test/test_pyenvlib.py, uncompyle6/linenumbers.py,
+	uncompyle6/main.py, uncompyle6/scanners/scanner2.py,
+	uncompyle6/semantics/make_function.py,
+	uncompyle6/semantics/pysource.py, uncompyle6/verify.py: Bug in 2.4
+	"if" dectection and...  Wrong language used in old-style exceptions: use "except Error,e"
+	not "except Error(e)""
+
+2016-11-24  rocky <rb@dustyfeet.com>
+
+	* __pkginfo__.py, pytest/test_grammar.py, uncompyle6/parser.py,
+	uncompyle6/parsers/parse26.py:  Python 2.6 grammary bug and..  __pkginfo.py__: Bump spark_parser
+	version for parse_flags 'dups'
+
 2016-11-23  rocky <rb@dustyfeet.com>
 
 	* __pkginfo__.py, pytest/test_grammar.py, uncompyle6/parser.py,
@@ -1485,8 +1924,17 @@
 
 2016-11-23  rocky <rb@dustyfeet.com>
 
-	* : commit 6aa1531972de83ecab15b4c96b89c873ea5a7458 Author: rocky
-	<rb@dustyfeet.com> Date:   Wed Nov 23 00:48:38 2016 -0500
+	* : commit 7133540c235e16f02d2db62cb903b70aa311de20 Author: rocky
+	<rb@dustyfeet.com> Date:   Wed Nov 23 08:26:12 2016 -0500
+
+2016-11-23  rocky <rb@dustyfeet.com>
+
+	* : commit a9349b8f3d12b2aa0cd88286617c1af9cccad018 Author: rocky
+	<rb@dustyfeet.com> Date:   Tue Nov 22 17:49:47 2016 -0500
+
+2016-11-23  rocky <rb@dustyfeet.com>
+
+	* circle.yml: Circle CI uses 2.7.10 and 2.7.12 is not available
 
 2016-11-22  rocky <rb@dustyfeet.com>
 

HOW-TO-REPORT-A-BUG.md

@@ -3,8 +3,9 @@
 ## The difficulty of the problem
 
 There is no Python decompiler yet, that I know about that will
-decompyle everything. This one probably does the
-best job of *any* Python decompiler. But it is a constant work in progress: Python keeps changing, and so does its code generation.
+decompyle everything. This one probably does the best job of *any*
+Python decompiler. But it is a constant work in progress: Python keeps
+changing, and so does its code generation.
 
 I have found bugs in *every* Python decompiler I have tried. Even
 those where authors/maintainers claim that they have used it on
@@ -14,6 +15,55 @@ but that the program is *semantically* not equivalent.
 
 So it is likely you'll find a mistranslation in decompiling.
 
+
+## Is it really a bug?
+
+If the code emitted is semantically equivalent, then this isn't a bug.
+
+For example the code might be
+
+```
+if a:
+  if b:
+     x = 1
+```
+
+and we might produce:
+
+```
+if a and b:
+  x = 1
+```
+
+These are equivalent. Sometimes
+
+```
+else:
+   if ...
+
+```
+
+may out as `elif`.
+
+
+As mentioned in the README. It is possible that Python changes what
+you write to be more efficient. For example, for:
+
+
+```
+if True:
+  x = 5
+```
+
+Python will generate code like:
+
+```
+x = 5
+```
+
+So just because the text isn't the same, does not
+necessarily mean there's a bug.
+
 ## What to send (minimum requirements)
 
 The basic requirement is pretty simple:
@@ -21,6 +71,12 @@ The basic requirement is pretty simple:
 * Python bytecode
 * Python source text
 
+Please don't put files on download services that one has to register
+for. If you can't attach it to the issue, or create a github gist,
+then the code you are sending is too large.
+
+Please also try to narrow the bug. See below.
+
 ## What to send (additional helpful information)
 
 Some kind folks also give the invocation they used and the output

Makefile

@@ -11,7 +11,7 @@ RM      ?= rm
 LINT    = flake8
 
 #EXTRA_DIST=ipython/ipy_trepan.py trepan
-PHONY=all check clean pytest check-long dist distclean lint flake8 test rmChangeLog clean_pyc
+PHONY=all check clean distcheck pytest check-long dist distclean lint flake8 test rmChangeLog clean_pyc
 
 TEST_TYPES=check-long check-short check-2.7 check-3.4
 
@@ -39,7 +39,7 @@ check-3.0 check-3.1 check-3.2 check-3.5 check-3.6:
 check-3.7: pytest
 
 #:Tests for Python 2.6 (doesn't have pytest)
-check-2.6:
+check-2.4 check-2.5 check-2.6:
 	$(MAKE) -C test $@
 
 #:PyPy 2.6.1 or PyPy 5.0.1
@@ -60,9 +60,13 @@ clean: clean_pyc
 	(cd test && $(MAKE) clean)
 
 #: Create source (tarball) and wheel distribution
-dist:
+dist: distcheck
 	$(PYTHON) ./setup.py sdist bdist_wheel
 
+# perform some checks on the package via setup.py
+distcheck:
+	$(PYTHON) ./setup.py check
+
 #: Remove .pyc files
 clean_pyc:
 	( cd uncompyle6 && $(RM) -f *.pyc */*.pyc )
@@ -89,7 +93,7 @@ bdist_egg:
 
 
 #: Create binary wheel distribution
-bdist_wheel:
+wheel:
 	$(PYTHON) ./setup.py bdist_wheel
 
 

NEWS

@@ -1,3 +1,30 @@
+uncompyle6 2.13.2 2017-10-12
+
+- Re-release using a more automated approach
+
+uncompyle6 2.13.1 2017-10-11
+
+- Re-release because Python 2.4 source uploaded rather than 2.6-3.6
+
+uncompyle6 2.13.0 2017-10-10
+
+- Fixes in deparsing lambda expressions
+- Improve table-semantics descriptions
+- Document hacky customize arg count better (until we can remove it)
+- Update to use xdis 3.7.0 or greater
+
+uncompyle6 2.12.0 2017-09-26
+
+- Use xdis 3.6.0 or greater now
+- Small semantic table cleanups
+- Python 3.4's terms a little names better
+- Slightly more Python 3.7, but still failing a lot
+- Cross Python 2/3 compatibility with annotation arguments
+
+uncompyle6 2.11.5 2017-08-31
+
+- Skeletal support for Python 3.7
+
 uncompyle6 2.11.4 2017-08-15
 
 * scanner and parser now allow 3-part version string lookups,

README.rst

@@ -4,7 +4,7 @@ uncompyle6
 ==========
 
 A native Python cross-version Decompiler and Fragment Decompiler.
-Follows in the tradition of decompyle, uncompyle, and uncompyle2.
+The successor to decompyle, uncompyle, and uncompyle2.
 
 
 Introduction

__pkginfo__.py

@@ -39,8 +39,8 @@ entry_points       = {
         'pydisassemble=uncompyle6.bin.pydisassemble:main',
     ]}
 ftp_url            = None
-install_requires   = ['spark-parser >= 1.6.1, < 1.7.0',
-                      'xdis >= 3.5.5, < 3.6.0', 'six']
+install_requires   = ['spark-parser >= 1.7.0, < 1.8.0',
+                      'xdis >= 3.6.0, < 3.7.0']
 license            = 'MIT'
 mailing_list       = 'python-debugger@googlegroups.com'
 modname            = 'uncompyle6'

admin-tools/README.md

@@ -0,0 +1,11 @@
+Making a release is a somewhat tedious process so I've automated it a little
+
+
+Here are tools that I, rocky, use to check and build a distribution.
+
+They are customized to my environment:
+- I use pyenv to various Python versions installed
+- I have git repos for xdis, and spark parser at the same level as uncompyle6
+
+There may be other rocky-specific things that need customization.
+how-to-make-a-release.txt has overall how I make a release

admin-tools/check-newer-versions.sh

@@ -0,0 +1,26 @@
+#!/bin/bash
+function finish {
+  cd $owd
+}
+
+# FIXME put some of the below in a common routine
+owd=$(pwd)
+trap finish EXIT
+
+cd $(dirname ${BASH_SOURCE[0]})
+if ! source ./pyenv-newer-versions ; then
+    exit $?
+fi
+if ! source ./setup-master.sh ; then
+    exit $?
+fi
+cd ..
+for version in $PYVERSIONS; do
+    if ! pyenv local $version ; then
+	exit $?
+    fi
+    make clean && pip install -e .
+    if ! make check; then
+	exit $?
+    fi
+done

admin-tools/check-older-versions.sh

@@ -0,0 +1,25 @@
+#!/bin/bash
+function finish {
+  cd $owd
+}
+owd=$(pwd)
+trap finish EXIT
+
+cd $(dirname ${BASH_SOURCE[0]})
+if ! source ./pyenv-older-versions ; then
+    exit $?
+fi
+if ! source ./setup-python-2.4.sh ; then
+    exit $?
+fi
+
+cd ..
+for version in $PYVERSIONS; do
+    if ! pyenv local $version ; then
+	exit $?
+    fi
+    make clean && python setup.py develop
+    if ! make check ; then
+	exit $?
+    fi
+done

admin-tools/how-to-make-a-release.md

@@ -0,0 +1,78 @@
+<!-- markdown-toc start - Don't edit this section. Run M-x markdown-toc-refresh-toc -->
+**Table of Contents**
+
+- [Get latest sources:](#get-latest-sources)
+- [Change version in uncompyle6/version.py. Then:](#change-version-in-uncompyle6versionpy-then)
+- [Update ChangeLog:](#update-changelog)
+- [Update NEWS from ChangeLog. Then:](#update-news-from-changelog-then)
+- [Make sure pyenv is running and check newer versions](#make-sure-pyenv-is-running-and-check-newer-versions)
+- [Switch to python-2.4, sync that up and build that first since it creates a tarball which we don't want.](#switch-to-python-24-sync-that-up-and-build-that-first-since-it-creates-a-tarball-which-we-dont-want)
+- [Update NEWS from master branch](#update-news-from-master-branch)
+- [Check against all versions](#check-against-all-versions)
+- [Make packages and tag](#make-packages-and-tag)
+- [Upload single package and look at Rst Formating](#upload-single-package-and-look-at-rst-formating)
+- [Upload rest of versions](#upload-rest-of-versions)
+- [Push tags:](#push-tags)
+
+<!-- markdown-toc end -->
+# Get latest sources:
+
+    $ . ./admin-tool/update-sources.sh
+
+# Change version in uncompyle6/version.py. Then:
+
+	$ emacs uncompyle6/version.py
+    $ source uncompyle6/version.py
+    $ echo $VERSION
+    $ git commit -m"Get ready for release $VERSION" .
+
+# Update ChangeLog:
+
+    $ make ChangeLog
+
+#  Update NEWS from ChangeLog. Then:
+
+	$ emacs NEWS
+    $ make check
+    $ git commit --amend .
+    $ git push   # get CI testing going early
+
+# Make sure pyenv is running and check newer versions
+
+    $ pyenv local && source admin-tools/check-newer-versions.sh
+
+# Switch to python-2.4, sync that up and build that first since it creates a tarball which we don't want.
+
+    $ source admin-tools/setup-python-2.4.sh
+    $ rm ChangeLog
+
+    # $ git merge master ?
+
+# Update NEWS from master branch
+
+    $ git commit -m"Get ready for release $VERSION" .
+
+# Check against all versions
+
+    $ source admin-tools/check-older-versions.sh
+    $ source admin-tools/check-newer-versions.sh
+
+# Make packages and tag
+
+    $ make-dist-older.sh
+    $ git tag release-python-2.4-$VERSION
+
+    $ make-dist-newer.sh
+    $ git tag release-$VERSION
+
+# Upload single package and look at Rst Formating
+
+    $ twine upload dist/uncompyle6-${VERSION}-py3.3.egg
+
+# Upload rest of versions
+
+    $ twine upload dist/uncompyle6-${VERSION}*
+
+# Push tags:
+
+    $ git push --tags

admin-tools/how-to-make-a-release.txt

@@ -0,0 +1,46 @@
+git pull
+
+Change version in uncompyle6/version.py
+source uncompyle6/version.py
+echo $VERSION
+git commit -m"Get ready for release $VERSION" .
+
+Update ChangeLog:
+  make ChangeLog
+
+Update NEWS from ChangeLog
+make check
+
+git commit --amend .
+
+git push
+
+Make sure pyenv is running
+# Pyenv
+
+source admin-tools/check-newer-versions.sh
+
+
+# Switch to python-2.4 and build that first...
+source admin-tools/setup-python-2.4
+
+rm ChangeLog
+git merge master
+
+Update NEWS from master branch
+
+git commit -m"Get ready for release $VERSION" .
+
+source admin-tools/check-older-versions.sh
+source admin-tools/check-newer-versions.sh
+
+make-dist-older.sh
+
+git tag release-python-2.4-$VERSION
+
+./make-dist-newer.sh
+
+git tag release-$VERSION
+
+
+twine upload dist/uncompyle6-${VERSION}*

admin-tools/make-dist-newer.sh

@@ -0,0 +1,38 @@
+#!/bin/bash
+PACKAGE=uncompyle6
+
+# FIXME put some of the below in a common routine
+function finish {
+  cd $owd
+}
+
+cd $(dirname ${BASH_SOURCE[0]})
+owd=$(pwd)
+trap finish EXIT
+
+if ! source ./pyenv-newer-versions ; then
+    exit $?
+fi
+if ! source ./setup-master.sh ; then
+    exit $?
+fi
+
+cd ..
+source $PACKAGE/version.py
+echo $VERSION
+
+for pyversion in $PYVERSIONS; do
+    if ! pyenv local $pyversion ; then
+	exit $?
+    fi
+    # pip bdist_egg create too-general wheels. So
+    # we narrow that by moving the generated wheel.
+
+    # Pick out first two number of version, e.g. 3.5.1 -> 35
+    first_two=$(echo $pyversion | cut -d'.' -f 1-2 | sed -e 's/\.//')
+    rm -fr build
+    python setup.py bdist_egg bdist_wheel
+    mv -v dist/uncompyle6-$VERSION-{py2.py3,py$first_two}-none-any.whl
+done
+
+python ./setup.py sdist

admin-tools/make-dist-older.sh

@@ -0,0 +1,39 @@
+#!/bin/bash
+PACKAGE=uncompyle6
+
+# FIXME put some of the below in a common routine
+function finish {
+  cd $owd
+}
+owd=$(pwd)
+trap finish EXIT
+
+cd $(dirname ${BASH_SOURCE[0]})
+if ! source ./pyenv-older-versions ; then
+    exit $?
+fi
+if ! source ./setup-python-2.4.sh ; then
+    exit $?
+fi
+
+cd ..
+source $PACKAGE/version.py
+echo $VERSION
+
+for pyversion in $PYVERSIONS; do
+    if ! pyenv local $pyversion ; then
+	exit $?
+    fi
+
+    rm -fr build
+    python setup.py bdist_egg
+done
+
+# Pypi can only have one source tarball.
+# Tarballs can get created from the above setup, so make sure to remove them since we want
+# the tarball from master.
+
+tarball=dist/uncompyle6-$VERSION-tar.gz
+if [[ -f $tarball ]]; then
+    rm -v dist/uncompyle6-$VERSION-tar.gz
+fi

admin-tools/pyenv-newer-versions

@@ -0,0 +1,6 @@
+# -*- shell-script -*-
+if [[ $0 == ${BASH_SOURCE[0]} ]] ; then
+    echo "This script should be *sourced* rather than run directly through bash"
+    exit 1
+fi
+export PYVERSIONS='3.5.2 3.6.2 2.6.9 3.3.6 2.7.13 3.4.2'

admin-tools/pyenv-older-versions

@@ -0,0 +1,6 @@
+# -*- shell-script -*-
+if [[ $0 == ${BASH_SOURCE[0]} ]] ; then
+    echo "This script should be *sourced* rather than run directly through bash"
+    exit 1
+fi
+export PYVERSIONS='2.4.6 2.5.6'

admin-tools/setup-master.sh

@@ -0,0 +1,22 @@
+#!/bin/bash
+PYTHON_VERSION=3.6.3
+
+# FIXME put some of the below in a common routine
+function finish {
+  cd $owd
+}
+
+export PATH=$HOME/.pyenv/bin/pyenv:$PATH
+owd=$(pwd)
+bs=${BASH_SOURCE[0]}
+if [[ $0 == $bs ]] ; then
+    echo "This script should be *sourced* rather than run directly through bash"
+    exit 1
+fi
+mydir=$(dirname $bs)
+fulldir=$(readlink -f $mydir)
+cd $fulldir/..
+(cd ../python-spark && git checkout master && pyenv local $PYTHON_VERSION) && \
+    (cd ../python-xdis && git checkout master && pyenv local $PYTHON_VERSION) && \
+    git checkout master && pyenv local $PYTHON_VERSION
+cd $owd

admin-tools/setup-python-2.4.sh

@@ -0,0 +1,16 @@
+#!/bin/bash
+PYTHON_VERSION=2.4.6
+
+owd=$(pwd)
+bs=${BASH_SOURCE[0]}
+if [[ $0 == $bs ]] ; then
+    echo "This script should be *sourced* rather than run directly through bash"
+    exit 1
+fi
+mydir=$(dirname $bs)
+fulldir=$(readlink -f $mydir)
+cd $fulldir/..
+(cd ../python-spark && git checkout python-2.4 && pyenv local $PYTHON_VERSION) && \
+    (cd ../python-xdis && git checkout python-2.4 && pyenv local $PYTHON_VERSION) && \
+    git checkout python-2.4 &&  pyenv local $PYTHON_VERSION
+cd $owd

admin-tools/update-sources.sh

@@ -0,0 +1,3 @@
+#!/bin/bash
+cd $(dirname ${BASH_SOURCE[0]})/..
+git pull

circle.yml

@@ -10,4 +10,4 @@ dependencies:
         - pip install -r requirements-dev.txt
 test:
     override:
-        - python ./setup.py develop && make check-2.7
+        - python ./setup.py develop && make check-2.6

pytest/test_deparse.py

@@ -29,7 +29,7 @@ def list_comp():
     [y for y in range(3)]
 
 def get_parsed_for_fn(fn):
-    code = fn.__code__ if PYTHON3 else fn.func_code
+    code = fn.func_code
     return deparse(PYTHON_VERSION, code)
 
 def check_expect(expect, parsed):

pytest/test_docstring.py

@@ -10,7 +10,7 @@ else:
     maxint = sys.maxint
 from uncompyle6.semantics.helper import print_docstring
 
-class PrintFake():
+class PrintFake:
     def __init__(self):
         self.pending_newlines = 0
         self.f = StringIO()

pytest/test_fjt.py

@@ -21,9 +21,8 @@ def bug_loop(disassemble, tb=None):
     disassemble(tb)
 
 def test_if_in_for():
-    code = bug.__code__
+    code = bug.func_code
     scan = get_scanner(PYTHON_VERSION)
-    print(PYTHON_VERSION)
     if 2.7 <= PYTHON_VERSION <= 3.0 and not IS_PYPY:
         n = scan.setup_code(code)
         scan.build_lines_data(code, n)

pytest/test_fstring.py

@@ -1,150 +0,0 @@
-# std
-import os
-# test
-import pytest
-import hypothesis
-from hypothesis import strategies as st
-# uncompyle6
-from uncompyle6 import PYTHON_VERSION, deparse_code
-
-
-@st.composite
-def expressions(draw):
-    # todo : would be nice to generate expressions using hypothesis however
-    # this is pretty involved so for now just use a corpus of expressions
-    # from which to select.
-    return draw(st.sampled_from((
-        'abc',
-        'len(items)',
-        'x + 1',
-        'lineno',
-        'container',
-        'self.attribute',
-        'self.method()',
-        # These expressions are failing, I think these are control
-        # flow problems rather than problems with FORMAT_VALUE,
-        # however I need to confirm this...
-        #'sorted(items, key=lambda x: x.name)',
-        #'func(*args, **kwargs)',
-        #'text or default',
-        #'43 if life_the_universe and everything else None'
-    )))
-
-
-@st.composite
-def format_specifiers(draw):
-    """
-    Generate a valid format specifier using the rules:
-
-    format_spec ::=  [[fill]align][sign][#][0][width][,][.precision][type]
-    fill        ::=  <any character>
-    align       ::=  "<" | ">" | "=" | "^"
-    sign        ::=  "+" | "-" | " "
-    width       ::=  integer
-    precision   ::=  integer
-    type        ::=  "b" | "c" | "d" | "e" | "E" | "f" | "F" | "g" | "G" | "n" | "o" | "s" | "x" | "X" | "%"
-
-    See https://docs.python.org/2/library/string.html
-
-    :param draw: Let hypothesis draw from other strategies.
-
-    :return: An example format_specifier.
-    """
-    alphabet_strategy = st.characters(min_codepoint=ord('a'), max_codepoint=ord('z'))
-    fill = draw(st.one_of(alphabet_strategy, st.none()))
-    align = draw(st.sampled_from(list('<>=^')))
-    fill_align = (fill + align or '') if fill else ''
-
-    type_ = draw(st.sampled_from('bcdeEfFgGnosxX%'))
-    can_have_sign = type_ in 'deEfFgGnoxX%'
-    can_have_comma = type_ in 'deEfFgG%'
-    can_have_precision = type_ in 'fFgG'
-    can_have_pound = type_ in 'boxX%'
-    can_have_zero = type_ in 'oxX'
-
-    sign = draw(st.sampled_from(list('+- ') + [''])) if can_have_sign else ''
-    pound = draw(st.sampled_from(('#', '',))) if can_have_pound else ''
-    zero = draw(st.sampled_from(('0', '',))) if can_have_zero else ''
-
-    int_strategy = st.integers(min_value=1, max_value=1000)
-
-    width = draw(st.one_of(int_strategy, st.none()))
-    width = str(width) if width is not None else ''
-
-    comma = draw(st.sampled_from((',', '',))) if can_have_comma else ''
-    if can_have_precision:
-        precision = draw(st.one_of(int_strategy, st.none()))
-        precision = '.' + str(precision) if precision else ''
-    else:
-        precision = ''
-
-    return ''.join((fill_align, sign, pound, zero, width, comma, precision, type_,))
-
-
-@st.composite
-def fstrings(draw):
-    """
-    Generate a valid f-string.
-    See https://www.python.org/dev/peps/pep-0498/#specification
-
-    :param draw: Let hypothsis draw from other strategies.
-
-    :return: A valid f-string.
-    """
-    character_strategy = st.characters(
-        blacklist_characters='\r\n\'\\s{}',
-        min_codepoint=1,
-        max_codepoint=1000,
-    )
-    is_raw = draw(st.booleans())
-    integer_strategy = st.integers(min_value=0, max_value=3)
-    expression_count = draw(integer_strategy)
-    content = []
-    for _ in range(expression_count):
-        expression = draw(expressions())
-        conversion = draw(st.sampled_from(('', '!s', '!r', '!a',)))
-        has_specifier = draw(st.booleans())
-        specifier = ':' + draw(format_specifiers()) if has_specifier else ''
-        content.append('{{{}{}}}'.format(expression, conversion, specifier))
-        content.append(draw(st.text(character_strategy)))
-    content = ''.join(content)
-    return "f{}'{}'".format('r' if is_raw else '', content)
-
-
-@pytest.mark.skipif(PYTHON_VERSION < 3.6, reason='need at least python 3.6')
-@hypothesis.given(format_specifiers())
-def test_format_specifiers(format_specifier):
-    """Verify that format_specifiers generates valid specifiers"""
-    try:
-        exec('"{:' + format_specifier + '}".format(0)')
-    except ValueError as e:
-        if 'Unknown format code' not in str(e):
-            raise
-
-
-def run_test(text):
-    hypothesis.assume(len(text))
-    hypothesis.assume("f'{" in text)
-    expr = text + '\n'
-    code = compile(expr, '<string>', 'single')
-    deparsed = deparse_code(PYTHON_VERSION, code, compile_mode='single')
-    recompiled = compile(deparsed.text, '<string>', 'single')
-    if recompiled != code:
-        assert 'dis(' + deparsed.text.strip('\n') + ')' == 'dis(' + expr.strip('\n') + ')'
-
-
-@pytest.mark.skipif(PYTHON_VERSION < 3.6, reason='need at least python 3.6')
-@hypothesis.given(fstrings())
-def test_uncompyle_fstring(fstring):
-    """Verify uncompyling fstring bytecode"""
-    run_test(fstring)
-
-
-@pytest.mark.skipif(PYTHON_VERSION < 3.6, reason='need at least python 3.6')
-@pytest.mark.parametrize('fstring', [
-    "f'{abc}{abc!s}'",
-    "f'{abc}0'",
-])
-def test_uncompyle_direct(fstring):
-    """useful for debugging"""
-    run_test(fstring)

pytest/test_function_call.py

@@ -1,175 +0,0 @@
-# std
-import string
-# 3rd party
-from hypothesis import given, assume, example, settings, strategies as st
-import pytest
-# uncompyle
-from validate import validate_uncompyle
-from test_fstring import expressions
-
-
-alpha = st.sampled_from(string.ascii_lowercase)
-numbers = st.sampled_from(string.digits)
-alphanum = st.sampled_from(string.ascii_lowercase + string.digits)
-
-
-@st.composite
-def function_calls(draw,
-                   min_keyword_args=0, max_keyword_args=5,
-                   min_positional_args=0, max_positional_args=5,
-                   min_star_args=0, max_star_args=1,
-                   min_double_star_args=0, max_double_star_args=1):
-    """
-    Strategy factory for generating function calls.
-
-    :param draw: Callable which draws examples from other strategies.
-
-    :return: The function call text.
-    """
-    st_positional_args = st.lists(
-        alpha,
-        min_size=min_positional_args,
-        max_size=max_positional_args
-    )
-    st_keyword_args = st.lists(
-        alpha,
-        min_size=min_keyword_args,
-        max_size=max_keyword_args
-    )
-    st_star_args = st.lists(
-        alpha,
-        min_size=min_star_args,
-        max_size=max_star_args
-    )
-    st_double_star_args = st.lists(
-        alpha,
-        min_size=min_double_star_args,
-        max_size=max_double_star_args
-    )
-
-    positional_args = draw(st_positional_args)
-    keyword_args = draw(st_keyword_args)
-    st_values = st.lists(
-        expressions(),
-        min_size=len(keyword_args),
-        max_size=len(keyword_args)
-    )
-    keyword_args = [
-        x + '=' + e
-        for x, e in
-        zip(keyword_args, draw(st_values))
-    ]
-    star_args = ['*' + x for x in draw(st_star_args)]
-    double_star_args = ['**' + x for x in draw(st_double_star_args)]
-
-    arguments = positional_args + keyword_args + star_args + double_star_args
-    draw(st.randoms()).shuffle(arguments)
-    arguments = ','.join(arguments)
-
-    function_call = 'fn({arguments})'.format(arguments=arguments)
-    try:
-        # TODO: Figure out the exact rules for ordering of positional, keyword,
-        # star args, double star args and in which versions the various
-        # types of arguments are supported so we don't need to check that the
-        # expression compiles like this.
-        compile(function_call, '<string>', 'single')
-    except:
-        assume(False)
-    return function_call
-
-
-def test_function_no_args():
-    validate_uncompyle("fn()")
-
-
-def isolated_function_calls(which):
-    """
-    Returns a strategy for generating function calls, but isolated to 
-    particular types of arguments, for example only positional arguments.
-
-    This can help reason about debugging errors in specific types of function
-    calls.
-
-    :param which: One of 'keyword', 'positional', 'star', 'double_star'
-
-    :return: Strategy for generating an function call isolated to specific
-             argument types.
-    """
-    kwargs = dict(
-        max_keyword_args=0,
-        max_positional_args=0,
-        max_star_args=0,
-        max_double_star_args=0,
-    )
-    kwargs['_'.join(('min', which, 'args'))] = 1
-    kwargs['_'.join(('max', which, 'args'))] = 5 if 'star' not in which else 1
-    return function_calls(**kwargs)
-
-
-with settings(max_examples=25):
-
-    @given(isolated_function_calls('positional'))
-    @example("fn(0)")
-    def test_function_positional_only(expr):
-        validate_uncompyle(expr)
-
-    @given(isolated_function_calls('keyword'))
-    @example("fn(a=0)")
-    def test_function_call_keyword_only(expr):
-        validate_uncompyle(expr)
-
-    @given(isolated_function_calls('star'))
-    @example("fn(*items)")
-    def test_function_call_star_only(expr):
-        validate_uncompyle(expr)
-
-    @given(isolated_function_calls('double_star'))
-    @example("fn(**{})")
-    def test_function_call_double_star_only(expr):
-        validate_uncompyle(expr)
-
-
-@pytest.mark.xfail()
-def test_BUILD_CONST_KEY_MAP_BUILD_MAP_UNPACK_WITH_CALL_BUILD_TUPLE_CALL_FUNCTION_EX():
-    validate_uncompyle("fn(w=0,m=0,**v)")
-
-
-@pytest.mark.xfail()
-def test_BUILD_MAP_BUILD_MAP_UNPACK_WITH_CALL_BUILD_TUPLE_CALL_FUNCTION_EX():
-    validate_uncompyle("fn(a=0,**g)")
-
-
-@pytest.mark.xfail()
-def test_CALL_FUNCTION_EX():
-    validate_uncompyle("fn(*g,**j)")
-
-
-@pytest.mark.xfail()
-def test_BUILD_MAP_CALL_FUNCTION_EX():
-    validate_uncompyle("fn(*z,u=0)")
-
-
-@pytest.mark.xfail()
-def test_BUILD_TUPLE_CALL_FUNCTION_EX():
-    validate_uncompyle("fn(**a)")
-
-
-@pytest.mark.xfail()
-def test_BUILD_MAP_BUILD_TUPLE_BUILD_TUPLE_UNPACK_WITH_CALL_CALL_FUNCTION_EX():
-    validate_uncompyle("fn(b,b,b=0,*a)")
-
-
-@pytest.mark.xfail()
-def test_BUILD_TUPLE_BUILD_TUPLE_UNPACK_WITH_CALL_CALL_FUNCTION_EX():
-    validate_uncompyle("fn(*c,v)")
-
-
-@pytest.mark.xfail()
-def test_BUILD_CONST_KEY_MAP_CALL_FUNCTION_EX():
-    validate_uncompyle("fn(i=0,y=0,*p)")
-
-
-@pytest.mark.skip(reason='skipping property based test until all individual tests are passing')
-@given(function_calls())
-def test_function_call(function_call):
-    validate_uncompyle(function_call)

pytest/test_grammar.py

@@ -11,15 +11,16 @@ def test_grammar():
         remain_tokens = set([re.sub('_CONT$','', t) for t in remain_tokens])
         remain_tokens = set(remain_tokens) - opcode_set
         assert remain_tokens == set([]), \
-            "Remaining tokens %s\n====\n%s" % (remain_tokens, p.dumpGrammar())
+            "Remaining tokens %s\n====\n%s" % (remain_tokens, p.dump_grammar())
 
     p = get_python_parser(PYTHON_VERSION, is_pypy=IS_PYPY)
-    lhs, rhs, tokens, right_recursive = p.checkSets()
+    lhs, rhs, tokens, right_recursive = p.check_sets()
     expect_lhs = set(['expr1024', 'pos_arg'])
     unused_rhs = set(['build_list', 'call_function', 'mkfunc',
                       'mklambda',
                       'unpack', 'unpack_list'])
-    expect_right_recursive = [['designList', ('designator', 'DUP_TOP', 'designList')]]
+    expect_right_recursive = frozenset([('designList',
+                                         ('designator', 'DUP_TOP', 'designList'))])
     if PYTHON3:
         expect_lhs.add('load_genexpr')
 
@@ -39,13 +40,14 @@ def test_grammar():
     s = get_scanner(PYTHON_VERSION, IS_PYPY)
     ignore_set = set(
             """
-            JUMP_BACK CONTINUE RETURN_END_IF
+            JUMP_BACK CONTINUE
             COME_FROM COME_FROM_EXCEPT
             COME_FROM_EXCEPT_CLAUSE
             COME_FROM_LOOP COME_FROM_WITH
             COME_FROM_FINALLY ELSE
             LOAD_GENEXPR LOAD_ASSERT LOAD_SETCOMP LOAD_DICTCOMP
-            LAMBDA_MARKER RETURN_LAST
+            LAMBDA_MARKER
+            RETURN_END_IF RETURN_END_IF_LAMBDA RETURN_VALUE_LAMBDA RETURN_LAST
             """.split())
     if 2.6 <= PYTHON_VERSION <= 2.7:
         opcode_set = set(s.opc.opname).union(ignore_set)

pytest/test_pysource.py

@@ -0,0 +1,164 @@
+from uncompyle6 import PYTHON3
+from uncompyle6.semantics.consts import (
+    escape, NONE,
+    # RETURN_NONE, PASS, RETURN_LOCALS
+)
+
+if PYTHON3:
+    from io import StringIO
+else:
+    from StringIO import StringIO
+
+from uncompyle6.semantics.pysource import SourceWalker as SourceWalker
+
+def test_template_engine():
+    s = StringIO()
+    sw = SourceWalker(2.7, s, None)
+    sw.ast = NONE
+    sw.template_engine(('--%c--', 0), NONE)
+    print(sw.f.getvalue())
+    assert sw.f.getvalue() == '--None--'
+    # FIXME: and so on...
+
+from uncompyle6.semantics.consts import (
+    TABLE_R, TABLE_DIRECT,
+    )
+
+from uncompyle6.semantics.fragments import (
+    TABLE_DIRECT_FRAGMENT,
+    )
+
+skip_for_now = "DELETE_DEREF".split()
+
+def test_tables():
+    for t, name, fragment in (
+            (TABLE_DIRECT, 'TABLE_DIRECT', False),
+            (TABLE_R, 'TABLE_R', False),
+            (TABLE_DIRECT_FRAGMENT, 'TABLE_DIRECT_FRAGMENT', True)):
+        for k, entry in t.iteritems():
+            if k in skip_for_now:
+                continue
+            fmt = entry[0]
+            arg = 1
+            i = 0
+            m = escape.search(fmt)
+            print("%s[%s]" % (name, k))
+            while m:
+                i = m.end()
+                typ = m.group('type') or '{'
+                if typ in frozenset(['%', '+', '-', '|', ',', '{']):
+                    # No args
+                    pass
+                elif typ in frozenset(['c', 'p', 'P', 'C', 'D']):
+                    # One arg - should be int or tuple of int
+                    if typ == 'c':
+                        assert isinstance(entry[arg], int), (
+                            "%s[%s][%d] kind %s is '%s' should be an int but is %s. "
+                            "Full entry: %s" %
+                            (name, k, arg, typ, entry[arg], type(entry[arg]), entry)
+                            )
+                    elif typ in frozenset(['C', 'D']):
+                        tup = entry[arg]
+                        assert isinstance(tup, tuple), (
+                            "%s[%s][%d] type %s is %s should be an tuple but is %s. "
+                            "Full entry: %s" %
+                            (name, k, arg, typ, entry[arg], type(entry[arg]), entry)
+                            )
+                        assert len(tup) == 3
+                        for j, x in enumerate(tup[:-1]):
+                            assert isinstance(x, int), (
+                                "%s[%s][%d][%d] type %s is %s should be an tuple but is %s. "
+                                "Full entry: %s" %
+                                (name, k, arg, j, typ, x, type(x), entry)
+                                )
+                        assert isinstance(tup[-1], str) or tup[-1] is None, (
+                                "%s[%s][%d][%d] sep type %s is %s should be an string but is %s. "
+                                "Full entry: %s" %
+                                (name, k, arg, j, typ, tup[-1], type(x), entry)
+                                )
+
+                    elif typ == 'P':
+                        tup = entry[arg]
+                        assert isinstance(tup, tuple), (
+                            "%s[%s][%d] type %s is %s should be an tuple but is %s. "
+                            "Full entry: %s" %
+                            (name, k, arg, typ, entry[arg], type(entry[arg]), entry)
+                            )
+                        assert len(tup) == 4
+                        for j, x in enumerate(tup[:-2]):
+                            assert isinstance(x, int), (
+                                "%s[%s][%d][%d] type %s is '%s' should be an tuple but is %s. "
+                                "Full entry: %s" %
+                                (name, k, arg, j, typ, x, type(x), entry)
+                                )
+                        assert isinstance(tup[-2], str), (
+                                "%s[%s][%d][%d] sep type %s is '%s' should be an string but is %s. "
+                                "Full entry: %s" %
+                                (name, k, arg, j, typ, x, type(x), entry)
+                                )
+                        assert isinstance(tup[1], int), (
+                                "%s[%s][%d][%d] prec type %s is '%s' should be an int but is %s. "
+                                "Full entry: %s" %
+                                (name, k, arg, j, typ, x, type(x), entry)
+                                )
+
+                    else:
+                        # Should be a tuple which contains only ints
+                        tup = entry[arg]
+                        assert isinstance(tup, tuple), (
+                            "%s[%s][%d] type %s is '%s' should be an tuple but is %s. "
+                            "Full entry: %s" %
+                            (name, k, arg, typ, entry[arg], type(entry[arg]), entry)
+                            )
+                        assert len(tup) == 2
+                        for j, x in enumerate(tup):
+                            assert isinstance(x, int), (
+                                "%s[%s][%d][%d] type '%s' is '%s should be an int but is %s. Full entry: %s" %
+                                (name, k, arg, j, typ, x, type(x), entry)
+                                )
+                        pass
+                    arg += 1
+                elif typ in frozenset(['r']) and fragment:
+                    pass
+                elif typ == 'b' and fragment:
+                    assert isinstance(entry[arg], int), (
+                        "%s[%s][%d] type %s is '%s' should be an int but is %s. "
+                        "Full entry: %s" %
+                        (name, k, arg, typ, entry[arg], type(entry[arg]), entry)
+                        )
+                    arg += 1
+                elif typ == 'x' and fragment:
+                    tup = entry[arg]
+                    assert isinstance(tup, tuple), (
+                        "%s[%s][%d] type %s is '%s' should be an tuple but is %s. "
+                        "Full entry: %s" %
+                        (name, k, arg, typ, entry[arg], type(entry[arg]), entry)
+                            )
+                    assert len(tup) == 2
+                    assert isinstance(tup[0], int), (
+                        "%s[%s][%d] source type %s is '%s' should be an int but is %s. "
+                        "Full entry: %s" %
+                        (name, k, arg, typ, entry[arg], type(entry[arg]), entry)
+                            )
+                    assert isinstance(tup[1], tuple), (
+                        "%s[%s][%d] dest type %s is '%s' should be an tuple but is %s. "
+                        "Full entry: %s" %
+                        (name, k, arg, typ, entry[arg], type(entry[arg]), entry)
+                            )
+                    for j, x in enumerate(tup[1]):
+                        assert isinstance(x, int), (
+                            "%s[%s][%d][%d] type %s is %s should be an int but is %s. Full entry: %s" %
+                            (name, k, arg, j, typ, x, type(x), entry)
+                            )
+                    arg += 1
+                    pass
+                else:
+                    assert False, (
+                        "%s[%s][%d] type %s is not known. Full entry: %s" %
+                            (name, k, arg, typ, entry)
+                        )
+                m = escape.search(fmt, i)
+                pass
+            assert arg == len(entry), (
+                "%s[%s] arg %d should be length of entry %d. Full entry: %s" %
+                            (name, k, arg, len(entry), entry))

pytest/test_single_compile.py

@@ -1,19 +1,19 @@
-import pytest
-from uncompyle6 import PYTHON_VERSION, PYTHON3, deparse_code
+from uncompyle6 import PYTHON_VERSION, deparse_code
 
-def test_single_mode():
-    single_expressions = (
-        'i = 1',
-        'i and (j or k)',
-        'i += 1',
-        'i = j % 4',
-        'i = {}',
-        'i = []',
-        'for i in range(10):\n    i\n',
-        'for i in range(10):\n    for j in range(10):\n        i + j\n',
-        'try:\n    i\nexcept Exception:\n    j\nelse:\n    k\n'
-    )
+if PYTHON_VERSION >= 2.5:
+    def test_single_mode():
+        single_expressions = (
+            'i = 1',
+            'i and (j or k)',
+            'i += 1',
+            'i = j % 4',
+            'i = {}',
+            'i = []',
+            'for i in range(10):\n    i\n',
+            'for i in range(10):\n    for j in range(10):\n        i + j\n',
+            'try:\n    i\nexcept Exception:\n    j\nelse:\n    k\n'
+        )
 
-    for expr in single_expressions:
-        code = compile(expr + '\n', '<string>', 'single')
-        assert deparse_code(PYTHON_VERSION, code, compile_mode='single').text == expr + '\n'
+        for expr in single_expressions:
+            code = compile(expr + '\n', '<string>', 'single')
+            assert deparse_code(PYTHON_VERSION, code, compile_mode='single').text == expr + '\n'

pytest/validate.py

@@ -1,24 +1,25 @@
-# future
-from __future__ import print_function
 # std
 import os
 import difflib
 import subprocess
 import tempfile
-import functools
-# compatability
-import six
+
+from StringIO import StringIO
 # uncompyle6 / xdis
 from uncompyle6 import PYTHON_VERSION, IS_PYPY, deparse_code
 # TODO : I think we can get xdis to support the dis api (python 3 version) by doing something like this there
 from xdis.bytecode import Bytecode
 from xdis.main import get_opcode
 opc = get_opcode(PYTHON_VERSION, IS_PYPY)
-Bytecode = functools.partial(Bytecode, opc=opc)
 
+try:
+    import functools
+    Bytecode = functools.partial(Bytecode, opc=opc)
+    def _dis_to_text(co):
+        return Bytecode(co).dis()
+except:
+    pass
 
-def _dis_to_text(co):
-    return Bytecode(co).dis()
 
 
 def print_diff(original, uncompyled):
@@ -42,8 +43,11 @@ def print_diff(original, uncompyled):
         print('\nTo display diff highlighting run:\n    pip install BeautifulSoup4')
         diff = difflib.HtmlDiff().make_table(*args)
 
-    with tempfile.NamedTemporaryFile(delete=False) as f:
+    f = tempfile.NamedTemporaryFile(delete=False)
+    try:
         f.write(str(diff).encode('utf-8'))
+    finally:
+        f.close()
 
     try:
         print()
@@ -60,8 +64,7 @@ def print_diff(original, uncompyled):
         print('\nFor side by side diff install elinks')
         diff = difflib.Differ().compare(original_lines, uncompyled_lines)
         print('\n'.join(diff))
-    finally:
-        os.unlink(f.name)
+    os.unlink(f.name)
 
 
 def are_instructions_equal(i1, i2):
@@ -123,8 +126,9 @@ def validate_uncompyle(text, mode='exec'):
     original_text = text
 
     deparsed = deparse_code(PYTHON_VERSION, original_code,
+
                             compile_mode=mode,
-                            out=six.StringIO(),
+                            out=StringIO(),
                             is_pypy=IS_PYPY)
     uncompyled_text = deparsed.text
     uncompyled_code = compile(uncompyled_text, '<string>', 'exec')

setup.py

@@ -24,6 +24,6 @@ setup(
        py_modules         = py_modules,
        test_suite         = 'nose.collector',
        url                = web,
-       tests_require     = ['nose>=1.0'],
+       tests_require      = ['nose>=1.0'],
        version            = VERSION,
        zip_safe           = zip_safe)

test/Makefile

@@ -19,7 +19,7 @@ check:
 	$(MAKE) check-$(PYTHON_VERSION)
 
 #: Run working tests from Python 2.6 or 2.7
-check-2.6 check-2.7: check-bytecode-2 check-bytecode-3 check-bytecode-1 check-native-short
+check-2.4 check-2.5 check-2.6 check-2.7: check-bytecode-2 check-bytecode-3 check-bytecode-1  check-native-short
 
 #: Run working tests from Python 3.0
 check-3.0: check-bytecode
@@ -39,7 +39,7 @@ check-3.3: check-bytecode
 
 #: Run working tests from Python 3.4
 check-3.4: check-bytecode check-3.4-ok check-2.7-ok
-	$(PYTHON) test_pythonlib.py --bytecode-3.4 --verify $(COMPILE)
+	$(PYTHON) test_pythonlib.py --bytecode-3.4 --weak-verify $(COMPILE)
 
 #: Run working tests from Python 3.5
 check-3.5: check-bytecode
@@ -47,7 +47,7 @@ check-3.5: check-bytecode
 
 #: Run working tests from Python 3.6
 check-3.6: check-bytecode
-	$(PYTHON) test_pythonlib.py --bytecode-3.6 --verify $(COMPILE)
+	$(PYTHON) test_pythonlib.py --bytecode-3.6 --weak-verify $(COMPILE)
 
 #: Check deparsing only, but from a different Python version
 check-disasm:
@@ -67,7 +67,7 @@ check-bytecode-2:
 check-bytecode-3:
 	$(PYTHON) test_pythonlib.py --bytecode-3.0 \
        --bytecode-3.1 --bytecode-3.2 --bytecode-3.3 \
-       --bytecode-3.4 --bytecode-3.5 --bytecode-pypy3.2
+       --bytecode-3.4 --bytecode-3.5 --bytecode-3.6 --bytecode-pypy3.2
 
 #: Check deparsing bytecode that works running Python 2 and Python 3
 check-bytecode: check-bytecode-3
@@ -97,29 +97,6 @@ check-bytecode-2.4:
 check-bytecode-2.5:
 	$(PYTHON) test_pythonlib.py --bytecode-2.5
 
-#: Get grammar coverage for Python 2.5
-grammar-coverage-2.5:
-	SPARK_PARSER_COVERAGE=/tmp/spark-grammar-25.cover $(PYTHON) test_pythonlib.py --bytecode-2.5
-	SPARK_PARSER_COVERAGE=/tmp/spark-grammar-25.cover $(PYTHON) test_pyenvlib.py --2.5.6
-
-#: Get grammar coverage for Python 2.6
-grammar-coverage-2.6:
-	SPARK_PARSER_COVERAGE=/tmp/spark-grammar-26.cover $(PYTHON) test_pythonlib.py --bytecode-2.6
-	SPARK_PARSER_COVERAGE=/tmp/spark-grammar-26.cover $(PYTHON) test_pyenvlib.py --2.6.9
-
-#: Get grammar coverage for Python 2.7
-grammar-coverage-2.7:
-	SPARK_PARSER_COVERAGE=/tmp/spark-grammar-27.cover $(PYTHON) test_pythonlib.py --bytecode-2.7
-	SPARK_PARSER_COVERAGE=/tmp/spark-grammar-27.cover $(PYTHON) test_pyenvlib.py --2.7.13
-
-#: Check deparsing Python 2.6
-check-bytecode-2.6:
-	$(PYTHON) test_pythonlib.py --bytecode-2.6 --weak-verify
-
-#: Check deparsing Python 2.7
-check-bytecode-2.7:
-	$(PYTHON) test_pythonlib.py --bytecode-2.7 --verify
-
 #: Check deparsing Python 3.0
 check-bytecode-3.0:
 	$(PYTHON) test_pythonlib.py --bytecode-3.0
@@ -148,9 +125,33 @@ check-bytecode-3.5:
 check-bytecode-3.6:
 	$(PYTHON) test_pythonlib.py --bytecode-3.6
 
+#: Get grammar coverage for Python 2.4
+grammar-coverage-2.4:
+	SPARK_PARSER_COVERAGE=/tmp/spark-grammar-24.cover $(PYTHON) test_pythonlib.py --bytecode-2.4
+	SPARK_PARSER_COVERAGE=/tmp/spark-grammar-24.cover $(PYTHON) test_pyenvlib.py --2.4.6
+
+#: Get grammar coverage for Python 2.5
+grammar-coverage-2.5:
+	SPARK_PARSER_COVERAGE=/tmp/spark-grammar-25.cover $(PYTHON) test_pythonlib.py --bytecode-2.5
+	SPARK_PARSER_COVERAGE=/tmp/spark-grammar-25.cover $(PYTHON) test_pyenvlib.py --2.5.6
+
+#: Get grammar coverage for Python 2.6
+grammar-coverage-2.6:
+	SPARK_PARSER_COVERAGE=/tmp/spark-grammar-26.cover $(PYTHON) test_pythonlib.py --bytecode-2.6
+	SPARK_PARSER_COVERAGE=/tmp/spark-grammar-26.cover $(PYTHON) test_pyenvlib.py --2.6.9
+
+#: Get grammar coverage for Python 2.7
+grammar-coverage-2.7:
+	SPARK_PARSER_COVERAGE=/tmp/spark-grammar-27.cover $(PYTHON) test_pythonlib.py --bytecode-2.7
+	SPARK_PARSER_COVERAGE=/tmp/spark-grammar-27.cover $(PYTHON) test_pyenvlib.py --2.7.13
+
 #: short tests for bytecodes only for this version of Python
 check-native-short:
-	$(PYTHON) test_pythonlib.py --bytecode-$(PYTHON_VERSION) --verify $(COMPILE)
+	$(PYTHON) test_pythonlib.py --bytecode-$(PYTHON_VERSION) --weak-verify $(COMPILE)
+
+#: Run longer Python 2.6's lib files known to be okay
+check-2.4-ok:
+	$(PYTHON) test_pythonlib.py --ok-2.4 --verify $(COMPILE)
 
 #: Run longer Python 2.6's lib files known to be okay
 check-2.6-ok:

test/dis-compare.py

@@ -1,9 +1,8 @@
 #!/usr/bin/env python
 # Mode: -*- python -*-
 #
-# Copyright (c) 2015 by Rocky Bernstein <rb@dustyfeet.com>
+# Copyright (c) 2015, 2017 by Rocky Bernstein <rb@dustyfeet.com>
 #
-from __future__ import print_function
 
 
 import dis, os.path

test/simple-uncompyle-code-test.py

@@ -1,7 +1,5 @@
 #!/usr/bin/env python
 
-from __future__ import print_function
-
 from uncompyle6 import uncompyle
 import sys, inspect
 

test/simple_source/branching/02_ifelse_lambda.py

@@ -0,0 +1,19 @@
+# We have to do contortions here because
+# lambda's have to be more or less on a line
+
+f = lambda x: 1 if x<2 else 3
+f(5)
+<<<<<<< HEAD
+=======
+
+# If that wasn't enough ...
+# Python will create dead code
+# in the below. So we must make sure
+# not to include the else expression
+
+g = lambda: 1 if True else 3
+g()
+
+h = lambda: 1 if False else 3
+h()
+>>>>>>> master

test/test_pyenvlib.py

@@ -19,8 +19,6 @@ Step 2: Run the test:
 	  test_pyenvlib --mylib --verify # decompile verify 'mylib'
 """
 
-from __future__ import print_function
-
 from uncompyle6 import main, PYTHON3
 import os, time, shutil, sys
 from fnmatch import fnmatch

test/test_pythonlib.py

@@ -27,8 +27,6 @@ Step 2: Run the test:
   test_pythonlib.py --mylib --verify # decompile verify 'mylib'
 """
 
-from __future__ import print_function
-
 import getopt, os, py_compile, sys, shutil, tempfile, time
 
 from uncompyle6 import PYTHON_VERSION
@@ -127,8 +125,10 @@ def do_tests(src_dir, obj_patterns, target_dir, opts):
     if opts['do_compile']:
         compiled_version = opts['compiled_version']
         if compiled_version and PYTHON_VERSION != compiled_version:
-            print("Not compiling: desired Python version is %s but we are running %s" %
-                  (compiled_version, PYTHON_VERSION), file=sys.stderr)
+            sys.stderr.write("Not compiling: "
+                             "desired Python version is %s "
+                             "but we are running %s" %
+                             (compiled_version, PYTHON_VERSION))
         else:
             for root, dirs, basenames in os.walk(src_dir):
                 file_matches(files, root, basenames, PY)
@@ -146,8 +146,8 @@ def do_tests(src_dir, obj_patterns, target_dir, opts):
         file_matches(files, dirname, basenames, obj_patterns)
 
     if not files:
-        print("Didn't come up with any files to test! Try with --compile?",
-              file=sys.stderr)
+        sys.stderr.write("Didn't come up with any files to test! "
+                         "Try with --compile?")
         exit(1)
 
     os.chdir(cwd)
@@ -161,21 +161,21 @@ def do_tests(src_dir, obj_patterns, target_dir, opts):
         except ValueError:
             pass
 
-    print(time.ctime())
-    print('Source directory: ', src_dir)
-    print('Output directory: ', target_dir)
+    print time.ctime()
+    print 'Source directory: ', src_dir
+    print 'Output directory: ', target_dir
     try:
         _, _, failed_files, failed_verify = \
           main(src_dir, target_dir, files, [],
                do_verify=opts['do_verify'])
         if failed_files != 0:
-            exit(2)
+            sys.exit(2)
         elif failed_verify != 0:
-            exit(3)
+            sys.exit(3)
 
     except (KeyboardInterrupt, OSError):
         print()
-        exit(1)
+        sys.exit(1)
     if test_opts['rmtree']:
         parent_dir = os.path.dirname(target_dir)
         print("Everything good, removing %s" % parent_dir)
@@ -236,14 +236,13 @@ if __name__ == '__main__':
         if os.path.isdir(src_dir):
             checked_dirs.append([src_dir, pattern, target_dir])
         else:
-            print("Can't find directory %s. Skipping" % src_dir,
-                  file=sys.stderr)
+            sys.stderr.write("Can't find directory %s. Skipping" % src_dir)
             continue
         last_compile_version = compiled_version
         pass
 
     if not checked_dirs:
-        print("No directories found to check", file=sys.stderr)
+        sys.stderr.write("No directories found to check\n")
         sys.exit(1)
 
     test_opts['compiled_version'] = last_compile_version

uncompyle6/bin/pydisassemble.py

@@ -3,7 +3,6 @@
 #
 # Copyright (c) 2015-2016 by Rocky Bernstein <rb@dustyfeet.com>
 #
-from __future__ import print_function
 import sys, os, getopt
 
 from uncompyle6.disas import disassemble_file
@@ -26,7 +25,7 @@ Options:
   -V | --version     show version and stop
   -h | --help        show this message
 
-""".format(program)
+""" % (program, program)
 
 PATTERNS = ('*.pyc', '*.pyo')
 
@@ -37,15 +36,15 @@ Type -h for for full help.""" % program
     native = True
 
     if len(sys.argv) == 1:
-        print("No file(s) given", file=sys.stderr)
-        print(Usage_short, file=sys.stderr)
+        sys.stderr.write("No file(s) given\n")
+        sys.stderr.write(Usage_short)
         sys.exit(1)
 
     try:
         opts, files = getopt.getopt(sys.argv[1:], 'hVU',
                                     ['help', 'version', 'uncompyle6'])
-    except getopt.GetoptError as e:
-        print('%s: %s' % (os.path.basename(sys.argv[0]), e),  file=sys.stderr)
+    except getopt.GetoptError(e):
+        sys.stderr.write('%s: %s' % (os.path.basename(sys.argv[0]), e))
         sys.exit(-1)
 
     for opt, val in opts:
@@ -59,15 +58,14 @@ Type -h for for full help.""" % program
             native = False
         else:
             print(opt)
-            print(Usage_short, file=sys.stderr)
+            sys.stderr.write(Usage_short)
             sys.exit(1)
 
     for file in files:
         if os.path.exists(files[0]):
             disassemble_file(file, sys.stdout, native)
         else:
-            print("Can't read %s - skipping" % files[0],
-                  file=sys.stderr)
+            sys.stderr.write("Can't read %s - skipping\n" % files[0])
             pass
         pass
     return

uncompyle6/bin/uncompile.py

@@ -1,13 +1,12 @@
 #!/usr/bin/env python
 # Mode: -*- python -*-
 #
-# Copyright (c) 2015-2016 by Rocky Bernstein
+# Copyright (c) 2015-2017 by Rocky Bernstein
 # Copyright (c) 2000-2002 by hartmut Goebel <h.goebel@crazy-compilers.com>
 #
-from __future__ import print_function
 import sys, os, getopt, time
 
-program, ext = os.path.splitext(os.path.basename(__file__))
+program = 'uncompyle6'
 
 __doc__ = """
 Usage:
@@ -65,11 +64,11 @@ def usage():
 
 
 def main_bin():
-    if not (sys.version_info[0:2] in ((2, 6), (2, 7),
-                                      (3, 1), (3, 2), (3, 3),
+    if not (sys.version_info[0:2] in ((2, 4), (2, 5), (2, 6), (2, 7),
+                                      (3, 2), (3, 3),
                                       (3, 4), (3, 5), (3, 6))):
-        print('Error: %s requires Python 2.6-2.7, or 3.1-3.6' % program,
-              file=sys.stderr)
+        sys.stderr.write('Error: %s requires Python 2.4 2.5 2.6, 2.7, '
+                         '3.2, 3.3, 3.4, 3.5, or 3.6' % program)
         sys.exit(-1)
 
     do_verify = recurse_dirs = False
@@ -84,8 +83,8 @@ def main_bin():
         opts, files = getopt.getopt(sys.argv[1:], 'hagtdrVo:c:p:',
                                     'help asm grammar linemaps recurse timestamp tree '
                                     'verify version showgrammar'.split(' '))
-    except getopt.GetoptError as e:
-        print('%s: %s' % (os.path.basename(sys.argv[0]), e),  file=sys.stderr)
+    except getopt.GetoptError(e):
+        sys.stderr.write('%s: %s\n' % (os.path.basename(sys.argv[0]), e))
         sys.exit(-1)
 
     options = {}
@@ -119,7 +118,7 @@ def main_bin():
         elif opt in ('--recurse', '-r'):
             recurse_dirs = True
         else:
-            print(opt, file=sys.stderr)
+            sys.stderr.write(opt)
             usage()
 
     # expand directory if specified
@@ -144,7 +143,7 @@ def main_bin():
         files = [f[sb_len:] for f in files]
 
     if not files:
-        print("No files given", file=sys.stderr)
+        sys.stderr.write("No files given\n")
         usage()
 
     if outfile == '-':

uncompyle6/disas.py

@@ -16,8 +16,6 @@ Second, we need structured instruction information for the
 want to run on Python 2.7.
 """
 
-from __future__ import print_function
-
 import sys
 from collections import deque
 
@@ -37,10 +35,9 @@ def disco(version, co, out=None, is_pypy=False):
 
     # store final output stream for case of error
     real_out = out or sys.stdout
-    print('# Python %s' % version, file=real_out)
+    real_out.write('# Python %s\n' % version)
     if co.co_filename:
-        print('# Embedded file name: %s' % co.co_filename,
-              file=real_out)
+        real_out.write('# Embedded file name: %s\n' % co.co_filename)
 
     scanner = get_scanner(version, is_pypy=is_pypy)
 
@@ -52,16 +49,15 @@ def disco_loop(disasm, queue, real_out):
     while len(queue) > 0:
         co = queue.popleft()
         if co.co_name != '<module>':
-            print('\n# %s line %d of %s' %
-                      (co.co_name, co.co_firstlineno, co.co_filename),
-                      file=real_out)
+            real_out.write('\n# %s line %d of %s\n' %
+                      (co.co_name, co.co_firstlineno, co.co_filename))
         tokens, customize = disasm(co)
         for t in tokens:
             if iscode(t.pattr):
                 queue.append(t.pattr)
             elif iscode(t.attr):
                 queue.append(t.attr)
-            print(t, file=real_out)
+            real_out.write(t)
             pass
         pass
 

uncompyle6/linenumbers.py

@@ -10,7 +10,7 @@ def line_number_mapping(pyc_filename, src_filename):
      source_size) = load_module(pyc_filename)
     try:
         code2 = load_file(src_filename)
-    except SyntaxError as e:
+    except SyntaxError, e:
         return str(e)
 
     queue = deque([code1, code2])

uncompyle6/main.py

@@ -1,4 +1,3 @@
-from __future__ import print_function
 import datetime, os, subprocess, sys, tempfile
 
 from uncompyle6 import verify, IS_PYPY
@@ -22,31 +21,36 @@ def decompile(
 
     # store final output stream for case of error
     real_out = out or sys.stdout
-    co_pypy_str = 'PyPy ' if is_pypy else ''
-    run_pypy_str = 'PyPy ' if IS_PYPY else ''
-    print('# uncompyle6 version %s\n'
-          '# %sPython bytecode %s%s\n# Decompiled from: %sPython %s' %
-          (VERSION, co_pypy_str, bytecode_version,
-           " (%d)" % magic_int if magic_int else "",
-           run_pypy_str, '\n# '.join(sys.version.split('\n'))),
-           file=real_out)
-    if co.co_filename:
-        print('# Embedded file name: %s' % co.co_filename,
-              file=real_out)
-    if timestamp:
-        print('# Compiled at: %s' % datetime.datetime.fromtimestamp(timestamp),
-              file=real_out)
-    if source_size:
-        print('# Size of source mod 2**32: %d bytes' % source_size,
-               file=real_out)
+    if is_pypy:
+        co_pypy_str = 'PyPy '
+    else:
+        co_pypy_str = ''
 
-    try:
-        pysource.deparse_code(bytecode_version, co, out, showasm, showast,
-                              showgrammar, code_objects=code_objects,
-                              is_pypy=is_pypy)
-    except pysource.SourceWalkerError as e:
-        # deparsing failed
-        raise pysource.SourceWalkerError(str(e))
+    if IS_PYPY:
+        run_pypy_str = 'PyPy '
+    else:
+        run_pypy_str = ''
+
+    if magic_int:
+        m = str(magic_int)
+    else:
+        m = ""
+    real_out.write('# uncompyle6 version %s\n'
+          '# %sPython bytecode %s%s\n# Decompiled from: %sPython %s\n' %
+          (VERSION, co_pypy_str, bytecode_version,
+           " (%s)" % m, run_pypy_str,
+           '\n# '.join(sys.version.split('\n'))))
+    if co.co_filename:
+        real_out.write('# Embedded file name: %s\n' % co.co_filename)
+    if timestamp:
+        real_out.write('# Compiled at: %s\n' %
+                       datetime.datetime.fromtimestamp(timestamp))
+    if source_size:
+        real_out.write('# Size of source mod 2**32: %d bytes\n' % source_size)
+
+    pysource.deparse_code(bytecode_version, co, out, showasm, showast,
+                          showgrammar, code_objects=code_objects,
+                          is_pypy=is_pypy)
 
 # For compatiblity
 uncompyle = decompile
@@ -128,7 +132,10 @@ def main(in_base, out_base, files, codes, outfile=None,
                 junk, outfile = tempfile.mkstemp(suffix=".py",
                                              prefix=prefix)
                 # Unbuffer output if possible
-                buffering = -1 if sys.stdout.isatty() else 0
+                if sys.stdout.isatty():
+                    buffering = -1
+                else:
+                    buffering = 0
                 sys.stdout = os.fdopen(sys.stdout.fileno(), 'w', buffering)
                 tee = subprocess.Popen(["tee", outfile], stdin=subprocess.PIPE)
                 os.dup2(tee.stdin.fileno(), sys.stdout.fileno())
@@ -145,9 +152,9 @@ def main(in_base, out_base, files, codes, outfile=None,
         try:
             decompile_file(infile, outstream, showasm, showast, showgrammar)
             tot_files += 1
-        except (ValueError, SyntaxError, ParserError, pysource.SourceWalkerError) as e:
+        except (ValueError, SyntaxError, ParserError, pysource.SourceWalkerError):
             sys.stdout.write("\n")
-            sys.stderr.write("\n# file %s\n# %s\n" % (infile, e))
+            sys.stderr.write("# file %s\n" % (infile))
             failed_files += 1
         except KeyboardInterrupt:
             if outfile:
@@ -181,24 +188,28 @@ def main(in_base, out_base, files, codes, outfile=None,
                         msg = verify.compare_code_with_srcfile(infile, current_outfile, weak_verify=weak_verify)
                         if not current_outfile:
                             if not msg:
-                                print('\n# okay decompiling %s' % infile)
+                                print '\n# okay decompiling %s' % infile
                                 okay_files += 1
                             else:
-                                print('\n# %s\n\t%s', infile, msg)
-                    except verify.VerifyCmpError as e:
+                                print '\n# %s\n\t%s', infile, msg
+                    except verify.VerifyCmpError, e:
                         print(e)
                         verify_failed_files += 1
                         os.rename(current_outfile, current_outfile + '_unverified')
                         sys.stderr.write("### Error Verifying %s\n" % filename)
                         sys.stderr.write(str(e) + "\n")
                         if not outfile:
+                            sys.stder.write("### Error Verifiying %s" %
+                                            filename)
+                            sys.stderr.write(e)
                             if raise_on_error:
                                 raise
                             pass
                         pass
                 pass
             elif do_verify:
-                sys.stderr.write("\n### uncompile successful, but no file to compare against\n")
+                sys.stderr.write("\n### uncompile successful, "
+                                 "but no file to compare against")
                 pass
             else:
                 okay_files += 1

uncompyle6/parser.py

@@ -3,11 +3,9 @@
 #  Copyright (c) 2000-2002 by hartmut Goebel <h.goebel@crazy-compilers.com>
 #  Copyright (c) 1999 John Aycock
 """
-Common uncompyle parser routines.
+Common parser routines.
 """
 
-from __future__ import print_function
-
 import sys
 
 from xdis.code import iscode
@@ -30,13 +28,13 @@ class PythonParser(GenericASTBuilder):
 
     def __init__(self, AST, start, debug):
         super(PythonParser, self).__init__(AST, start, debug)
-        self.collect = frozenset(
-            ['stmts', 'except_stmts', '_stmts', 'load_attrs',
-             'exprlist', 'kvlist', 'kwargs', 'come_froms', '_come_from',
-              # Python < 3
-             'print_items',
-             # PyPy:
-             'kvlist_n'])
+        self.collect = [
+            'stmts', 'except_stmts', '_stmts', 'load_attrs',
+            'exprlist', 'kvlist', 'kwargs', 'come_froms', '_come_from',
+            # Python < 3
+            'print_items',
+            # PyPy:
+            'kvlist_n']
 
     def ast_first_offset(self, ast):
         if hasattr(ast, 'offset'):
@@ -44,21 +42,25 @@ class PythonParser(GenericASTBuilder):
         else:
             return self.ast_first_offset(ast[0])
 
-    def add_unique_rule(self, rule, opname, count, customize):
+    def add_unique_rule(self, rule, opname, arg_count, customize):
         """Add rule to grammar, but only if it hasn't been added previously
-           opname and count are used in the customize() semantic the actions
-           to add the semantic action rule. Often, count is not used.
+           opname and stack_count are used in the customize() semantic
+           the actions to add the semantic action rule. Stack_count is
+           used in custom opcodes like MAKE_FUNCTION to indicate how
+           many arguments it has. Often it is not used.
         """
         if rule not in self.new_rules:
             # print("XXX ", rule) # debug
             self.new_rules.add(rule)
             self.addRule(rule, nop_func)
-            customize[opname] = count
+            customize[opname] = arg_count
             pass
         return
 
     def add_unique_rules(self, rules, customize):
-        """Add rules (a list of string) to grammar
+        """Add rules (a list of string) to grammar. Note that
+        the rules must not be those that set arg_count in the
+        custom dictionary.
         """
         for rule in rules:
             if len(rule) == 0:
@@ -68,7 +70,9 @@ class PythonParser(GenericASTBuilder):
         return
 
     def add_unique_doc_rules(self, rules_str, customize):
-        """Add rules (a docstring-like list of rules) to grammar
+        """Add rules (a docstring-like list of rules) to grammar.
+        Note that the rules must not be those that set arg_count in the
+        custom dictionary.
         """
         rules = [r.strip() for r in rules_str.split("\n")]
         self.add_unique_rules(rules, customize)
@@ -85,14 +89,17 @@ class PythonParser(GenericASTBuilder):
         for i in dir(self):
             setattr(self, i, None)
 
-    def debug_reduce(self, rule, tokens, parent, i):
+    def debug_reduce(self, rule, tokens, parent, last_token_pos):
         """Customized format and print for our kind of tokens
         which gets called in debugging grammar reduce rules
         """
         def fix(c):
             s = str(c)
-            i = s.find('_')
-            return s if i == -1 else s[:i]
+            last_token_pos = s.find('_')
+            if last_token_pos == -1:
+                return s
+            else:
+                return s[:last_token_pos]
 
         prefix = ''
         if parent and tokens:
@@ -104,13 +111,13 @@ class PythonParser(GenericASTBuilder):
             if hasattr(p_token, 'offset'):
                 prefix += "%3s" % fix(p_token.offset)
                 if len(rule[1]) > 1:
-                    prefix += '-%-3s ' % fix(tokens[i-1].offset)
+                    prefix += '-%-3s ' % fix(tokens[last_token_pos-1].offset)
                 else:
                     prefix += '     '
         else:
             prefix = '               '
 
-        print("%s%s ::= %s" % (prefix, rule[0], ' '.join(rule[1])))
+        print("%s%s ::= %s (%d)" % (prefix, rule[0], ' '.join(rule[1]), last_token_pos))
 
     def error(self, instructions, index):
         # Find the last line boundary
@@ -123,12 +130,15 @@ class PythonParser(GenericASTBuilder):
         err_token = instructions[index]
         print("Instruction context:")
         for i in range(start, finish):
-            indent = '   ' if i != index else '-> '
+            if i != index:
+                indent = '   '
+            else:
+                indent = '-> '
             print("%s%s" % (indent, instructions[i]))
         raise ParserError(err_token, err_token.offset)
 
     def typestring(self, token):
-        return token.type
+        return token.kind
 
     def nonterminal(self, nt, args):
         if nt in self.collect and len(args) > 1:
@@ -250,8 +260,11 @@ class PythonParser(GenericASTBuilder):
 
         stmt ::= return_stmt
         return_stmt ::= ret_expr RETURN_VALUE
+        return_stmt_lambda ::= ret_expr RETURN_VALUE_LAMBDA
+
         return_stmts ::= return_stmt
         return_stmts ::= _stmts return_stmt
+
         """
         pass
 
@@ -526,7 +539,9 @@ class PythonParser(GenericASTBuilder):
         stmt ::= return_lambda
         stmt ::= conditional_lambda
 
-        return_lambda ::= ret_expr RETURN_VALUE LAMBDA_MARKER
+        return_lambda ::= ret_expr RETURN_VALUE_LAMBDA LAMBDA_MARKER
+        return_lambda ::= ret_expr RETURN_VALUE_LAMBDA
+
         conditional_lambda ::= expr jmp_false return_if_stmt return_stmt LAMBDA_MARKER
 
         cmp ::= cmp_list
@@ -722,7 +737,7 @@ def get_python_parser(
             else:
                 p = parse3.Python3ParserSingle(debug_parser)
     p.version = version
-    # p.dumpGrammar() # debug
+    # p.dump_grammar() # debug
     return p
 
 class PythonParserSingle(PythonParser):

uncompyle6/parsers/astnode.py

@@ -16,7 +16,7 @@ class AST(spark_AST):
         return self.__repr1__('', None)
 
     def __repr1__(self, indent, sibNum=None):
-        rv = str(self.type)
+        rv = str(self.kind)
         if sibNum is not None:
             rv = "%2d. %s" % (sibNum, rv)
         enumerate_children = False

uncompyle6/parsers/parse15.py

@@ -29,8 +29,8 @@ class Python15ParserSingle(Python21Parser, PythonParserSingle):
 if __name__ == '__main__':
     # Check grammar
     p = Python15Parser()
-    p.checkGrammar()
-    p.dumpGrammar()
+    p.check_grammar()
+    p.dump_grammar()
 
 # local variables:
 # tab-width: 4

uncompyle6/parsers/parse2.py

@@ -12,8 +12,6 @@ If we succeed in creating a parse tree, then we have a Python program
 that a later phase can turn into a sequence of ASCII text.
 """
 
-from __future__ import print_function
-
 from uncompyle6.parser import PythonParser, PythonParserSingle, nop_func
 from uncompyle6.parsers.astnode import AST
 from spark_parser import DEFAULT_DEBUG as PARSER_DEFAULT_DEBUG
@@ -397,6 +395,8 @@ class Python2Parser(PythonParser):
         return
 
     def reduce_is_invalid(self, rule, ast, tokens, first, last):
+        if tokens is None:
+            return False
         lhs = rule[0]
         if lhs in ('augassign1', 'augassign2') and ast[0][0] == 'and':
             return True
@@ -417,4 +417,4 @@ class Python2ParserSingle(Python2Parser, PythonParserSingle):
 if __name__ == '__main__':
     # Check grammar
     p = Python2Parser()
-    p.checkGrammar()
+    p.check_grammar()

uncompyle6/parsers/parse21.py

@@ -33,8 +33,8 @@ class Python21ParserSingle(Python22Parser, PythonParserSingle):
 if __name__ == '__main__':
     # Check grammar
     p = Python21Parser()
-    p.checkGrammar()
-    p.dumpGrammar()
+    p.check_grammar()
+    p.dump_grammar()
 
 # local variables:
 # tab-width: 4

uncompyle6/parsers/parse22.py

@@ -26,8 +26,8 @@ class Python22ParserSingle(Python23Parser, PythonParserSingle):
 if __name__ == '__main__':
     # Check grammar
     p = Python22Parser()
-    p.checkGrammar()
-    p.dumpGrammar()
+    p.check_grammar()
+    p.dump_grammar()
 
 # local variables:
 # tab-width: 4

uncompyle6/parsers/parse23.py

@@ -67,8 +67,8 @@ class Python23ParserSingle(Python23Parser, PythonParserSingle):
 if __name__ == '__main__':
     # Check grammar
     p = Python23Parser()
-    p.checkGrammar()
-    p.dumpGrammar()
+    p.check_grammar()
+    p.dump_grammar()
 
 # local variables:
 # tab-width: 4

uncompyle6/parsers/parse24.py

@@ -55,13 +55,14 @@ class Python24Parser(Python25Parser):
         invalid = super(Python24Parser,
                         self).reduce_is_invalid(rule, ast,
                                                 tokens, first, last)
-        if invalid:
+        if invalid or tokens is None:
             return invalid
 
-        # FiXME: this code never gets called...
         lhs = rule[0]
         if lhs == 'nop_stmt':
-            return not int(tokens[first].pattr) == tokens[last].offset
+            l = len(tokens)
+            if 0 <= l < len(tokens):
+                return not int(tokens[first].pattr) == tokens[last].offset
 
         return False
 
@@ -71,4 +72,4 @@ class Python24ParserSingle(Python24Parser, PythonParserSingle):
 if __name__ == '__main__':
     # Check grammar
     p = Python24Parser()
-    p.checkGrammar()
+    p.check_grammar()

uncompyle6/parsers/parse25.py

@@ -60,4 +60,4 @@ class Python25ParserSingle(Python26Parser, PythonParserSingle):
 if __name__ == '__main__':
     # Check grammar
     p = Python25Parser()
-    p.checkGrammar()
+    p.check_grammar()

uncompyle6/parsers/parse26.py

@@ -247,7 +247,9 @@ class Python26Parser(Python2Parser):
         and  ::= expr JUMP_IF_FALSE POP_TOP expr JUMP_IF_FALSE POP_TOP
         cmp_list ::= expr cmp_list1 ROT_TWO COME_FROM POP_TOP _come_from
 
-        conditional_lambda ::= expr jmp_false_then return_if_stmt return_stmt LAMBDA_MARKER
+        return_if_lambda   ::= RETURN_END_IF_LAMBDA POP_TOP
+        conditional_lambda ::= expr jmp_false_then expr return_if_lambda
+                               return_stmt_lambda LAMBDA_MARKER
         """
 
     def add_custom_rules(self, tokens, customize):
@@ -258,7 +260,7 @@ class Python26Parser(Python2Parser):
         invalid = super(Python26Parser,
                         self).reduce_is_invalid(rule, ast,
                                                 tokens, first, last)
-        if invalid:
+        if invalid or tokens is None:
             return invalid
         if rule == ('and', ('expr', 'jmp_false', 'expr', '\\e_come_from_opt')):
             # Test that jmp_false jumps to the end of "and"
@@ -274,10 +276,10 @@ class Python26ParserSingle(Python2Parser, PythonParserSingle):
 if __name__ == '__main__':
     # Check grammar
     p = Python26Parser()
-    p.checkGrammar()
+    p.check_grammar()
     from uncompyle6 import PYTHON_VERSION, IS_PYPY
     if PYTHON_VERSION == 2.6:
-        lhs, rhs, tokens, right_recursive = p.checkSets()
+        lhs, rhs, tokens, right_recursive = p.check_sets()
         from uncompyle6.scanner import get_scanner
         s = get_scanner(PYTHON_VERSION, IS_PYPY)
         opcode_set = set(s.opc.opname).union(set(

uncompyle6/parsers/parse27.py

@@ -94,6 +94,10 @@ class Python27Parser(Python2Parser):
                 WITH_CLEANUP END_FINALLY
 
         # Common with 2.6
+        return_if_lambda   ::= RETURN_END_IF_LAMBDA COME_FROM
+        conditional_lambda ::= expr jmp_false expr return_if_lambda
+                               return_stmt_lambda LAMBDA_MARKER
+
         while1stmt ::= SETUP_LOOP return_stmts bp_come_from
         while1stmt ::= SETUP_LOOP return_stmts COME_FROM
         """
@@ -125,10 +129,10 @@ class Python27ParserSingle(Python27Parser, PythonParserSingle):
 if __name__ == '__main__':
     # Check grammar
     p = Python27Parser()
-    p.checkGrammar()
+    p.check_grammar()
     from uncompyle6 import PYTHON_VERSION, IS_PYPY
     if PYTHON_VERSION == 2.7:
-        lhs, rhs, tokens, right_recursive = p.checkSets()
+        lhs, rhs, tokens, right_recursive = p.check_sets()
         from uncompyle6.scanner import get_scanner
         s = get_scanner(PYTHON_VERSION, IS_PYPY)
         opcode_set = set(s.opc.opname).union(set(
@@ -144,4 +148,5 @@ if __name__ == '__main__':
                              for t in remain_tokens])
         remain_tokens = set(remain_tokens) - opcode_set
         print(remain_tokens)
-        # p.dumpGrammar()
+        p.check_grammar()
+        p.dump_grammar()

uncompyle6/parsers/parse3.py

@@ -15,11 +15,10 @@ If we succeed in creating a parse tree, then we have a Python program
 that a later phase can turn into a sequence of ASCII text.
 """
 
-from __future__ import print_function
-
 from uncompyle6.parser import PythonParser, PythonParserSingle, nop_func
 from uncompyle6.parsers.astnode import AST
 from spark_parser import DEFAULT_DEBUG as PARSER_DEFAULT_DEBUG
+from xdis import PYTHON3
 
 class Python3Parser(PythonParser):
 
@@ -156,8 +155,13 @@ class Python3Parser(PythonParser):
         # of missing "else" clauses. Therefore we include grammar
         # rules with and without ELSE.
 
-        ifelsestmt ::= testexpr c_stmts_opt JUMP_FORWARD else_suite opt_come_from_except
-        ifelsestmt ::= testexpr c_stmts_opt jump_forward_else else_suite _come_from
+        ifelsestmt ::= testexpr c_stmts_opt JUMP_FORWARD
+                       else_suite opt_come_from_except
+        ifelsestmt ::= testexpr c_stmts_opt jump_forward_else
+                       else_suite _come_from
+
+        # ifelsestmt ::= testexpr c_stmts_opt jump_forward_else
+        #                passstmt  _come_from
 
         ifelsestmtc ::= testexpr c_stmts_opt JUMP_ABSOLUTE else_suitec
         ifelsestmtc ::= testexpr c_stmts_opt jump_absolute_else else_suitec
@@ -253,8 +257,14 @@ class Python3Parser(PythonParser):
                 POP_BLOCK LOAD_CONST COME_FROM_WITH
                 WITH_CLEANUP END_FINALLY
 
+        ## FIXME: Right now we have erroneous jump targets
+        ## This below is probably not correct when the COME_FROM is put in the right place
         and ::= expr jmp_false expr COME_FROM
         or  ::= expr jmp_true  expr COME_FROM
+
+        # # something like the below is needed when the jump targets are fixed
+        ## or  ::= expr JUMP_IF_TRUE_OR_POP COME_FROM expr
+        ## and ::= expr JUMP_IF_FALSE_OR_POP COME_FROM expr
         '''
 
     def p_misc3(self, args):
@@ -417,6 +427,13 @@ class Python3Parser(PythonParser):
         # a JUMP_ABSOLUTE with no COME_FROM
         conditional    ::= expr jmp_false expr jump_absolute_else expr
 
+        return_if_lambda   ::= RETURN_END_IF_LAMBDA
+        conditional_lambda ::= expr jmp_false return_stmt_lambda
+                               return_stmt_lambda LAMBDA_MARKER
+        conditional_lambda ::= expr jmp_false expr return_if_lambda
+                               return_stmt_lambda LAMBDA_MARKER
+
+
         expr ::= LOAD_CLASSNAME
 
         # Python 3.4+
@@ -427,7 +444,7 @@ class Python3Parser(PythonParser):
     @staticmethod
     def call_fn_name(token):
         """Customize CALL_FUNCTION to add the number of positional arguments"""
-        return '%s_%i' % (token.type, token.attr)
+        return '%s_%i' % (token.kind, token.attr)
 
     def custom_build_class_rule(self, opname, i, token, tokens, customize):
         '''
@@ -443,16 +460,16 @@ class Python3Parser(PythonParser):
         # FIXME: I bet this can be simplified
         # look for next MAKE_FUNCTION
         for i in range(i+1, len(tokens)):
-            if tokens[i].type.startswith('MAKE_FUNCTION'):
+            if tokens[i].kind.startswith('MAKE_FUNCTION'):
                 break
-            elif tokens[i].type.startswith('MAKE_CLOSURE'):
+            elif tokens[i].kind.startswith('MAKE_CLOSURE'):
                 break
             pass
         assert i < len(tokens), "build_class needs to find MAKE_FUNCTION or MAKE_CLOSURE"
-        assert tokens[i+1].type == 'LOAD_CONST', \
+        assert tokens[i+1].kind == 'LOAD_CONST', \
           "build_class expecting CONST after MAKE_FUNCTION/MAKE_CLOSURE"
         for i in range(i, len(tokens)):
-            if tokens[i].type == 'CALL_FUNCTION':
+            if tokens[i].kind == 'CALL_FUNCTION':
                 call_fn_tok = tokens[i]
                 break
         assert call_fn_tok, "build_class custom rule needs to find CALL_FUNCTION"
@@ -493,7 +510,7 @@ class Python3Parser(PythonParser):
         # Yes, this computation based on instruction name is a little bit hoaky.
         nak = ( len(opname)-len('CALL_FUNCTION') ) // 3
 
-        token.type = self.call_fn_name(token)
+        token.kind = self.call_fn_name(token)
         uniq_param = args_kw + args_pos
         if self.version == 3.5 and opname.startswith('CALL_FUNCTION_VAR'):
             # Python 3.5 changes the stack position of *args. KW args come
@@ -505,39 +522,42 @@ class Python3Parser(PythonParser):
                 kw = ''
             rule = ('call_function ::= expr expr ' +
                     ('pos_arg ' * args_pos) +
-                    ('kwarg ' * args_kw) + kw + token.type)
-            self.add_unique_rule(rule, token.type, uniq_param, customize)
+                    ('kwarg ' * args_kw) + kw + token.kind)
+            self.add_unique_rule(rule, token.kind, uniq_param, customize)
         if self.version >= 3.6 and opname == 'CALL_FUNCTION_EX_KW':
             rule = ('call_function36 ::= '
                     'expr build_tuple_unpack_with_call build_map_unpack_with_call '
                     'CALL_FUNCTION_EX_KW_1')
-            self.add_unique_rule(rule, token.type, uniq_param, customize)
+            self.add_unique_rule(rule, token.kind, uniq_param, customize)
             rule = 'call_function ::= call_function36'
         else:
             rule = ('call_function ::= expr ' +
                     ('pos_arg ' * args_pos) +
                     ('kwarg ' * args_kw) +
-                    'expr ' * nak + token.type)
+                    'expr ' * nak + token.kind)
 
-        self.add_unique_rule(rule, token.type, uniq_param, customize)
+        self.add_unique_rule(rule, token.kind, uniq_param, customize)
         if self.version >= 3.5:
             rule = ('async_call_function ::= expr ' +
                     ('pos_arg ' * args_pos) +
                     ('kwarg ' * args_kw) +
-                    'expr ' * nak + token.type +
+                    'expr ' * nak + token.kind +
                     ' GET_AWAITABLE LOAD_CONST YIELD_FROM')
-            self.add_unique_rule(rule, token.type, uniq_param, customize)
-            self.add_unique_rule('expr ::= async_call_function', token.type, uniq_param, customize)
+            self.add_unique_rule(rule, token.kind, uniq_param, customize)
+            self.add_unique_rule('expr ::= async_call_function', token.kind, uniq_param, customize)
 
         rule = ('classdefdeco2 ::= LOAD_BUILD_CLASS mkfunc %s%s_%d'
                 %  (('expr ' * (args_pos-1)), opname, args_pos))
-        self.add_unique_rule(rule, token.type, uniq_param, customize)
+        self.add_unique_rule(rule, token.kind, uniq_param, customize)
 
     def add_make_function_rule(self, rule, opname, attr, customize):
         """Python 3.3 added a an addtional LOAD_CONST before MAKE_FUNCTION and
         this has an effect on many rules.
         """
-        new_rule = rule % (('LOAD_CONST ') * (1 if  self.version >= 3.3 else 0))
+        if self.version >= 3.3:
+            new_rule = rule % (('LOAD_CONST ') * 1)
+        else:
+            new_rule = rule % (('LOAD_CONST ') * 0)
         self.add_unique_rule(new_rule, opname, attr, customize)
 
     def add_custom_rules(self, tokens, customize):
@@ -605,7 +625,7 @@ class Python3Parser(PythonParser):
             call_function ::= expr CALL_METHOD
         """
         for i, token in enumerate(tokens):
-            opname = token.type
+            opname = token.kind
             opname_base = opname[:opname.rfind('_')]
 
             if opname == 'PyPy':
@@ -889,7 +909,11 @@ class Python3Parser(PythonParser):
         elif lhs == 'annotate_tuple':
             return not isinstance(tokens[first].attr, tuple)
         elif lhs == 'kwarg':
-            return not isinstance(tokens[first].attr, str)
+            arg = tokens[first].attr
+            if PYTHON3:
+                return not isinstance(arg, str)
+            else:
+                return not (isinstance(arg, str) or isinstance(arg, unicode))
         elif lhs == 'while1elsestmt':
             # if SETUP_LOOP target spans the else part, then this is
             # not while1else. Also do for whileTrue?
@@ -898,7 +922,8 @@ class Python3Parser(PythonParser):
                 last += 1
             return tokens[first].attr == tokens[last].offset
         elif lhs == 'while1stmt':
-            if tokens[last] in ('COME_FROM_LOOP', 'JUMP_BACK'):
+            if (0 <= last < len(tokens)
+                and tokens[last] in ('COME_FROM_LOOP', 'JUMP_BACK')):
                 # jump_back should be right afer SETUP_LOOP. Test?
                 last += 1
             while last < len(tokens) and isinstance(tokens[last].offset, str):
@@ -942,10 +967,10 @@ def info(args):
             p = Python32Parser()
         elif arg == '3.0':
             p = Python30Parser()
-    p.checkGrammar()
+    p.check_grammar()
     if len(sys.argv) > 1 and sys.argv[1] == 'dump':
         print('-' * 50)
-        p.dumpGrammar()
+        p.dump_grammar()
 
 if __name__ == '__main__':
     import sys

uncompyle6/parsers/parse30.py

@@ -2,7 +2,6 @@
 """
 spark grammar differences over Python 3.1 for Python 3.0.
 """
-from __future__ import print_function
 
 from uncompyle6.parser import PythonParserSingle
 from uncompyle6.parsers.parse31 import Python31Parser

uncompyle6/parsers/parse31.py

@@ -2,7 +2,6 @@
 """
 spark grammar differences over Python 3.2 for Python 3.1.
 """
-from __future__ import print_function
 
 from uncompyle6.parser import PythonParserSingle
 from uncompyle6.parsers.parse32 import Python32Parser

uncompyle6/parsers/parse32.py

@@ -2,8 +2,6 @@
 """
 spark grammar differences over Python 3 for Python 3.2.
 """
-from __future__ import print_function
-
 from uncompyle6.parser import PythonParserSingle
 from uncompyle6.parsers.parse3 import Python3Parser
 
@@ -44,7 +42,7 @@ class Python32Parser(Python3Parser):
     def add_custom_rules(self, tokens, customize):
         super(Python32Parser, self).add_custom_rules(tokens, customize)
         for i, token in enumerate(tokens):
-            opname = token.type
+            opname = token.kind
             if opname.startswith('MAKE_FUNCTION_A'):
                 args_pos, args_kw, annotate_args  = token.attr
                 # Check that there are 2 annotated params?

uncompyle6/parsers/parse33.py

@@ -2,7 +2,6 @@
 """
 spark grammar differences over Python 3.2 for Python 3.3.
 """
-from __future__ import print_function
 
 from uncompyle6.parser import PythonParserSingle
 from uncompyle6.parsers.parse32 import Python32Parser

uncompyle6/parsers/parse34.py

@@ -29,10 +29,10 @@ class Python34ParserSingle(Python34Parser, PythonParserSingle):
 if __name__ == '__main__':
     # Check grammar
     p = Python34Parser()
-    p.checkGrammar()
+    p.check_grammar()
     from uncompyle6 import PYTHON_VERSION, IS_PYPY
     if PYTHON_VERSION == 3.4:
-        lhs, rhs, tokens, right_recursive = p.checkSets()
+        lhs, rhs, tokens, right_recursive = p.check_sets()
         from uncompyle6.scanner import get_scanner
         s = get_scanner(PYTHON_VERSION, IS_PYPY)
         opcode_set = set(s.opc.opname).union(set(

uncompyle6/parsers/parse35.py

@@ -2,7 +2,6 @@
 """
 spark grammar differences over Python 3.4 for Python 3.5.
 """
-from __future__ import print_function
 
 from uncompyle6.parser import PythonParserSingle
 from spark_parser import DEFAULT_DEBUG as PARSER_DEFAULT_DEBUG
@@ -143,7 +142,7 @@ class Python35Parser(Python34Parser):
     def add_custom_rules(self, tokens, customize):
         super(Python35Parser, self).add_custom_rules(tokens, customize)
         for i, token in enumerate(tokens):
-            opname = token.type
+            opname = token.kind
             if opname == 'BUILD_MAP_UNPACK_WITH_CALL':
                 nargs = token.attr % 256
                 map_unpack_n = "map_unpack_%s" % nargs
@@ -153,7 +152,7 @@ class Python35Parser(Python34Parser):
                 self.add_unique_rule(rule, opname, token.attr, customize)
                 call_token = tokens[i+1]
                 if self.version == 3.5:
-                    rule = 'call_function ::= expr unmapexpr ' + call_token.type
+                    rule = 'call_function ::= expr unmapexpr ' + call_token.kind
                     self.add_unique_rule(rule, opname, token.attr, customize)
                 pass
             pass
@@ -165,10 +164,10 @@ class Python35ParserSingle(Python35Parser, PythonParserSingle):
 if __name__ == '__main__':
     # Check grammar
     p = Python35Parser()
-    p.checkGrammar()
+    p.check_grammar()
     from uncompyle6 import PYTHON_VERSION, IS_PYPY
     if PYTHON_VERSION == 3.5:
-        lhs, rhs, tokens, right_recursive = p.checkSets()
+        lhs, rhs, tokens, right_recursive = p.check_sets()
         from uncompyle6.scanner import get_scanner
         s = get_scanner(PYTHON_VERSION, IS_PYPY)
         opcode_set = set(s.opc.opname).union(set(

uncompyle6/parsers/parse36.py

@@ -2,7 +2,6 @@
 """
 spark grammar differences over Python 3.5 for Python 3.6.
 """
-from __future__ import print_function
 
 from uncompyle6.parser import PythonParserSingle
 from spark_parser import DEFAULT_DEBUG as PARSER_DEFAULT_DEBUG
@@ -37,7 +36,7 @@ class Python36Parser(Python35Parser):
     def add_custom_rules(self, tokens, customize):
         super(Python36Parser, self).add_custom_rules(tokens, customize)
         for i, token in enumerate(tokens):
-            opname = token.type
+            opname = token.kind
 
             if opname == 'FORMAT_VALUE':
                 rules_str = """
@@ -65,10 +64,10 @@ class Python36Parser(Python35Parser):
 
         if opname.startswith('CALL_FUNCTION_KW'):
             values = 'expr ' * token.attr
-            rule = 'call_function ::= expr kwargs_only_36 {token.type}'.format(**locals())
-            self.add_unique_rule(rule, token.type, token.attr, customize)
+            rule = 'call_function ::= expr kwargs_only_36 {token.kind}'.format(**locals())
+            self.add_unique_rule(rule, token.kind, token.attr, customize)
             rule = 'kwargs_only_36 ::= {values} LOAD_CONST'.format(**locals())
-            self.add_unique_rule(rule, token.type, token.attr, customize)
+            self.add_unique_rule(rule, token.kind, token.attr, customize)
         else:
             super(Python36Parser, self).custom_classfunc_rule(opname, token, customize)
 
@@ -79,10 +78,10 @@ class Python36ParserSingle(Python36Parser, PythonParserSingle):
 if __name__ == '__main__':
     # Check grammar
     p = Python36Parser()
-    p.checkGrammar()
+    p.check_grammar()
     from uncompyle6 import PYTHON_VERSION, IS_PYPY
     if PYTHON_VERSION == 3.6:
-        lhs, rhs, tokens, right_recursive = p.checkSets()
+        lhs, rhs, tokens, right_recursive = p.check_sets()
         from uncompyle6.scanner import get_scanner
         s = get_scanner(PYTHON_VERSION, IS_PYPY)
         opcode_set = set(s.opc.opname).union(set(

uncompyle6/parsers/parse37.py

@@ -2,7 +2,6 @@
 """
 spark grammar differences over Python 3.6 for Python 3.7
 """
-from __future__ import print_function
 
 from uncompyle6.parser import PythonParserSingle
 from spark_parser import DEFAULT_DEBUG as PARSER_DEFAULT_DEBUG
@@ -21,10 +20,10 @@ class Python37ParserSingle(Python37Parser, PythonParserSingle):
 if __name__ == '__main__':
     # Check grammar
     p = Python37Parser()
-    p.checkGrammar()
+    p.check_grammar()
     from uncompyle6 import PYTHON_VERSION, IS_PYPY
     if PYTHON_VERSION == 3.7:
-        lhs, rhs, tokens, right_recursive = p.checkSets()
+        lhs, rhs, tokens, right_recursive = p.check_sets()
         from uncompyle6.scanner import get_scanner
         s = get_scanner(PYTHON_VERSION, IS_PYPY)
         opcode_set = set(s.opc.opname).union(set(

uncompyle6/scanner.py

@@ -1,4 +1,4 @@
-#  Copyright (c) 2016 by Rocky Bernstein
+#  Copyright (c) 2016-2017 by Rocky Bernstein
 #  Copyright (c) 2005 by Dan Pascu <dan@windowmaker.org>
 #  Copyright (c) 2000-2002 by hartmut Goebel <h.goebel@crazy-compilers.com>
 #  Copyright (c) 1999 John Aycock
@@ -10,8 +10,6 @@ scanner/ingestion module. From here we call various version-specific
 scanners, e.g. for Python 2.7 or 3.4.
 """
 
-from __future__ import print_function
-
 import sys
 
 from uncompyle6 import PYTHON3, IS_PYPY
@@ -22,7 +20,7 @@ from xdis.magics import py_str2float
 # The byte code versions we support
 PYTHON_VERSIONS = (1.5,
                    2.1, 2.2, 2.3, 2.4, 2.5, 2.6, 2.7,
-                   3.0, 3.1, 3.2, 3.3, 3.4, 3.5, 3.6)
+                   3.0, 3.1, 3.2, 3.3, 3.4, 3.5, 3.6, 3.7)
 
 # FIXME: DRY
 if PYTHON3:
@@ -56,7 +54,7 @@ class Scanner(object):
 
         if version in PYTHON_VERSIONS:
             if is_pypy:
-                v_str = "opcode_pypy%s" % (int(version * 10))
+                v_str = "opcode_%spypy" % (int(version * 10))
             else:
                 v_str = "opcode_%s" % (int(version * 10))
             exec("from xdis.opcodes import %s" % v_str)
@@ -65,6 +63,7 @@ class Scanner(object):
             raise TypeError("%s is not a Python version I know about" % version)
 
         self.opname = self.opc.opname
+
         # FIXME: This weird Python2 behavior is not Python3
         self.resetTokenClass()
 
@@ -101,7 +100,7 @@ class Scanner(object):
     def print_bytecode(self):
         for i in self.op_range(0, len(self.code)):
             op = self.code[i]
-            if op in self.JUMP_OPs:
+            if op in self.JUMP_OPS:
                 dest = self.get_target(i, op)
                 print('%i\t%s\t%i' % (i, self.opname[op], dest))
             else:

uncompyle6/scanners/pypy27.py

@@ -1,4 +1,4 @@
-#  Copyright (c) 2016 by Rocky Bernstein
+#  Copyright (c) 2016-2017 by Rocky Bernstein
 """
 Python PyPy 2.7 bytecode scanner/deparser
 
@@ -10,8 +10,8 @@ information for later use in deparsing.
 import uncompyle6.scanners.scanner27 as scan
 
 # bytecode verification, verify(), uses JUMP_OPs from here
-from xdis.opcodes import opcode_pypy27
-JUMP_OPs = opcode_pypy27.JUMP_OPs
+from xdis.opcodes import opcode_27pypy
+JUMP_OPS = opcode_27pypy.JUMP_OPS
 
 # We base this off of 2.6 instead of the other way around
 # because we cleaned things up this way.

uncompyle6/scanners/pypy35.py

@@ -8,9 +8,9 @@ make things easier for decompilation.
 
 import uncompyle6.scanners.scanner35 as scan
 
-# bytecode verification, verify(), uses JUMP_OPs from here
+# bytecode verification, verify(), uses JUMP_OPS from here
 from xdis.opcodes import opcode_35 as opc  # is this right?
-JUMP_OPs = map(lambda op: opc.opname[op], opc.hasjrel + opc.hasjabs)
+JUMP_OPs = opc.JUMP_OPS
 
 # We base this off of 3.5
 class ScannerPyPy35(scan.Scanner35):

uncompyle6/scanners/scanner15.py

@@ -11,15 +11,15 @@ import uncompyle6.scanners.scanner21 as scan
 
 # bytecode verification, verify(), uses JUMP_OPs from here
 from xdis.opcodes import opcode_15
-JUMP_OPs = opcode_15.JUMP_OPs
+JUMP_OPS = opcode_15.JUMP_OPS
 
-# We base this off of 2.2 instead of the other way around
+# We base this off of 2.1 instead of the other way around
 # because we cleaned things up this way.
 # The history is that 2.7 support is the cleanest,
 # then from that we got 2.6 and so on.
 class Scanner15(scan.Scanner21):
     def __init__(self, show_asm=False):
-        scan.Scanner21.__init__(self, show_asm)
+        scan.Scanner21.__init__(self, show_asm=False)
         self.opc = opcode_15
         self.opname = opcode_15.opname
         self.version = 1.5

uncompyle6/scanners/scanner2.py

@@ -20,9 +20,13 @@ For example:
 Finally we save token information.
 """
 
-from __future__ import print_function
+from uncompyle6 import PYTHON_VERSION
+
+if PYTHON_VERSION < 2.6:
+    from xdis.namedtuple24 import namedtuple
+else:
+    from collections import namedtuple
 
-from collections import namedtuple
 from array import array
 
 from uncompyle6.scanner import L65536
@@ -85,7 +89,9 @@ class Scanner2(Scanner):
         cause specific rules for the specific number of arguments they take.
         """
 
-        show_asm = self.show_asm if not show_asm else show_asm
+        if not show_asm:
+            show_asm = self.show_asm
+
         # show_asm = 'after'
         if show_asm in ('both', 'before'):
             from xdis.bytecode import Bytecode
@@ -93,12 +99,18 @@ class Scanner2(Scanner):
             for instr in bytecode.get_instructions(co):
                 print(instr._disassemble())
 
-        # Container for tokens
+        # list of tokens/instructions
         tokens = []
 
+        # "customize" is a dict whose keys are nonterminals
+        # and the value is the argument stack entries for that
+        # nonterminal. The count is a little hoaky. It is mostly
+        # not used, but sometimes it is.
+        # "customize" is a dict whose keys are nonterminals
         customize = {}
+
         if self.is_pypy:
-            customize['PyPy'] = 1
+            customize['PyPy'] = 0
 
         Token = self.Token # shortcut
 
@@ -1026,8 +1038,10 @@ class Scanner2(Scanner):
 
                             # FIXME: rocky: I think we need something like this...
                             if offset not in set(self.ignore_if) or self.version == 2.7:
-                                source = (self.setup_loops[label]
-                                          if label in self.setup_loops else offset)
+                                if label in self.setup_loops:
+                                    source = self.setup_loops[label]
+                                else:
+                                    source = offset
                                 targets[label] = targets.get(label, []) + [source]
                             pass
 

uncompyle6/scanners/scanner21.py

@@ -1,4 +1,4 @@
-#  Copyright (c) 2016 by Rocky Bernstein
+#  Copyright (c) 2016-2017 by Rocky Bernstein
 """
 Python 2.1 bytecode scanner/deparser
 
@@ -11,7 +11,7 @@ import uncompyle6.scanners.scanner22 as scan
 
 # bytecode verification, verify(), uses JUMP_OPs from here
 from xdis.opcodes import opcode_21
-JUMP_OPs = opcode_21.JUMP_OPs
+JUMP_OPS = opcode_21.JUMP_OPS
 
 # We base this off of 2.2 instead of the other way around
 # because we cleaned things up this way.
@@ -19,7 +19,7 @@ JUMP_OPs = opcode_21.JUMP_OPs
 # then from that we got 2.6 and so on.
 class Scanner21(scan.Scanner22):
     def __init__(self, show_asm=False):
-        scan.Scanner22.__init__(self, show_asm)
+        scan.Scanner22.__init__(self, show_asm=False)
         self.opc = opcode_21
         self.opname = opcode_21.opname
         self.version = 2.1

uncompyle6/scanners/scanner22.py

@@ -1,4 +1,4 @@
-#  Copyright (c) 2016 by Rocky Bernstein
+#  Copyright (c) 2016-2017 by Rocky Bernstein
 """
 Python 2.2 bytecode ingester.
 
@@ -11,7 +11,7 @@ import uncompyle6.scanners.scanner23 as scan
 
 # bytecode verification, verify(), uses JUMP_OPs from here
 from xdis.opcodes import opcode_22
-JUMP_OPs = opcode_22.JUMP_OPs
+JUMP_OPS = opcode_22.JUMP_OPS
 
 # We base this off of 2.3 instead of the other way around
 # because we cleaned things up this way.
@@ -19,7 +19,7 @@ JUMP_OPs = opcode_22.JUMP_OPs
 # then from that we got 2.6 and so on.
 class Scanner22(scan.Scanner23):
     def __init__(self, show_asm=False):
-        scan.Scanner23.__init__(self, show_asm)
+        scan.Scanner23.__init__(self, show_asm=False)
         self.opc = opcode_22
         self.opname = opcode_22.opname
         self.version = 2.2
@@ -30,5 +30,5 @@ class Scanner22(scan.Scanner23):
 
     def ingest22(self, co, classname=None, code_objects={}, show_asm=None):
         tokens, customize = self.parent_ingest(co, classname, code_objects, show_asm)
-        tokens = [t for t in tokens if t.type != 'SET_LINENO']
+        tokens = [t for t in tokens if t.kind != 'SET_LINENO']
         return tokens, customize

uncompyle6/scanners/scanner23.py

@@ -1,4 +1,4 @@
-#  Copyright (c) 2016 by Rocky Bernstein
+#  Copyright (c) 2016-2017 by Rocky Bernstein
 """
 Python 2.3 bytecode scanner/deparser
 
@@ -10,7 +10,7 @@ import uncompyle6.scanners.scanner24 as scan
 
 # bytecode verification, verify(), uses JUMP_OPs from here
 from xdis.opcodes import opcode_23
-JUMP_OPs = opcode_23.JUMP_OPs
+JUMP_OPS = opcode_23.JUMP_OPS
 
 # We base this off of 2.4 instead of the other way around
 # because we cleaned things up this way.

uncompyle6/scanners/scanner24.py

@@ -1,4 +1,4 @@
-#  Copyright (c) 2016 by Rocky Bernstein
+#  Copyright (c) 2016-2017 by Rocky Bernstein
 """
 Python 2.4 bytecode scanner/deparser
 
@@ -10,7 +10,7 @@ import uncompyle6.scanners.scanner25 as scan
 
 # bytecode verification, verify(), uses JUMP_OPs from here
 from xdis.opcodes import opcode_24
-JUMP_OPs = opcode_24.JUMP_OPs
+JUMP_OPS = opcode_24.JUMP_OPS
 
 # We base this off of 2.5 instead of the other way around
 # because we cleaned things up this way.

uncompyle6/scanners/scanner25.py

@@ -1,4 +1,4 @@
-#  Copyright (c) 2015-2016 by Rocky Bernstein
+#  Copyright (c) 2015-2017 by Rocky Bernstein
 """
 Python 2.5 bytecode scanner/deparser
 
@@ -11,7 +11,7 @@ import uncompyle6.scanners.scanner26 as scan
 
 # bytecode verification, verify(), uses JUMP_OPs from here
 from xdis.opcodes import opcode_25
-JUMP_OPs = opcode_25.JUMP_OPs
+JUMP_OPS = opcode_25.JUMP_OPS
 
 # We base this off of 2.6 instead of the other way around
 # because we cleaned things up this way.

uncompyle6/scanners/scanner26.py

@@ -19,7 +19,7 @@ from uncompyle6.scanner import L65536
 
 # bytecode verification, verify(), uses JUMP_OPs from here
 from xdis.opcodes import opcode_26
-JUMP_OPs = opcode_26.JUMP_OPs
+JUMP_OPS = opcode_26.JUMP_OPS
 
 class Scanner26(scan.Scanner2):
     def __init__(self, show_asm=False):
@@ -87,7 +87,9 @@ class Scanner26(scan.Scanner2):
         cause specific rules for the specific number of arguments they take.
         """
 
-        show_asm = self.show_asm if not show_asm else show_asm
+        if not show_asm:
+            show_asm = self.show_asm
+
         # show_asm = 'after'
         if show_asm in ('both', 'before'):
             from xdis.bytecode import Bytecode
@@ -215,8 +217,8 @@ class Scanner26(scan.Scanner2):
                         # FIXME: this is a hack to catch stuff like:
                         #   if x: continue
                         # the "continue" is not on a new line.
-                        if len(tokens) and tokens[-1].type == 'JUMP_BACK':
-                            tokens[-1].type = intern('CONTINUE')
+                        if len(tokens) and tokens[-1].kind == 'JUMP_BACK':
+                            tokens[-1].kind = intern('CONTINUE')
 
                 elif op in self.opc.JABS_OPS:
                     pattr = repr(oparg)
@@ -256,18 +258,18 @@ class Scanner26(scan.Scanner2):
                         and self.code[offset+3] not in (self.opc.END_FINALLY,
                                                           self.opc.POP_BLOCK)):
                         if ((offset in self.linestartoffsets and
-                            tokens[-1].type == 'JUMP_BACK')
+                            tokens[-1].kind == 'JUMP_BACK')
                             or offset not in self.not_continue):
                             op_name = 'CONTINUE'
                     else:
                         # FIXME: this is a hack to catch stuff like:
                         #   if x: continue
                         # the "continue" is not on a new line.
-                        if tokens[-1].type == 'JUMP_BACK':
+                        if tokens[-1].kind == 'JUMP_BACK':
                             # We need 'intern' since we have
                             # already have processed the previous
                             # token.
-                            tokens[-1].type = intern('CONTINUE')
+                            tokens[-1].kind = intern('CONTINUE')
 
             elif op == self.opc.LOAD_GLOBAL:
                 if offset in self.load_asserts:

uncompyle6/scanners/scanner27.py

@@ -7,8 +7,6 @@ grammar parsing.
 """
 
 
-from __future__ import print_function
-
 from uncompyle6.scanners.scanner2 import Scanner2
 
 from uncompyle6 import PYTHON3
@@ -18,7 +16,7 @@ if PYTHON3:
 
 # bytecode verification, verify(), uses JUMP_OPs from here
 from xdis.opcodes import opcode_27
-JUMP_OPs = opcode_27.JUMP_OPs
+JUMP_OPS = opcode_27.JUMP_OPs
 
 class Scanner27(Scanner2):
     def __init__(self, show_asm=False, is_pypy=False):
@@ -94,9 +92,9 @@ class Scanner27(Scanner2):
             # the "continue" is not on a new line.
             n = len(tokens)
             if (n > 2 and
-                tokens[-1].type == 'JUMP_BACK' and
+                tokens[-1].kind == 'JUMP_BACK' and
                 self.code[offset+3] == self.opc.END_FINALLY):
-                tokens[-1].type = intern('CONTINUE')
+                tokens[-1].kind = intern('CONTINUE')
 
     pass
 

uncompyle6/scanners/scanner3.py

@@ -20,9 +20,13 @@ For example:
 Finally we save token information.
 """
 
-from __future__ import print_function
+from uncompyle6 import PYTHON_VERSION
+
+if PYTHON_VERSION < 2.6:
+    from xdis.namedtuple24 import namedtuple
+else:
+    from collections import namedtuple
 
-from collections import namedtuple
 from array import array
 
 from uncompyle6.scanner import Scanner
@@ -162,19 +166,25 @@ class Scanner3(Scanner):
         cause specific rules for the specific number of arguments they take.
         """
 
-        show_asm = self.show_asm if not show_asm else show_asm
-        # show_asm = 'both'
+        if not show_asm:
+            show_asm = self.show_asm
+
+        # show_asm = 'after'
         if show_asm in ('both', 'before'):
             bytecode = Bytecode(co, self.opc)
             for instr in bytecode.get_instructions(co):
                 print(instr._disassemble())
 
-        # Container for tokens
+        # list of tokens/instructions
         tokens = []
 
+        # "customize" is a dict whose keys are nonterminals
+        # and the value is the argument stack entries for that
+        # nonterminal. The count is a little hoaky. It is mostly
+        # not used, but sometimes it is.
         customize = {}
         if self.is_pypy:
-            customize['PyPy'] = 1
+            customize['PyPy'] = 0
 
         self.code = array('B', co.co_code)
         self.build_lines_data(co)
@@ -330,7 +340,7 @@ class Scanner3(Scanner):
                     attr = (pos_args, name_pair_args, annotate_args)
                 tokens.append(
                     Token(
-                        type_ = opname,
+                        opname = opname,
                         attr = attr,
                         pattr = pattr,
                         offset = inst.offset,
@@ -390,12 +400,12 @@ class Scanner3(Scanner):
                         # the "continue" is not on a new line.
                         # There are other situations where we don't catch
                         # CONTINUE as well.
-                        if tokens[-1].type == 'JUMP_BACK' and tokens[-1].attr <= argval:
-                            if tokens[-2].type == 'BREAK_LOOP':
+                        if tokens[-1].kind == 'JUMP_BACK' and tokens[-1].attr <= argval:
+                            if tokens[-2].kind == 'BREAK_LOOP':
                                 del tokens[-1]
                             else:
                                 # intern is used because we are changing the *previous* token
-                                tokens[-1].type = intern('CONTINUE')
+                                tokens[-1].kind = intern('CONTINUE')
                     if last_op_was_break and opname == 'CONTINUE':
                         last_op_was_break = False
                         continue
@@ -408,7 +418,7 @@ class Scanner3(Scanner):
             last_op_was_break = opname == 'BREAK_LOOP'
             tokens.append(
                 Token(
-                    type_ = opname,
+                    opname = opname,
                     attr = argval,
                     pattr = pattr,
                     offset = inst.offset,
@@ -775,6 +785,10 @@ class Scanner3(Scanner):
 
             if ((code[prev_op[target]] in self.pop_jump_if_pop) and
                 (target > offset) and prev_op[target] != offset):
+                # FIXME: this is not accurate The commented out below
+                # is what it should be. However grammar rules right now
+                # assume the incorrect offsets.
+                # self.fixed_jumps[offset] = target
                 self.fixed_jumps[offset] = prev_op[target]
                 self.structs.append({'type': 'and/or',
                                      'start': start,
@@ -943,7 +957,7 @@ class Scanner3(Scanner):
                                 return
                             pass
                     pass
-                if code[pre_rtarget] == self.opc.RETURN_VALUE and self.version < 3.5:
+                if code[pre_rtarget] == self.opc.RETURN_VALUE:
                     self.return_end_ifs.add(pre_rtarget)
                 else:
                     self.fixed_jumps[offset] = rtarget
@@ -963,7 +977,7 @@ class Scanner3(Scanner):
             if target > next_offset:
                 next_op = code[next_offset]
                 if (self.opc.JUMP_ABSOLUTE == next_op and
-                    END_FINALLY != code[xdis.next_offset(next_op, self.opc, next_offset)]):
+                    self.opc.END_FINALLY != code[xdis.next_offset(next_op, self.opc, next_offset)]):
                     self.fixed_jumps[next_offset] = target
                     self.except_targets[target] = next_offset
 
@@ -986,7 +1000,8 @@ class Scanner3(Scanner):
             # misclassified as RETURN_END_IF. Handle that here.
             # In RETURN_VALUE, JUMP_ABSOLUTE, RETURN_VALUE is never RETURN_END_IF
             if op == self.opc.RETURN_VALUE:
-                if (offset+1 < len(code) and code[offset+1] == self.opc.JUMP_ABSOLUTE and
+                next_offset = xdis.next_offset(op, self.opc, offset)
+                if (next_offset < len(code) and code[next_offset] == self.opc.JUMP_ABSOLUTE and
                     offset in self.return_end_ifs):
                     self.return_end_ifs.remove(offset)
                     pass

uncompyle6/scanners/scanner30.py

@@ -6,13 +6,11 @@ This sets up opcodes Python's 3.0 and calls a generalized
 scanner routine for Python 3.
 """
 
-from __future__ import print_function
-
 # bytecode verification, verify(), uses JUMP_OPs from here
 from xdis.opcodes import opcode_30 as opc
 from xdis.bytecode import op_size
 
-JUMP_OPs = map(lambda op: opc.opname[op], opc.hasjrel + opc.hasjabs)
+JUMP_OPS = opc.JUMP_OPS
 
 JUMP_TF = frozenset([opc.JUMP_IF_FALSE, opc.JUMP_IF_TRUE])
 

uncompyle6/scanners/scanner31.py

@@ -1,4 +1,4 @@
-#  Copyright (c) 2016 by Rocky Bernstein
+#  Copyright (c) 2016-2017 by Rocky Bernstein
 """
 Python 3.1 bytecode scanner/deparser
 
@@ -6,11 +6,9 @@ This sets up opcodes Python's 3.1 and calls a generalized
 scanner routine for Python 3.
 """
 
-from __future__ import print_function
-
 # bytecode verification, verify(), uses JUMP_OPs from here
 from xdis.opcodes import opcode_31 as opc
-JUMP_OPs = map(lambda op: opc.opname[op], opc.hasjrel + opc.hasjabs)
+JUMP_OPS = opc.JUMP_OPS
 
 from uncompyle6.scanners.scanner3 import Scanner3
 class Scanner31(Scanner3):

uncompyle6/scanners/scanner32.py

@@ -9,11 +9,9 @@ This sets up opcodes Python's 3.2 and calls a generalized
 scanner routine for Python 3.
 """
 
-from __future__ import print_function
-
 # bytecode verification, verify(), uses JUMP_OPs from here
 from xdis.opcodes import opcode_32 as opc
-JUMP_OPs = map(lambda op: opc.opname[op], opc.hasjrel + opc.hasjabs)
+JUMP_OPS = opc.JUMP_OPS
 
 from uncompyle6.scanners.scanner3 import Scanner3
 class Scanner32(Scanner3):

uncompyle6/scanners/scanner33.py

@@ -1,4 +1,4 @@
-#  Copyright (c) 2015-2016 by Rocky Bernstein
+#  Copyright (c) 2015-2017 by Rocky Bernstein
 """
 Python 3.3 bytecode scanner/deparser
 
@@ -6,11 +6,9 @@ This sets up opcodes Python's 3.3 and calls a generalized
 scanner routine for Python 3.
 """
 
-from __future__ import print_function
-
 # bytecode verification, verify(), uses JUMP_OPs from here
 from xdis.opcodes import opcode_33 as opc
-JUMP_OPs = map(lambda op: opc.opname[op], opc.hasjrel + opc.hasjabs)
+JUMP_OPS = opc.JUMP_OPS
 
 from uncompyle6.scanners.scanner3 import Scanner3
 class Scanner33(Scanner3):

uncompyle6/scanners/scanner34.py

@@ -9,12 +9,10 @@ This sets up opcodes Python's 3.4 and calls a generalized
 scanner routine for Python 3.
 """
 
-from __future__ import print_function
-
 from xdis.opcodes import opcode_34 as opc
 
 # bytecode verification, verify(), uses JUMP_OPs from here
-JUMP_OPs = map(lambda op: opc.opname[op], opc.hasjrel + opc.hasjabs)
+JUMP_OPS = opc.JUMP_OPS
 
 
 from uncompyle6.scanners.scanner3 import Scanner3

uncompyle6/scanners/scanner35.py

@@ -9,13 +9,11 @@ This sets up opcodes Python's 3.5 and calls a generalized
 scanner routine for Python 3.
 """
 
-from __future__ import print_function
-
 from uncompyle6.scanners.scanner3 import Scanner3
 
 # bytecode verification, verify(), uses JUMP_OPs from here
 from xdis.opcodes import opcode_35 as opc
-JUMP_OPs = map(lambda op: opc.opname[op], opc.hasjrel + opc.hasjabs)
+JUMP_OPS = opc.JUMP_OPS
 
 class Scanner35(Scanner3):
 

uncompyle6/scanners/scanner36.py

@@ -9,13 +9,11 @@ This sets up opcodes Python's 3.6 and calls a generalized
 scanner routine for Python 3.
 """
 
-from __future__ import print_function
-
 from uncompyle6.scanners.scanner3 import Scanner3
 
-# bytecode verification, verify(), uses JUMP_OPs from here
+# bytecode verification, verify(), uses JUMP_OPS from here
 from xdis.opcodes import opcode_36 as opc
-JUMP_OPs = map(lambda op: opc.opname[op], opc.hasjrel + opc.hasjabs)
+JUMP_OPS = opc.JUMP_OPS
 
 class Scanner36(Scanner3):
 
@@ -29,14 +27,14 @@ class Scanner36(Scanner3):
             # The lowest bit of flags indicates whether the
             # var-keyword argument is placed at the top of the stack
             if t.op == self.opc.CALL_FUNCTION_EX and t.attr & 1:
-                t.type = 'CALL_FUNCTION_EX_KW'
+                t.kind = 'CALL_FUNCTION_EX_KW'
                 pass
             elif t.op == self.opc.CALL_FUNCTION_KW:
-                t.type = 'CALL_FUNCTION_KW_{t.attr}'.format(**locals())
+                t.kind = 'CALL_FUNCTION_KW_{t.attr}'.format(**locals())
             elif t.op == self.opc.BUILD_TUPLE_UNPACK_WITH_CALL:
-                t.type = 'BUILD_TUPLE_UNPACK_WITH_CALL_%d' % t.attr
+                t.kind = 'BUILD_TUPLE_UNPACK_WITH_CALL_%d' % t.attr
             elif t.op == self.opc.BUILD_MAP_UNPACK_WITH_CALL:
-                t.type = 'BUILD_MAP_UNPACK_WITH_CALL_%d' % t.attr
+                t.kind = 'BUILD_MAP_UNPACK_WITH_CALL_%d' % t.attr
             pass
         return tokens, customize
 

uncompyle6/scanners/scanner37.py

@@ -0,0 +1,36 @@
+#  Copyright (c) 2016-2017 by Rocky Bernstein
+"""
+Python 3.7 bytecode decompiler scanner
+
+Does some additional massaging of xdis-disassembled instructions to
+make things easier for decompilation.
+
+This sets up opcodes Python's 3.6 and calls a generalized
+scanner routine for Python 3.
+"""
+
+from uncompyle6.scanners.scanner3 import Scanner3
+
+# bytecode verification, verify(), uses JUMP_OPs from here
+from xdis.opcodes import opcode_36 as opc
+JUMP_OPs = opc.JUMP_OPS
+
+class Scanner37(Scanner3):
+
+    def __init__(self, show_asm=None):
+        Scanner3.__init__(self, 3.7, show_asm)
+        return
+    pass
+
+if __name__ == "__main__":
+    from uncompyle6 import PYTHON_VERSION
+    if PYTHON_VERSION == 3.7:
+        import inspect
+        co = inspect.currentframe().f_code
+        tokens, customize = Scanner37().ingest(co)
+        for t in tokens:
+            print(t.format())
+        pass
+    else:
+        print("Need to be Python 3.7 to demo; I am %s." %
+              PYTHON_VERSION)

uncompyle6/scanners/tok.py

@@ -1,4 +1,4 @@
-#  Copyright (c) 2016 by Rocky Bernstein
+#  Copyright (c) 2016-2017 by Rocky Bernstein
 #  Copyright (c) 2000-2002 by hartmut Goebel <h.goebel@crazy-compilers.com>
 #  Copyright (c) 1999 John Aycock
 
@@ -16,13 +16,12 @@ class Token:
     the contents of one line as output by dis.dis().
     """
     # FIXME: match Python 3.4's terms:
-    #    type_ should be opname
     #    linestart = starts_line
     #    attr = argval
     #    pattr = argrepr
-    def __init__(self, type_, attr=None, pattr=None, offset=-1,
+    def __init__(self, opname, attr=None, pattr=None, offset=-1,
                  linestart=None, op=None, has_arg=None, opc=None):
-        self.type = intern(type_)
+        self.kind = intern(opname)
         self.op = op
         self.has_arg = has_arg
         self.attr = attr
@@ -37,31 +36,37 @@ class Token:
     def __eq__(self, o):
         """ '==', but it's okay if offsets and linestarts are different"""
         if isinstance(o, Token):
-            # Both are tokens: compare type and attr
+            # Both are tokens: compare kind and attr
             # It's okay if offsets are different
-            return (self.type == o.type) and (self.pattr == o.pattr)
+            return (self.kind == o.kind) and (self.pattr == o.pattr)
         else:
-            return self.type == o
+            return self.kind == o
 
     def __repr__(self):
-        return str(self.type)
+        return str(self.kind)
 
     # def __str__(self):
     #     pattr = self.pattr if self.pattr is not None else ''
     #     prefix = '\n%3d   ' % self.linestart if self.linestart else (' ' * 6)
     #     return (prefix +
-    #             ('%9s  %-18s %r' % (self.offset, self.type, pattr)))
+    #             ('%9s  %-18s %r' % (self.offset, self.kind, pattr)))
 
     def __str__(self):
         return self.format(line_prefix='')
 
     def format(self, line_prefix=''):
-        prefix = ('\n%s%4d  ' % (line_prefix, self.linestart)
-                  if self.linestart else (' ' * (6 + len(line_prefix))))
-        offset_opname = '%6s  %-17s' % (self.offset, self.type)
+        if self.linestart:
+            prefix = '\n%s%4d  ' % (line_prefix, self.linestart)
+        else:
+            prefix = ' ' * (6 + len(line_prefix))
+        offset_opname = '%6s  %-17s' % (self.offset, self.kind)
         if not self.has_arg:
             return "%s%s" % (prefix, offset_opname)
-        argstr = "%6d " % self.attr if isinstance(self.attr, int) else (' '*7)
+
+        if isinstance(self.attr, int):
+            argstr = "%6d " % self.attr
+        else:
+            argstr = ' '*7
         if self.pattr:
             pattr = self.pattr
             if self.opc:
@@ -78,14 +83,14 @@ class Token:
                         pattr = self.opc.cmp_op[self.attr]
                 # And so on. See xdis/bytecode.py get_instructions_bytes
                 pass
-        elif re.search('_\d+$', self.type):
+        elif re.search('_\d+$', self.kind):
             return "%s%s%s" % (prefix, offset_opname,  argstr)
         else:
             pattr = ''
         return "%s%s%s %r" % (prefix, offset_opname,  argstr, pattr)
 
     def __hash__(self):
-        return hash(self.type)
+        return hash(self.kind)
 
     def __getitem__(self, i):
         raise IndexError

uncompyle6/semantics/check_ast.py

@@ -9,16 +9,16 @@ before reduction and don't reduce when there is a problem.
 """
 
 def checker(ast, in_loop, errors):
-    in_loop = in_loop or ast.type in ('while1stmt', 'whileTruestmt',
+    in_loop = in_loop or ast.kind in ('while1stmt', 'whileTruestmt',
                                       'whilestmt', 'whileelsestmt', 'while1elsestmt',
                                       'for_block')
-    if ast.type in ('augassign1', 'augassign2') and ast[0][0] == 'and':
+    if ast.kind in ('augassign1', 'augassign2') and ast[0][0] == 'and':
         text = str(ast)
         error_text = '\n# improper augmented assigment (e.g. +=, *=, ...):\n#\t' + '\n# '.join(text.split("\n")) + '\n'
         errors.append(error_text)
 
     for node in ast:
-        if not in_loop and node.type in ('continue_stmt', 'break_stmt'):
+        if not in_loop and node.kind in ('continue_stmt', 'break_stmt'):
             text = str(node)
             error_text = '\n# not in loop:\n#\t' + '\n# '.join(text.split("\n"))
             errors.append(error_text)

uncompyle6/semantics/consts.py

@@ -1,5 +1,5 @@
 #  Copyright (c) 2017 by Rocky Bernstein
-"""Constants used in pysource.py"""
+"""Constants and initial table values used in pysource.py and fragments.py"""
 
 import re, sys
 from uncompyle6.parsers.astnode import AST
@@ -57,9 +57,7 @@ INDENT_PER_LEVEL = ' ' # additional intent per pretty-print level
 
 TABLE_R = {
     'STORE_ATTR':	( '%c.%[1]{pattr}', 0),
-#   'STORE_SUBSCR':	( '%c[%c]', 0, 1 ),
     'DELETE_ATTR':	( '%|del %c.%[-1]{pattr}\n', 0 ),
-#   'EXEC_STMT':	( '%|exec %c in %[1]C\n', 0, (0,maxint,', ') ),
 }
 
 TABLE_R0 = {
@@ -67,8 +65,9 @@ TABLE_R0 = {
 #    'BUILD_TUPLE':	( '(%C)',      (0,-1,', ') ),
 #    'CALL_FUNCTION':	( '%c(%P)', 0, (1,-1,', ') ),
 }
+
 TABLE_DIRECT = {
-    'BINARY_ADD':	( '+' ,),
+    'BINARY_ADD':	    ( '+' ,),
     'BINARY_SUBTRACT':	( '-' ,),
     'BINARY_MULTIPLY':	( '*' ,),
     'BINARY_DIVIDE':	( '/' ,),
@@ -76,13 +75,13 @@ TABLE_DIRECT = {
     'BINARY_TRUE_DIVIDE':	( '/' ,),   # Not in <= 2.1
     'BINARY_FLOOR_DIVIDE':	( '//' ,),
     'BINARY_MODULO':	( '%%',),
-    'BINARY_POWER':	( '**',),
+    'BINARY_POWER':	    ( '**',),
     'BINARY_LSHIFT':	( '<<',),
     'BINARY_RSHIFT':	( '>>',),
-    'BINARY_AND':	( '&' ,),
-    'BINARY_OR':	( '|' ,),
-    'BINARY_XOR':	( '^' ,),
-    'INPLACE_ADD':	( '+=' ,),
+    'BINARY_AND':	    ( '&' ,),
+    'BINARY_OR':	    ( '|' ,),
+    'BINARY_XOR':	    ( '^' ,),
+    'INPLACE_ADD':	    ( '+=' ,),
     'INPLACE_SUBTRACT':	( '-=' ,),
     'INPLACE_MULTIPLY':	( '*=' ,),
     'INPLACE_MATRIX_MULTIPLY':	( '@=' ,),
@@ -93,125 +92,126 @@ TABLE_DIRECT = {
     'INPLACE_POWER':	( '**=',),
     'INPLACE_LSHIFT':	( '<<=',),
     'INPLACE_RSHIFT':	( '>>=',),
-    'INPLACE_AND':	( '&=' ,),
-    'INPLACE_OR':	( '|=' ,),
-    'INPLACE_XOR':	( '^=' ,),
-    'binary_expr':	( '%c %c %c', 0, -1, 1 ),
+    'INPLACE_AND':	    ( '&=' ,),
+    'INPLACE_OR':	    ( '|=' ,),
+    'INPLACE_XOR':	    ( '^=' ,),
+    'binary_expr':	    ( '%c %c %c', 0, -1, 1 ),
 
     'UNARY_POSITIVE':	( '+',),
     'UNARY_NEGATIVE':	( '-',),
-    'UNARY_INVERT':	( '~%c'),
-    'unary_expr':   ( '%c%c', 1, 0),
+    'UNARY_INVERT':	    ( '~'),
+    'unary_expr':       ( '%c%c', 1, 0),
 
-    'unary_not':	( 'not %c', 0 ),
+    'unary_not':	    ( 'not %c', 0 ),
     'unary_convert':	( '`%c`', 0 ),
-    'get_iter':	( 'iter(%c)', 0 ),
-    'slice0':		( '%c[:]', 0 ),
-    'slice1':		( '%c[%p:]', 0, (1, 100) ),
-    'slice2':		( '%c[:%p]', 0, (1, 100) ),
-    'slice3':		( '%c[%p:%p]', 0, (1, 100), (2, 100) ),
+    'get_iter':	        ( 'iter(%c)', 0 ),
+    'slice0':		    ( '%c[:]', 0 ),
+    'slice1':		    ( '%c[%p:]', 0, (1, 100) ),
+    'slice2':		    ( '%c[:%p]', 0, (1, 100) ),
+    'slice3':		    ( '%c[%p:%p]', 0, (1, 100), (2, 100) ),
 
-    'IMPORT_FROM':	( '%{pattr}', ),
-    'load_attr':	( '%c.%[1]{pattr}', 0),
-    'LOAD_FAST':	( '%{pattr}', ),
-    'LOAD_NAME':	( '%{pattr}', ),
+    'IMPORT_FROM':      ( '%{pattr}', ),
+    'load_attr':	    ( '%c.%[1]{pattr}', 0),
+    'LOAD_FAST':	    ( '%{pattr}', ),
+    'LOAD_NAME':	    ( '%{pattr}', ),
     'LOAD_CLASSNAME':	( '%{pattr}', ),
-    'LOAD_GLOBAL':	( '%{pattr}', ),
-    'LOAD_DEREF':	( '%{pattr}', ),
-    'LOAD_LOCALS':	( 'locals()', ),
-    'LOAD_ASSERT':  ( '%{pattr}', ),
+    'LOAD_GLOBAL':	    ( '%{pattr}', ),
+    'LOAD_DEREF':	    ( '%{pattr}', ),
+    'LOAD_LOCALS':	    ( 'locals()', ),
+    'LOAD_ASSERT':      ( '%{pattr}', ),
 #   'LOAD_CONST':	( '%{pattr}', ),	# handled by n_LOAD_CONST
-    'DELETE_FAST':	( '%|del %{pattr}\n', ),
-    'DELETE_NAME':	( '%|del %{pattr}\n', ),
+    'DELETE_FAST':	    ( '%|del %{pattr}\n', ),
+    'DELETE_NAME':	    ( '%|del %{pattr}\n', ),
     'DELETE_GLOBAL':	( '%|del %{pattr}\n', ),
     'delete_subscr':	( '%|del %c[%c]\n', 0, 1,),
     'binary_subscr':	( '%c[%p]', 0, (1, 100)),
     'binary_subscr2':	( '%c[%p]', 0, (1, 100)),
-    'store_subscr':	( '%c[%c]', 0, 1),
-    'STORE_FAST':	( '%{pattr}', ),
-    'STORE_NAME':	( '%{pattr}', ),
-    'STORE_GLOBAL':	( '%{pattr}', ),
-    'STORE_DEREF':	( '%{pattr}', ),
-    'unpack':		( '%C%,', (1, maxint, ', ') ),
+    'store_subscr':	    ( '%c[%c]', 0, 1),
+    'STORE_FAST':	    ( '%{pattr}', ),
+    'STORE_NAME':	    ( '%{pattr}', ),
+    'STORE_GLOBAL':	    ( '%{pattr}', ),
+    'STORE_DEREF':	    ( '%{pattr}', ),
+    'unpack':		    ( '%C%,', (1, maxint, ', ') ),
 
     # This nonterminal we create on the fly in semantic routines
     'unpack_w_parens':	( '(%C%,)', (1, maxint, ', ') ),
 
-    'unpack_list':	( '[%C]', (1, maxint, ', ') ),
-    'build_tuple2':	( '%P', (0, -1, ', ', 100) ),
+    'unpack_list':	    ( '[%C]', (1, maxint, ', ') ),
+    'build_tuple2':	    ( '%P', (0, -1, ', ', 100) ),
 
     # 'list_compr':	( '[ %c ]', -2),	# handled by n_list_compr
-    'list_iter':	( '%c', 0),
-    'list_for':		( ' for %c in %c%c', 2, 0, 3 ),
-    'list_if':		( ' if %c%c', 0, 2 ),
+    'list_iter':	    ( '%c', 0 ),
+    'list_for':		    ( ' for %c in %c%c', 2, 0, 3 ),
+    'list_if':		    ( ' if %c%c', 0, 2 ),
     'list_if_not':		( ' if not %p%c', (0, 22), 2 ),
-    'lc_body':		( '', ),	# ignore when recusing
+    'lc_body':		    ( '', ),	# ignore when recusing
 
-    'comp_iter':	( '%c', 0),
-    'comp_if':		( ' if %c%c', 0, 2 ),
-    'comp_ifnot':	( ' if not %p%c', (0, 22), 2 ),
-    'comp_body':	( '', ),	# ignore when recusing
+    'comp_iter':	    ( '%c', 0 ),
+    'comp_if':		    ( ' if %c%c', 0, 2 ),
+    'comp_ifnot':	    ( ' if not %p%c', (0, 22), 2 ),
+    'comp_body':	    ( '', ),	# ignore when recusing
     'set_comp_body':    ( '%c', 0 ),
     'gen_comp_body':    ( '%c', 0 ),
     'dict_comp_body':   ( '%c:%c', 1, 0 ),
 
-    'assign':		( '%|%c = %p\n', -1, (0, 200) ),
+    'assign':		    ( '%|%c = %p\n', -1, (0, 200) ),
 
     # The 2nd parameter should have a = suffix.
     # There is a rule with a 4th parameter "designator"
     # which we don't use here.
-    'augassign1':	( '%|%c %c %c\n', 0, 2, 1),
+    'augassign1':	    ( '%|%c %c %c\n', 0, 2, 1),
 
-    'augassign2':	( '%|%c.%[2]{pattr} %c %c\n', 0, -3, -4),
-    'designList':	( '%c = %c', 0, -1 ),
+    'augassign2':	    ( '%|%c.%[2]{pattr} %c %c\n', 0, -3, -4 ),
+    'designList':	    ( '%c = %c', 0, -1 ),
     'and':          	( '%c and %c', 0, 2 ),
     'ret_and':        	( '%c and %c', 0, 2 ),
     'and2':          	( '%c', 3 ),
     'or':           	( '%c or %c', 0, 2 ),
-    'ret_or':           	( '%c or %c', 0, 2 ),
-    'conditional':  ( '%p if %p else %p', (2, 27), (0, 27), (4, 27)),
-    'conditionalTrue':  ( '%p if 1 else %p', (0, 27), (2, 27)),
-    'ret_cond':     ( '%p if %p else %p', (2, 27), (0, 27), (-1, 27)),
-    'conditionalnot':  ( '%p if not %p else %p', (2, 27), (0, 22), (4, 27)),
-    'ret_cond_not':  ( '%p if not %p else %p', (2, 27), (0, 22), (-1, 27)),
-    'conditional_lambda':  ( '(%c if %c else %c)', 2, 0, 3),
-    'return_lambda':    ('%c', 0),
-    'compare':		( '%p %[-1]{pattr.replace("-", " ")} %p', (0, 19), (1, 19) ),
-    'cmp_list':		( '%p %p', (0, 29), (1, 30)),
-    'cmp_list1':	( '%[3]{pattr} %p %p', (0, 19), (-2, 19)),
-    'cmp_list2':	( '%[1]{pattr} %p', (0, 19)),
+    'ret_or':           ( '%c or %c', 0, 2 ),
+    'conditional':      ( '%p if %p else %p', (2, 27), (0, 27), (4, 27) ),
+    'conditionalTrue':  ( '%p if 1 else %p', (0, 27), (2, 27) ),
+    'ret_cond':         ( '%p if %p else %p', (2, 27), (0, 27), (-1, 27) ),
+    'conditionalnot':   ( '%p if not %p else %p', (2, 27), (0, 22), (4, 27) ),
+    'ret_cond_not':     ( '%p if not %p else %p', (2, 27), (0, 22), (-1, 27) ),
+    'conditional_lambda':  ( '%c if %c else %c', 2, 0, 4),
+
+    'compare':		    ( '%p %[-1]{pattr.replace("-", " ")} %p', (0, 19), (1, 19) ),
+    'cmp_list':		    ( '%p %p', (0, 29), (1, 30)),
+    'cmp_list1':	    ( '%[3]{pattr} %p %p', (0, 19), (-2, 19)),
+    'cmp_list2':	    ( '%[1]{pattr} %p', (0, 19)),
 #   'classdef': 	(), # handled by n_classdef()
-    'funcdef':  	( '\n\n%|def %c\n', -2), # -2 to handle closures
+    'funcdef':  	    ( '\n\n%|def %c\n', -2), # -2 to handle closures
     'funcdefdeco':  	( '\n\n%c', 0),
-    'mkfuncdeco':  	( '%|@%c\n%c', 0, 1),
+    'mkfuncdeco':  	    ( '%|@%c\n%c', 0, 1),
     'mkfuncdeco0':  	( '%|def %c\n', 0),
     'classdefdeco':  	( '\n\n%c', 0),
     'classdefdeco1':  	( '%|@%c\n%c', 0, 1),
-    'kwarg':    	( '%[0]{pattr}=%c', 1),
-    'kwargs':    	( '%D', (0, maxint, ', ') ),
+    'kwarg':    	    ( '%[0]{pattr}=%c', 1),
+    'kwargs':    	    ( '%D', (0, maxint, ', ') ),
 
-    'assert_expr_or': ( '%c or %c', 0, 2 ),
-    'assert_expr_and':    ( '%c and %c', 0, 2 ),
-    'print_items_stmt': ( '%|print %c%c,\n', 0, 2),  # Python 2 only
-    'print_items_nl_stmt': ( '%|print %c%c\n', 0, 2),
-    'print_item':  ( ', %c', 0),
-    'print_nl':	( '%|print\n', ),
-    'print_to':		( '%|print >> %c, %c,\n', 0, 1 ),
-    'print_to_nl':	( '%|print >> %c, %c\n', 0, 1 ),
-    'print_nl_to':	( '%|print >> %c\n', 0 ),
+    'assert_expr_or':   ( '%c or %c', 0, 2 ),
+    'assert_expr_and':  ( '%c and %c', 0, 2 ),
+    'print_items_stmt': ( '%|print %c%c,\n', 0, 2 ),  # Python 2 only
+    'print_items_nl_stmt': ( '%|print %c%c\n', 0, 2 ),
+    'print_item':       ( ', %c', 0),
+    'print_nl':	        ( '%|print\n', ),
+    'print_to':		    ( '%|print >> %c, %c,\n', 0, 1 ),
+    'print_to_nl':	    ( '%|print >> %c, %c\n', 0, 1 ),
+    'print_nl_to':	    ( '%|print >> %c\n', 0 ),
     'print_to_items':	( '%C', (0, 2, ', ') ),
 
-    'call_stmt':	( '%|%p\n', (0, 200)),
-    'break_stmt':	( '%|break\n', ),
+    'call_stmt':	    ( '%|%p\n', (0, 200)),
+    'break_stmt':	    ( '%|break\n', ),
     'continue_stmt':	( '%|continue\n', ),
 
-    'raise_stmt0':	( '%|raise\n', ),
-    'raise_stmt1':	( '%|raise %c\n', 0),
-    'raise_stmt3':	( '%|raise %c, %c, %c\n', 0, 1, 2),
+    'raise_stmt0':	    ( '%|raise\n', ),
+    'raise_stmt1':	    ( '%|raise %c\n', 0),
+    'raise_stmt3':	    ( '%|raise %c, %c, %c\n', 0, 1, 2),
 #    'yield':	( 'yield %c', 0),
 #    'return_stmt':	( '%|return %c\n', 0),
+    'return_if_stmt':	( 'return %c\n', 0),
 
-    'ifstmt':		( '%|if %c:\n%+%c%-', 0, 1 ),
+    'ifstmt':		    ( '%|if %c:\n%+%c%-', 0, 1 ),
     'iflaststmt':		( '%|if %c:\n%+%c%-', 0, 1 ),
     'iflaststmtl':		( '%|if %c:\n%+%c%-', 0, 1 ),
     'testtrue':         ( 'not %p', (0, 22) ),
@@ -229,37 +229,37 @@ TABLE_DIRECT = {
     'elifelsestmtr2':	( '%|elif %c:\n%+%c%-%|else:\n%+%c%-\n\n', 0, 1, 3 ), # has COME_FROM
 
     'whileTruestmt':	( '%|while True:\n%+%c%-\n\n', 1 ),
-    'whilestmt':	( '%|while %c:\n%+%c%-\n\n', 1, 2 ),
-    'while1stmt':	( '%|while 1:\n%+%c%-\n\n', 1 ),
-    'while1elsestmt':  ( '%|while 1:\n%+%c%-%|else:\n%+%c%-\n\n', 1, -2 ),
+    'whilestmt':	    ( '%|while %c:\n%+%c%-\n\n', 1, 2 ),
+    'while1stmt':	    ( '%|while 1:\n%+%c%-\n\n', 1 ),
+    'while1elsestmt':   ( '%|while 1:\n%+%c%-%|else:\n%+%c%-\n\n', 1, -2 ),
     'whileelsestmt':	( '%|while %c:\n%+%c%-%|else:\n%+%c%-\n\n', 1, 2, -2 ),
     'whileelselaststmt':	( '%|while %c:\n%+%c%-%|else:\n%+%c%-', 1, 2, -2 ),
-    'forstmt':		( '%|for %c in %c:\n%+%c%-\n\n', 3, 1, 4 ),
-    'forelsestmt':	(
-        '%|for %c in %c:\n%+%c%-%|else:\n%+%c%-\n\n', 3, 1, 4, -2),
+    'forstmt':		    ( '%|for %c in %c:\n%+%c%-\n\n', 3, 1, 4 ),
+    'forelsestmt':	    (
+        '%|for %c in %c:\n%+%c%-%|else:\n%+%c%-\n\n', 3, 1, 4, -2 ),
     'forelselaststmt':	(
-        '%|for %c in %c:\n%+%c%-%|else:\n%+%c%-', 3, 1, 4, -2),
+        '%|for %c in %c:\n%+%c%-%|else:\n%+%c%-', 3, 1, 4, -2 ),
     'forelselaststmtl':	(
-        '%|for %c in %c:\n%+%c%-%|else:\n%+%c%-\n\n', 3, 1, 4, -2),
-    'trystmt':		( '%|try:\n%+%c%-%c\n\n', 1, 3 ),
-    'tryelsestmt':	( '%|try:\n%+%c%-%c%|else:\n%+%c%-\n\n', 1, 3, 4 ),
-    'tryelsestmtc':	( '%|try:\n%+%c%-%c%|else:\n%+%c%-', 1, 3, 4 ),
-    'tryelsestmtl':	( '%|try:\n%+%c%-%c%|else:\n%+%c%-', 1, 3, 4 ),
-    'tf_trystmt':	( '%c%-%c%+', 1, 3 ),
+        '%|for %c in %c:\n%+%c%-%|else:\n%+%c%-\n\n', 3, 1, 4, -2 ),
+    'trystmt':		    ( '%|try:\n%+%c%-%c\n\n', 1, 3 ),
+    'tryelsestmt':	    ( '%|try:\n%+%c%-%c%|else:\n%+%c%-\n\n', 1, 3, 4 ),
+    'tryelsestmtc':	    ( '%|try:\n%+%c%-%c%|else:\n%+%c%-', 1, 3, 4 ),
+    'tryelsestmtl':	    ( '%|try:\n%+%c%-%c%|else:\n%+%c%-', 1, 3, 4 ),
+    'tf_trystmt':	    ( '%c%-%c%+', 1, 3 ),
     'tf_tryelsestmt':	( '%c%-%c%|else:\n%+%c', 1, 3, 4 ),
     'tryfinallystmt':	( '%|try:\n%+%c%-%|finally:\n%+%c%-\n\n', 1, 5 ),
     'except':           ( '%|except:\n%+%c%-', 3 ),
-    'except_cond1':	( '%|except %c:\n', 1 ),
+    'except_cond1':	    ( '%|except %c:\n', 1 ),
     'except_suite':     ( '%+%c%-%C', 0, (1, maxint, '') ),
     'except_suite_finalize':     ( '%+%c%-%C', 1, (3, maxint, '') ),
-    'passstmt':		( '%|pass\n', ),
-    'STORE_FAST':	( '%{pattr}', ),
-    'kv':		( '%c: %c', 3, 1 ),
-    'kv2':		( '%c: %c', 1, 2 ),
-    'mapexpr':		( '{%[1]C}', (0, maxint, ', ') ),
-    'importstmt': ( '%|import %c\n', 2),
-    'importfrom': ( '%|from %[2]{pattr} import %c\n', 3 ),
-    'importstar': ( '%|from %[2]{pattr} import *\n', ),
+    'passstmt':		    ( '%|pass\n', ),
+    'STORE_FAST':	    ( '%{pattr}', ),
+    'kv':		        ( '%c: %c', 3, 1 ),
+    'kv2':		        ( '%c: %c', 1, 2 ),
+    'mapexpr':		    ( '{%[1]C}', (0, maxint, ', ') ),
+    'importstmt':       ( '%|import %c\n', 2),
+    'importfrom':       ( '%|from %[2]{pattr} import %c\n', 3 ),
+    'importstar':       ( '%|from %[2]{pattr} import *\n', ),
 }
 
 
@@ -276,7 +276,7 @@ MAP = {
 }
 
 # Operator precidence
-# See https://docs.python.org/3/reference/expressions.html
+# See https://docs.python.org/2/reference/expressions.html
 # or https://docs.python.org/3/reference/expressions.html
 # for a list.
 PRECEDENCE = {
@@ -332,6 +332,7 @@ PRECEDENCE = {
     'ret_or':               26,
 
     'conditional':          28,
+    'conditional_lamdba':   28,
     'conditionalnot':       28,
     'ret_cond':             28,
     'ret_cond_not':         28,

uncompyle6/semantics/fragments.py

@@ -1,6 +1,4 @@
-#  Copyright (c) 2015, 2016 by Rocky Bernstein
-#  Copyright (c) 2005 by Dan Pascu <dan@windowmaker.org>
-#  Copyright (c) 2000-2002 by hartmut Goebel <h.goebel@crazy-compilers.com>
+#  Copyright (c) 2015-2017 by Rocky Bernstein
 #  Copyright (c) 1999 John Aycock
 
 """
@@ -8,8 +6,8 @@ Creates Python source code from an uncompyle6 abstract syntax tree,
 and indexes fragments which can be accessed by instruction offset
 address.
 
-See the comments in pysource for information on the abstract sytax tree
-and how semantic actions are written.
+See https://github.com/rocky/python-uncompyle6/wiki/Table-driven-semantic-actions.
+for a more complete explanation, nicely marked up and with examples.
 
 We add some format specifiers here not used in pysource
 
@@ -40,7 +38,8 @@ do it recursively which is where offsets are probably located.
 2. %b
 -----
 
-   %b associates the text from the previous start node up to what we have now
+   %b associates the text from the specified index to what we have now.
+      it takes an integer argument.
 
 For example in:
   'importmultiple':   ( '%|import%b %c%c\n', 0, 2, 3 ),
@@ -51,8 +50,6 @@ The node position 0 will be associated with "import".
 
 # FIXME: DRY code with pysource
 
-from __future__ import print_function
-
 import re, sys
 
 from xdis.code import iscode
@@ -79,7 +76,12 @@ from uncompyle6.semantics.consts import (
 from spark_parser import DEFAULT_DEBUG as PARSER_DEFAULT_DEBUG
 from spark_parser.ast import GenericASTTraversalPruningException
 
-from collections import namedtuple
+from uncompyle6 import PYTHON_VERSION
+if PYTHON_VERSION < 2.6:
+    from xdis.namedtuple24 import namedtuple
+else:
+    from collections import namedtuple
+
 NodeInfo = namedtuple("NodeInfo", "node start finish")
 ExtractInfo = namedtuple("ExtractInfo",
                          "lineNo lineStartOffset markerLine selectedLine selectedText nonterminal")
@@ -93,9 +95,8 @@ TABLE_DIRECT_FRAGMENT = {
     'importfrom':	( '%|from %[2]{pattr}%x import %c\n', (2, (0, 1)), 3),
     'importmultiple':   ( '%|import%b %c%c\n', 0, 2, 3 ),
     'list_for':	  	(' for %c%x in %c%c', 2, (2, (1, )), 0, 3 ),
-    'forstmt':	  	( '%|for%b %c%x in %c:\n%+%c%-\n\n', 0, 3, (3, (2, )), 1, 4 ),
     'forelsestmt': 	(
-        '%|for %c in %c%x:\n%+%c%-%|else:\n%+%c%-\n\n', 3, (3, (2,)), 1, 4, -2),
+        '%|for %c%x in %c:\n%+%c%-%|else:\n%+%c%-\n\n', 3, (3, (2,)), 1, 4, -2),
     'forelselaststmt':	(
         '%|for %c%x in %c:\n%+%c%-%|else:\n%+%c%-', 3, (3, (2,)), 1, 4, -2),
     'forelselaststmtl':	(
@@ -307,11 +308,11 @@ class FragmentsWalker(pysource.SourceWalker, object):
     def n_expr(self, node):
         start = len(self.f.getvalue())
         p = self.prec
-        if node[0].type.startswith('binary_expr'):
+        if node[0].kind.startswith('binary_expr'):
             n = node[0][-1][0]
         else:
             n = node[0]
-        self.prec = PRECEDENCE.get(n.type, -2)
+        self.prec = PRECEDENCE.get(n.kind, -2)
         if n == 'LOAD_CONST' and repr(n.pattr)[0] == '-':
             n.parent = node
             self.set_pos_info(n, start, len(self.f.getvalue()))
@@ -404,7 +405,7 @@ class FragmentsWalker(pysource.SourceWalker, object):
     def n_ifelsestmtr(self, node):
         if node[2] == 'COME_FROM':
             return_stmts_node = node[3]
-            node.type = 'ifelsestmtr2'
+            node.kind = 'ifelsestmtr2'
         else:
             return_stmts_node = node[2]
         if len(return_stmts_node) != 2:
@@ -421,10 +422,10 @@ class FragmentsWalker(pysource.SourceWalker, object):
         self.write(self.indent, 'if ')
         self.preorder(node[0])
         self.println(':')
-        self.indentMore()
+        self.indent_more()
         node[1].parent = node
         self.preorder(node[1])
-        self.indentLess()
+        self.indent_less()
 
         if_ret_at_end = False
         if len(node[2][0]) >= 3:
@@ -437,23 +438,23 @@ class FragmentsWalker(pysource.SourceWalker, object):
         for n in return_stmts_node[0]:
             if (n[0] == 'ifstmt' and n[0][1][0] == 'return_if_stmts'):
                 if prev_stmt_is_if_ret:
-                    n[0].type = 'elifstmt'
+                    n[0].kind = 'elifstmt'
                 prev_stmt_is_if_ret = True
             else:
                 prev_stmt_is_if_ret = False
                 if not past_else and not if_ret_at_end:
                     self.println(self.indent, 'else:')
-                    self.indentMore()
+                    self.indent_more()
                     past_else = True
             n.parent = node
             self.preorder(n)
         if not past_else or if_ret_at_end:
             self.println(self.indent, 'else:')
-            self.indentMore()
+            self.indent_more()
         node[2][1].parent = node
         self.preorder(node[2][1])
         self.set_pos_info(node, start, len(self.f.getvalue()))
-        self.indentLess()
+        self.indent_less()
         self.prune()
 
     def n_elifelsestmtr(self, node):
@@ -470,20 +471,20 @@ class FragmentsWalker(pysource.SourceWalker, object):
         node[0].parent = node
         self.preorder(node[0])
         self.println(':')
-        self.indentMore()
+        self.indent_more()
         node[1].parent = node
         self.preorder(node[1])
-        self.indentLess()
+        self.indent_less()
 
         for n in node[2][0]:
-            n[0].type = 'elifstmt'
+            n[0].kind = 'elifstmt'
             n.parent = node
             self.preorder(n)
         self.println(self.indent, 'else:')
-        self.indentMore()
+        self.indent_more()
         node[2][1].parent = node
         self.preorder(node[2][1])
-        self.indentLess()
+        self.indent_less()
         self.set_pos_info(node, start, len(self.f.getvalue()))
         self.prune()
 
@@ -492,7 +493,7 @@ class FragmentsWalker(pysource.SourceWalker, object):
         iname = node[0].pattr
 
         store_import_node = node[-1][-1]
-        assert store_import_node.type.startswith('STORE_')
+        assert store_import_node.kind.startswith('STORE_')
 
         sname = store_import_node.pattr
         self.write(iname)
@@ -527,7 +528,7 @@ class FragmentsWalker(pysource.SourceWalker, object):
         self.write(func_name)
         self.set_pos_info(code_node, start, len(self.f.getvalue()))
 
-        self.indentMore()
+        self.indent_more()
         start = len(self.f.getvalue())
         self.make_function(node, isLambda=False, codeNode=code_node)
 
@@ -537,7 +538,7 @@ class FragmentsWalker(pysource.SourceWalker, object):
             self.write('\n\n')
         else:
             self.write('\n\n\n')
-        self.indentLess()
+        self.indent_less()
         self.prune() # stop recursing
 
     def n_list_compr(self, node):
@@ -553,7 +554,7 @@ class FragmentsWalker(pysource.SourceWalker, object):
             elif n == 'list_if':	n = n[2]
             elif n == 'list_if_not': n= n[2]
         assert n == 'lc_body'
-        if node[0].type.startswith('BUILD_LIST'):
+        if node[0].kind.startswith('BUILD_LIST'):
             start = len(self.f.getvalue())
             self.set_pos_info(node[0], start, start+1)
         self.write( '[ ')
@@ -686,7 +687,7 @@ class FragmentsWalker(pysource.SourceWalker, object):
 
         # Python 2.7+ starts including set_comp_body
         # Python 3.5+ starts including setcomp_func
-        assert n.type in ('lc_body', 'comp_body', 'setcomp_func', 'set_comp_body'), ast
+        assert n.kind in ('lc_body', 'comp_body', 'setcomp_func', 'set_comp_body'), ast
         assert designator, "Couldn't find designator in list/set comprehension"
 
         old_name = self.name
@@ -713,7 +714,7 @@ class FragmentsWalker(pysource.SourceWalker, object):
             self.preorder(if_node)
         self.prec = p
         self.name = old_name
-        if node[-1].type.startswith('CALL_FUNCTION'):
+        if node[-1].kind.startswith('CALL_FUNCTION'):
             self.set_pos_info(node[-1], gen_start, len(self.f.getvalue()))
 
     def listcomprehension_walk2(self, node):
@@ -742,7 +743,7 @@ class FragmentsWalker(pysource.SourceWalker, object):
                 n = n[3]
             elif n in ('list_if', 'list_if_not'):
                 # FIXME: just a guess
-                if n[0].type == 'expr':
+                if n[0].kind == 'expr':
                     list_if = n
                 else:
                     list_if = n[1]
@@ -772,7 +773,10 @@ class FragmentsWalker(pysource.SourceWalker, object):
     def n_genexpr(self, node):
         start = len(self.f.getvalue())
         self.write('(')
-        code_index = -6 if self.version > 3.2 else -5
+        if self.version > 3.2:
+            code_index = -6
+        else:
+            code_index = -5
         self.comprehension_walk(node, iter_index=3, code_index=code_index)
         self.write(')')
         self.set_pos_info(node, start, len(self.f.getvalue()))
@@ -785,7 +789,7 @@ class FragmentsWalker(pysource.SourceWalker, object):
             start = len(self.f.getvalue())
             self.set_pos_info(node[0], start-1, start)
             self.comprehension_walk3(node, 1, 0)
-        elif node[0].type == 'load_closure':
+        elif node[0].kind == 'load_closure':
             self.setcomprehension_walk3(node, collection_index=4)
         else:
             self.comprehension_walk(node, iter_index=4)
@@ -804,7 +808,7 @@ class FragmentsWalker(pysource.SourceWalker, object):
         self.set_pos_info(node[0], start, len(self.f.getvalue()))
         self.write(': {')
         start = len(self.f.getvalue())
-        assert node[0].type.startswith('BUILD_SET')
+        assert node[0].kind.startswith('BUILD_SET')
         self.set_pos_info(node[0], start-1, start)
         designator = node[3]
         assert designator == 'designator'
@@ -813,7 +817,7 @@ class FragmentsWalker(pysource.SourceWalker, object):
         fin = len(self.f.getvalue())
         self.set_pos_info(designator, start, fin)
         for_iter_node = node[2]
-        assert for_iter_node.type == 'FOR_ITER'
+        assert for_iter_node.kind == 'FOR_ITER'
         self.set_pos_info(for_iter_node, start, fin)
         self.write(" for ")
         self.preorder(designator)
@@ -832,7 +836,7 @@ class FragmentsWalker(pysource.SourceWalker, object):
 
     def n_listcomp(self, node):
         self.write('[')
-        if node[0].type == 'load_closure':
+        if node[0].kind == 'load_closure':
             self.listcomprehension_walk2(node)
         else:
             if node[0] == 'LOAD_LISTCOMP':
@@ -846,7 +850,7 @@ class FragmentsWalker(pysource.SourceWalker, object):
         if len(node) > 1:
             if (node[0] == 'c_stmts_opt' and
                 node[0][0] == 'passstmt' and
-                node[1].type.startswith('JUMP_FORWARD')):
+                node[1].kind.startswith('JUMP_FORWARD')):
                 self.set_pos_info(node[1], node[0][0].start, node[0][0].finish)
 
     def setcomprehension_walk3(self, node, collection_index):
@@ -877,7 +881,7 @@ class FragmentsWalker(pysource.SourceWalker, object):
                 n = n[3]
             elif n in ('list_if', 'list_if_not', 'comp_if', 'comp_if_not'):
                 # FIXME: just a guess
-                if n[0].type == 'expr':
+                if n[0].kind == 'expr':
                     list_if = n
                 else:
                     list_if = n[1]
@@ -926,7 +930,10 @@ class FragmentsWalker(pysource.SourceWalker, object):
                         subclass = n.attr
                         break
                     pass
-                subclass_info = node if node == 'classdefdeco2' else node[0]
+                if node == 'classdefdeco2':
+                    subclass_info = node
+                else:
+                    subclass_info = node[0]
             elif buildclass[1][0] == 'load_closure':
                 # Python 3 with closures not functions
                 load_closure = buildclass[1]
@@ -950,7 +957,10 @@ class FragmentsWalker(pysource.SourceWalker, object):
                 subclass = buildclass[1][0].attr
                 subclass_info = node[0]
         else:
-            buildclass = node if (node == 'classdefdeco2') else node[0]
+            if node == 'classdefdeco2':
+                buildclass = node
+            else:
+                buildclass = node[0]
             build_list = buildclass[1][0]
             if hasattr(buildclass[-3][0], 'attr'):
                 subclass = buildclass[-3][0].attr
@@ -977,9 +987,9 @@ class FragmentsWalker(pysource.SourceWalker, object):
         self.println(':')
 
         # class body
-        self.indentMore()
+        self.indent_more()
         self.build_class(subclass)
-        self.indentLess()
+        self.indent_less()
 
         self.currentclass = cclass
         self.set_pos_info(node, start, len(self.f.getvalue()))
@@ -1016,7 +1026,9 @@ class FragmentsWalker(pysource.SourceWalker, object):
             tokens.append(Token('LAMBDA_MARKER'))
             try:
                 ast = parser.parse(self.p, tokens, customize)
-            except (parser.ParserError, AssertionError) as e:
+            except parser.ParserError(e):
+                raise ParserError(e, tokens)
+            except AssertionError(e):
                 raise ParserError(e, tokens)
             maybe_show_ast(self.showast, ast)
             return ast
@@ -1032,8 +1044,8 @@ class FragmentsWalker(pysource.SourceWalker, object):
         # NOTE: this differs from behavior in pysource.py
 
         if len(tokens) >= 2 and not noneInNames:
-            if tokens[-1].type == 'RETURN_VALUE':
-                if tokens[-2].type != 'LOAD_CONST':
+            if tokens[-1].kind == 'RETURN_VALUE':
+                if tokens[-2].kind != 'LOAD_CONST':
                     tokens.append(Token('RETURN_LAST'))
         if len(tokens) == 0:
             return
@@ -1041,7 +1053,9 @@ class FragmentsWalker(pysource.SourceWalker, object):
         # Build AST from disassembly.
         try:
             ast = parser.parse(self.p, tokens, customize)
-        except (parser.ParserError, AssertionError) as e:
+        except parser.ParserError(e):
+            raise ParserError(e, tokens)
+        except AssertionError(e):
             raise ParserError(e, tokens)
 
         maybe_show_ast(self.showast, ast)
@@ -1285,10 +1299,10 @@ class FragmentsWalker(pysource.SourceWalker, object):
         # as a custom rule
         start = len(self.f.getvalue())
         n = len(node)-1
-        assert node[n].type.startswith('CALL_FUNCTION')
+        assert node[n].kind.startswith('CALL_FUNCTION')
 
         for i in range(n-2, 0, -1):
-            if not node[i].type in ['expr', 'LOAD_CLASSNAME']:
+            if not node[i].kind in ['expr', 'LOAD_CLASSNAME']:
                 break
             pass
 
@@ -1316,14 +1330,14 @@ class FragmentsWalker(pysource.SourceWalker, object):
         p = self.prec
         self.prec = 100
 
-        self.indentMore(INDENT_PER_LEVEL)
+        self.indent_more(INDENT_PER_LEVEL)
         line_seperator = ',\n' + self.indent
         sep = INDENT_PER_LEVEL[:-1]
         start = len(self.f.getvalue())
         self.write('{')
 
         if self.version > 3.0:
-            if node[0].type.startswith('kvlist'):
+            if node[0].kind.startswith('kvlist'):
                 # Python 3.5+ style key/value list in mapexpr
                 kv_node = node[0]
                 l = list(kv_node)
@@ -1338,11 +1352,11 @@ class FragmentsWalker(pysource.SourceWalker, object):
                     i += 2
                     pass
                 pass
-            elif node[1].type.startswith('kvlist'):
+            elif node[1].kind.startswith('kvlist'):
                 # Python 3.0..3.4 style key/value list in mapexpr
                 kv_node = node[1]
                 l = list(kv_node)
-                if len(l) > 0 and l[0].type == 'kv3':
+                if len(l) > 0 and l[0].kind == 'kv3':
                     # Python 3.2 does this
                     kv_node = node[1][0]
                     l = list(kv_node)
@@ -1365,7 +1379,7 @@ class FragmentsWalker(pysource.SourceWalker, object):
             pass
         else:
             # Python 2 style kvlist
-            assert node[-1].type.startswith('kvlist')
+            assert node[-1].kind.startswith('kvlist')
             kv_node = node[-1] # goto kvlist
 
             for kv in kv_node:
@@ -1393,7 +1407,7 @@ class FragmentsWalker(pysource.SourceWalker, object):
             n.parent = node
             self.set_pos_info(n, start, finish)
         self.set_pos_info(node, start, finish)
-        self.indentLess(INDENT_PER_LEVEL)
+        self.indent_less(INDENT_PER_LEVEL)
         self.prec = p
         self.prune()
 
@@ -1404,7 +1418,7 @@ class FragmentsWalker(pysource.SourceWalker, object):
         p = self.prec
         self.prec = 100
         n = node.pop()
-        lastnode = n.type
+        lastnode = n.kind
         start = len(self.f.getvalue())
         if lastnode.startswith('BUILD_LIST'):
             self.write('['); endchar = ']'
@@ -1429,7 +1443,7 @@ class FragmentsWalker(pysource.SourceWalker, object):
             else:
                 flat_elems.append(elem)
 
-        self.indentMore(INDENT_PER_LEVEL)
+        self.indent_more(INDENT_PER_LEVEL)
         if len(node) > 3:
             line_separator = ',\n' + self.indent
         else:
@@ -1454,14 +1468,14 @@ class FragmentsWalker(pysource.SourceWalker, object):
         n.parent = node.parent
         self.set_pos_info(n, start, finish)
         self.set_pos_info(node, start, finish)
-        self.indentLess(INDENT_PER_LEVEL)
+        self.indent_less(INDENT_PER_LEVEL)
         self.prec = p
         self.prune()
 
-    def engine(self, entry, startnode):
+    def template_engine(self, entry, startnode):
         """The format template interpetation engine.  See the comment at the
-        beginning of this module for the how we interpret format specifications such as
-        %c, %C, and so on.
+        beginning of this module for the how we interpret format
+        specifications such as %c, %C, and so on.
         """
 
         # print("-----")
@@ -1498,8 +1512,8 @@ class FragmentsWalker(pysource.SourceWalker, object):
                 self.write('%')
                 self.set_pos_info(node, start, len(self.f.getvalue()))
 
-            elif typ == '+': self.indentMore()
-            elif typ == '-': self.indentLess()
+            elif typ == '+': self.indent_more()
+            elif typ == '-': self.indent_less()
             elif typ == '|': self.write(self.indent)
             # no longer used, since BUILD_TUPLE_n is pretty printed:
             elif typ == 'r': recurse_node = True
@@ -1519,7 +1533,7 @@ class FragmentsWalker(pysource.SourceWalker, object):
                 # for loops have two positions that correspond to a single text
                 # location. In "for i in ..." there is the initialization "i" code as well
                 # as the iteration code with "i"
-                match = re.search(r'^for', startnode.type)
+                match = re.search(r'^for', startnode.kind)
                 if match and entry[arg] == 3:
                     self.set_pos_info(node[0], start, finish)
                     for n in node[2]:
@@ -1613,7 +1627,7 @@ class FragmentsWalker(pysource.SourceWalker, object):
         # 2. subroutine calls. It the last op is the call and for purposes of printing
         # we don't need to print anything special there. However it encompases the
         # entire string of the node fn(...)
-        match = re.search(r'^call_function', startnode.type)
+        match = re.search(r'^call_function', startnode.kind)
         if match:
             last_node = startnode[-1]
             # import traceback; traceback.print_stack()
@@ -1754,7 +1768,7 @@ if __name__ == '__main__':
             nodeInfo = walk.offsets[name, offset]
             node = nodeInfo.node
             extractInfo = walk.extract_node_info(node)
-            print("code: %s" % node.type)
+            print("code: %s" % node.kind)
             # print extractInfo
             print(extractInfo.selectedText)
             print(extractInfo.selectedLine)
@@ -1764,7 +1778,7 @@ if __name__ == '__main__':
                 print("Contained in...")
                 print(extractInfo.selectedLine)
                 print(extractInfo.markerLine)
-                print("code: %s" % p.type)
+                print("code: %s" % p.kind)
                 print('=' * 40)
                 pass
             pass
@@ -1783,7 +1797,7 @@ if __name__ == '__main__':
             nodeInfo = walk.offsets[name, offset]
             node = nodeInfo.node
             extractInfo = walk.extract_node_info(node)
-            print("code: %s" % node.type)
+            print("code: %s" % node.kind)
             # print extractInfo
             print(extractInfo.selectedText)
             print(extractInfo.selectedLine)
@@ -1793,7 +1807,7 @@ if __name__ == '__main__':
                 print("Contained in...")
                 print(extractInfo.selectedLine)
                 print(extractInfo.markerLine)
-                print("code: %s" % p.type)
+                print("code: %s" % p.kind)
                 print('=' * 40)
                 pass
             pass

uncompyle6/semantics/make_function.py

@@ -6,14 +6,9 @@ All the crazy things we have to do to handle Python functions
 from xdis.code import iscode, code_has_star_arg, code_has_star_star_arg
 from uncompyle6.scanner import Code
 from uncompyle6.parsers.astnode import AST
-from uncompyle6 import PYTHON3
 from uncompyle6.semantics.parser_error import ParserError
 from uncompyle6.semantics.helper import print_docstring
 
-if PYTHON3:
-    from itertools import zip_longest
-else:
-    from itertools import izip_longest as zip_longest
 
 from uncompyle6.show import maybe_show_ast_param_default
 
@@ -22,7 +17,7 @@ def find_all_globals(node, globs):
     for n in node:
         if isinstance(n, AST):
             globs = find_all_globals(n, globs)
-        elif n.type in ('STORE_GLOBAL', 'DELETE_GLOBAL', 'LOAD_GLOBAL'):
+        elif n.kind in ('STORE_GLOBAL', 'DELETE_GLOBAL', 'LOAD_GLOBAL'):
             globs.add(n.pattr)
     return globs
 
@@ -31,7 +26,7 @@ def find_globals(node, globs):
     for n in node:
         if isinstance(n, AST):
             globs = find_globals(n, globs)
-        elif n.type in ('STORE_GLOBAL', 'DELETE_GLOBAL'):
+        elif n.kind in ('STORE_GLOBAL', 'DELETE_GLOBAL'):
             globs.add(n.pattr)
     return globs
 
@@ -41,7 +36,7 @@ def find_none(node):
             if n not in ('return_stmt', 'return_if_stmt'):
                 if find_none(n):
                     return True
-        elif n.type == 'LOAD_CONST' and n.pattr is None:
+        elif n.kind == 'LOAD_CONST' and n.pattr is None:
             return True
     return False
 
@@ -69,7 +64,7 @@ def make_function3_annotate(self, node, isLambda, nested=1,
             return name
 
     # MAKE_FUNCTION_... or MAKE_CLOSURE_...
-    assert node[-1].type.startswith('MAKE_')
+    assert node[-1].kind.startswith('MAKE_')
 
     annotate_tuple = None
     for annotate_last in range(len(node)-1, -1, -1):
@@ -85,7 +80,7 @@ def make_function3_annotate(self, node, isLambda, nested=1,
         i = -1
         j = annotate_last-1
         l = -len(node)
-        while j >= l and node[j].type in ('annotate_arg' 'annotate_tuple'):
+        while j >= l and node[j].kind in ('annotate_arg' 'annotate_tuple'):
             annotate_args[annotate_tup[i]] = node[j][0]
             i -= 1
             j -= 1
@@ -111,7 +106,7 @@ def make_function3_annotate(self, node, isLambda, nested=1,
         lambda_index = None
 
     if lambda_index and isLambda and iscode(node[lambda_index].attr):
-        assert node[lambda_index].type == 'LOAD_LAMBDA'
+        assert node[lambda_index].kind == 'LOAD_LAMBDA'
         code = node[lambda_index].attr
     else:
         code = codeNode.attr
@@ -128,7 +123,7 @@ def make_function3_annotate(self, node, isLambda, nested=1,
                              code._customize,
                              isLambda = isLambda,
                              noneInNames = ('None' in code.co_names))
-    except ParserError as p:
+    except ParserError, p:
         self.write(str(p))
         self.ERROR = p
         return
@@ -159,7 +154,7 @@ def make_function3_annotate(self, node, isLambda, nested=1,
         self.write(suffix, param)
         suffix = ', '
         if param in annotate_tuple[0].attr:
-            p = annotate_tuple[0].attr.index(param)
+            p = [x for x in annotate_tuple[0].attr].index(param)
             self.write(': ')
             self.preorder(node[p])
             if (line_number != self.line_number):
@@ -171,7 +166,10 @@ def make_function3_annotate(self, node, isLambda, nested=1,
             # else:
             #     self.write(': %s' % value)
 
-    suffix = ', ' if i > 0 else ''
+    if i > 0:
+        suffix = ', '
+    else:
+        suffix = ''
     for n in node:
         if n == 'pos_arg':
             no_paramnames = False
@@ -320,7 +318,7 @@ def make_function2(self, node, isLambda, nested=1, codeNode=None):
             return name
 
     # MAKE_FUNCTION_... or MAKE_CLOSURE_...
-    assert node[-1].type.startswith('MAKE_')
+    assert node[-1].kind.startswith('MAKE_')
 
     args_node = node[-1]
     if isinstance(args_node.attr, tuple):
@@ -336,7 +334,7 @@ def make_function2(self, node, isLambda, nested=1, codeNode=None):
     lambda_index = None
 
     if lambda_index and isLambda and iscode(node[lambda_index].attr):
-        assert node[lambda_index].type == 'LOAD_LAMBDA'
+        assert node[lambda_index].kind == 'LOAD_LAMBDA'
         code = node[lambda_index].attr
     else:
         code = codeNode.attr
@@ -356,17 +354,21 @@ def make_function2(self, node, isLambda, nested=1, codeNode=None):
                              code._customize,
                              isLambda = isLambda,
                              noneInNames = ('None' in code.co_names))
-    except ParserError as p:
+    except ParserError, p:
         self.write(str(p))
         self.ERROR = p
         return
 
-    kw_pairs = args_node.attr[1] if self.version >= 3.0 else 0
+    if self.version >= 3.0:
+        kw_pairs = args_node.attr[1]
+    else:
+        kw_pairs = 0
     indent = self.indent
 
     # build parameters
+    tup = [paramnames, defparams]
     params = [build_param(ast, name, default) for
-              name, default in zip_longest(paramnames, defparams, fillvalue=None)]
+              name, default in map(lambda *tup:tup, *tup)]
     params.reverse() # back to correct order
 
     if code_has_star_arg(code):
@@ -427,34 +429,10 @@ def make_function2(self, node, isLambda, nested=1, codeNode=None):
 
 
 def make_function3(self, node, isLambda, nested=1, codeNode=None):
-    """Dump function definition, doc string, and function body in
-      Python version 3.0 and above
-    """
+    """Dump function definition, doc string, and function body."""
 
-    # For Python 3.3, the evaluation stack in MAKE_FUNCTION is:
-
-    # * default argument objects in positional order
-    # * pairs of name and default argument, with the name just below
-    #   the object on the stack, for keyword-only parameters
-    # * parameter annotation objects
-    # * a tuple listing the parameter names for the annotations
-    #   (only if there are ony annotation objects)
-    # * the code associated with the function (at TOS1)
-    # * the qualified name of the function (at TOS)
-
-    # For Python 3.0 .. 3.2 the evaluation stack is:
-    # The function object is defined to have argc default parameters,
-    # which are found below TOS.
-    # * first come positional args in the order they are given in the source,
-    # * next come the keyword args in the order they given in the source,
-    # * finally is the code associated with the function (at TOS)
-    #
-    # Note: There is no qualified name at TOS
-
-    # MAKE_CLOSURE adds an additional closure slot
-
-    # Thank you, Python, for a such a well-thought out system that has
-    # changed 4 or so times.
+    # FIXME: call make_function3 if we are self.version >= 3.0
+    # and then simplify the below.
 
     def build_param(ast, name, default):
         """build parameters:
@@ -472,33 +450,23 @@ def make_function3(self, node, isLambda, nested=1, codeNode=None):
             return name
 
     # MAKE_FUNCTION_... or MAKE_CLOSURE_...
-    assert node[-1].type.startswith('MAKE_')
-
-
-    # Python 3.3+ adds a qualified name at TOS (-1)
-    # moving down the LOAD_LAMBDA instruction
-    if 3.0 <= self.version <= 3.2:
-        lambda_index = -2
-    elif 3.03 <= self.version:
-        lambda_index = -3
-    else:
-        lambda_index = None
+    assert node[-1].kind.startswith('MAKE_')
 
     args_node = node[-1]
     if isinstance(args_node.attr, tuple):
-        pos_args, kw_args, annotate_argc  = args_node.attr
-        if self.version <= 3.3 and len(node) > 2 and node[lambda_index] != 'LOAD_LAMBDA':
-            # args are after kwargs; kwargs are bundled as one node
+        if self.version <= 3.3 and len(node) > 2 and node[-3] != 'LOAD_LAMBDA':
+            # positional args are after kwargs
             defparams = node[1:args_node.attr[0]+1]
         else:
-            # args are before kwargs; kwags as bundled as one node
+            # positional args are before kwargs
             defparams = node[:args_node.attr[0]]
+        pos_args, kw_args, annotate_argc  = args_node.attr
     else:
         if self.version < 3.6:
             defparams = node[:args_node.attr]
         else:
             default, kw, annotate, closure = args_node.attr
-            # FIXME: start here for Python 3.6 and above:
+            # FIXME: start here.
             defparams = []
             # if default:
             #     defparams = node[-(2 +  kw + annotate  + closure)]
@@ -508,9 +476,15 @@ def make_function3(self, node, isLambda, nested=1, codeNode=None):
         kw_args  = 0
         pass
 
+    if 3.0 <= self.version <= 3.2:
+        lambda_index = -2
+    elif 3.03 <= self.version:
+        lambda_index = -3
+    else:
+        lambda_index = None
 
     if lambda_index and isLambda and iscode(node[lambda_index].attr):
-        assert node[lambda_index].type == 'LOAD_LAMBDA'
+        assert node[lambda_index].kind == 'LOAD_LAMBDA'
         code = node[lambda_index].attr
     else:
         code = codeNode.attr
@@ -523,7 +497,7 @@ def make_function3(self, node, isLambda, nested=1, codeNode=None):
     paramnames = list(code.co_varnames[:argc])
 
     # defaults are for last n parameters, thus reverse
-    if not 3.0 <= self.version <= 3.1:
+    if not 3.0 <= self.version <= 3.2:
         paramnames.reverse(); defparams.reverse()
 
     try:
@@ -531,33 +505,71 @@ def make_function3(self, node, isLambda, nested=1, codeNode=None):
                              code._customize,
                              isLambda = isLambda,
                              noneInNames = ('None' in code.co_names))
-    except ParserError as p:
+    except ParserError, p:
         self.write(str(p))
         self.ERROR = p
         return
 
-    kw_pairs = args_node.attr[1] if self.version >= 3.0 else 0
+    if self.version >= 3.0:
+        kw_pairs = args_node.attr[1]
+    else:
+        kw_pairs = 0
+    indent = self.indent
 
     # build parameters
-    params = [build_param(ast, name, d) for
-              name, d in zip_longest(paramnames, defparams, fillvalue=None)]
-
-    if not 3.0 <= self.version <= 3.1:
+    if self.version != 3.2:
+        tup = [paramnames, defparams]
+        params = [build_param(ast, name, default) for
+              name, default in map(lambda *tup:tup, *tup)]
         params.reverse() # back to correct order
 
-    if code_has_star_arg(code):
-        if self.version > 3.0:
-            params.append('*%s' % code.co_varnames[argc + kw_pairs])
-        else:
-            params.append('*%s' % code.co_varnames[argc])
-        argc += 1
+        if code_has_star_arg(code):
+            if self.version > 3.0:
+                params.append('*%s' % code.co_varnames[argc + kw_pairs])
+            else:
+                params.append('*%s' % code.co_varnames[argc])
+            argc += 1
+
+        # dump parameter list (with default values)
+        if isLambda:
+            self.write("lambda ", ", ".join(params))
+        else:
+            self.write("(", ", ".join(params))
+        # self.println(indent, '#flags:\t', int(code.co_flags))
 
-    # dump parameter list (with default values)
-    if isLambda:
-        self.write("lambda ", ", ".join(params))
     else:
-        self.write("(", ", ".join(params))
-    # self.println(indent, '#flags:\t', int(code.co_flags))
+        if isLambda:
+            self.write("lambda ")
+        else:
+            self.write("(")
+            pass
+
+        last_line = self.f.getvalue().split("\n")[-1]
+        l = len(last_line)
+        indent = ' ' * l
+        line_number = self.line_number
+
+        if code_has_star_arg(code):
+            self.write('*%s' % code.co_varnames[argc + kw_pairs])
+            argc += 1
+
+        i = len(paramnames) - len(defparams)
+        self.write(", ".join(paramnames[:i]))
+        if i > 0:
+            suffix = ', '
+        else:
+            suffix = ''
+        for n in node:
+            if n == 'pos_arg':
+                self.write(suffix)
+                self.write(paramnames[i] + '=')
+                i += 1
+                self.preorder(n)
+                if (line_number != self.line_number):
+                    suffix = ",\n" + indent
+                    line_number = self.line_number
+                else:
+                    suffix = ', '
 
     if kw_args > 0:
         if not (4 & code.co_flags):
@@ -573,7 +585,7 @@ def make_function3(self, node, isLambda, nested=1, codeNode=None):
             for n in node:
                 if n == 'pos_arg':
                     continue
-                elif self.version >= 3.4 and not (n.type in ('kwargs', 'kwarg')):
+                elif self.version >= 3.4 and not (n.kind in ('kwargs', 'kwarg')):
                     continue
                 else:
                     self.preorder(n)

uncompyle6/semantics/pysource.py

@@ -11,63 +11,101 @@ and what they mean).
 
 Upper levels of the grammar is a more-or-less conventional grammar for
 Python.
-
-Semantic action rules for nonterminal symbols can be specified here by
-creating a method prefaced with "n_" for that nonterminal. For
-example, "n_exec_stmt" handles the semantic actions for the
-"exec_smnt" nonterminal symbol. Similarly if a method with the name
-of the nonterminal is suffixed with "_exit" it will be called after
-all of its children are called.
-
-Another other way to specify a semantic rule for a nonterminal is via
-rule given in one of the tables MAP_R0, MAP_R, or MAP_DIRECT.
-
-These uses a printf-like syntax to direct substitution from attributes
-of the nonterminal and its children..
-
-The rest of the below describes how table-driven semantic actions work
-and gives a list of the format specifiers. The default() and engine()
-methods implement most of the below.
-
-  Step 1 determines a table (T) and a path to a
-  table key (K) from the node type (N) (other nodes are shown as O):
-
-         N                  N               N&K
-     / | ... \          / | ... \        / | ... \
-    O  O      O        O  O      K      O  O      O
-              |
-              K
-
-  MAP_R0 (TABLE_R0)  MAP_R (TABLE_R)  MAP_DIRECT (TABLE_DIRECT)
-
-  The default is a direct mapping.  The key K is then extracted from the
-  subtree and used to find a table entry T[K], if any.  The result is a
-  format string and arguments (a la printf()) for the formatting engine.
-  Escapes in the format string are:
-
-    %c  evaluate children N[A] recursively*
-    %C  evaluate children N[A[0]]..N[A[1]-1] recursively, separate by A[2]*
-    %P  same as %C but sets operator precedence
-    %D  same as %C but is for left-recursive lists like kwargs which
-        goes to epsilon at the beginning. Using %C an extra separator
-        with an epsilon appears at the beginning
-    %,  print ',' if last %C only printed one item. This is mostly for tuples
-        on the LHS of an assignment statement since BUILD_TUPLE_n pretty-prints
-        other tuples.
-    %|  tab to current indentation level
-    %+ increase current indentation level
-    %- decrease current indentation level
-    %{...} evaluate ... in context of N
-    %% literal '%'
-    %p evaluate N setting precedence
-
-
-  * indicates an argument (A) required.
-
-  The '%' may optionally be followed by a number (C) in square brackets, which
-  makes the engine walk down to N[C] before evaluating the escape code.
 """
-from __future__ import print_function
+
+# The below is a bit long, but still it is somehwat abbreviated.
+# See https://github.com/rocky/python-uncompyle6/wiki/Table-driven-semantic-actions.
+# for a more complete explanation, nicely marked up and with examples.
+#
+#
+# Semantic action rules for nonterminal symbols can be specified here by
+# creating a method prefaced with "n_" for that nonterminal. For
+# example, "n_exec_stmt" handles the semantic actions for the
+# "exec_stmt" nonterminal symbol. Similarly if a method with the name
+# of the nonterminal is suffixed with "_exit" it will be called after
+# all of its children are called.
+#
+# After a while writing methods this way, you'll find many routines which do similar
+# sorts of things, and soon you'll find you want a short notation to
+# describe rules and not have to create methods at all.
+#
+# So another other way to specify a semantic rule for a nonterminal is via
+# one of the tables MAP_R0, MAP_R, or MAP_DIRECT where the key is the
+# nonterminal name.
+#
+# These dictionaries use a printf-like syntax to direct substitution
+# from attributes of the nonterminal and its children..
+#
+# The rest of the below describes how table-driven semantic actions work
+# and gives a list of the format specifiers. The default() and
+# template_engine() methods implement most of the below.
+#
+# We allow for a couple of ways to interact with a node in a tree.  So
+# step 1 after not seeing a custom method for a nonterminal is to
+# determine from what point of view tree-wise the rule is applied.
+
+# In the diagram below, N is a nonterminal name, and K also a nonterminal
+# name but the one used as a key in the table.
+# we show where those are with respect to each other in the
+# AST tree for N.
+#
+#
+#          N&K               N                  N
+#         / | ... \        / | ... \          / | ... \
+#        O  O      O      O  O      K         O O      O
+#                                                      |
+#                                                      K
+#      TABLE_DIRECT      TABLE_R             TABLE_R0
+#
+#   The default table is TABLE_DIRECT mapping By far, most rules used work this way.
+#   TABLE_R0 is rarely used.
+#
+#   The key K is then extracted from the subtree and used to find one
+#   of the tables, T listed above.  The result after applying T[K] is
+#   a format string and arguments (a la printf()) for the formatting
+#   engine.
+#
+#   Escapes in the format string are:
+#
+#     %c  evaluate the node recursively. Its argument is a single
+#         integer representing a node index.
+#
+#     %p  like %c but sets the operator precedence.
+#         Its argument then is a tuple indicating the node
+#         index and the precidence value, an integer.
+#
+#     %C  evaluate children recursively, with sibling children separated by the
+#         given string.  It needs a 3-tuple: a starting node, the maximimum
+#         value of an end node, and a string to be inserted between sibling children
+#
+#     %,  Append ',' if last %C only printed one item. This is mostly for tuples
+#         on the LHS of an assignment statement since BUILD_TUPLE_n pretty-prints
+#         other tuples. The specifier takes no arguments
+#
+#     %P same as %C but sets operator precedence.  Its argument is a 4-tuple:
+#         the node low and high indices, the separator, a string the precidence
+#         value, an integer.
+#
+#     %D Same as `%C` this is for left-recursive lists like kwargs where goes
+#         to epsilon at the beginning. It needs a 3-tuple: a starting node, the
+#         maximimum value of an end node, and a string to be inserted between
+#         sibling children. If we were to use `%C` an extra separator with an
+#         epsilon would appear at the beginning.
+#
+#     %|  Insert spaces to the current indentation level. Takes no arguments.
+#
+#     %+ increase current indentation level. Takes no arguments.
+#
+#     %- decrease current indentation level. Takes no arguments.
+#
+#     %{...} evaluate ... in context of N
+#
+#     %% literal '%'. Takes no arguments.
+#
+#
+#   The '%' may optionally be followed by a number (C) in square
+#   brackets, which makes the template_engine walk down to N[C] before
+#   evaluating the escape code.
 
 import sys
 
@@ -105,7 +143,7 @@ else:
 
 def is_docstring(node):
     try:
-        return (node[0][0].type == 'assign' and
+        return (node[0][0].kind == 'assign' and
             node[0][0][1][0].pattr == '__doc__')
     except:
         return False
@@ -124,6 +162,29 @@ class SourceWalker(GenericASTTraversal, object):
                  debug_parser=PARSER_DEFAULT_DEBUG,
                  compile_mode='exec', is_pypy=False,
                  linestarts={}):
+        """version is the Python version (a float) of the Python dialect
+
+        of both the AST and language we should produce.
+
+        out is IO-like file pointer to where the output should go. It
+        whould have a getvalue() method.
+
+        scanner is a method to call when we need to scan tokens. Sometimes
+        in producing output we will run across further tokens that need
+        to be scaned.
+
+        If showast is True, we print the AST tree.
+
+        compile_mode is is either 'exec' or 'single'. It isthe compile
+        mode that was used to create the AST and specifies a gramar variant within
+        a Python version to use.
+
+        is_pypy should be True if the AST was generated for PyPy.
+
+        linestarts is a dictionary of line number to bytecode offset. This
+        can sometimes assist in determinte which kind of source-code construct
+        to use when there is ambiguity.
+        """
         GenericASTTraversal.__init__(self, ast=None)
         self.scanner = scanner
         params = {
@@ -306,7 +367,7 @@ class SourceWalker(GenericASTTraversal, object):
                     # MAKE_FUNCTION ..
                     code = node[-3]
 
-                self.indentMore()
+                self.indent_more()
                 for annotate_last in range(len(node)-1, -1, -1):
                     if node[annotate_last] == 'annotate_tuple':
                         break
@@ -326,7 +387,7 @@ class SourceWalker(GenericASTTraversal, object):
                     self.write('\n\n')
                 else:
                     self.write('\n\n\n')
-                self.indentLess()
+                self.indent_less()
                 self.prune() # stop recursing
             self.n_mkfunc_annotate = n_mkfunc_annotate
 
@@ -358,11 +419,13 @@ class SourceWalker(GenericASTTraversal, object):
                     })
                     def n_async_call_function(node):
                         self.f.write('async ')
-                        node.type == 'call_function'
+                        node.kind == 'call_function'
                         p = self.prec
                         self.prec = 80
-                        self.engine(('%c(%P)', 0, (1, -4, ', ', 100)), node)
+                        self.template_engine(('%c(%P)', 0,
+                                              (1, -4, ', ', 100)), node)
                         self.prec = p
+                        node.kind == 'async_call_function'
                         self.prune()
                     self.n_async_call_function = n_async_call_function
                     self.n_build_list_unpack = self.n_build_list
@@ -375,13 +438,13 @@ class SourceWalker(GenericASTTraversal, object):
                             for i in mapping[1:]:
                                 key = key[i]
                                 pass
-                            if key.type.startswith('CALL_FUNCTION_VAR_KW'):
+                            if key.kind.startswith('CALL_FUNCTION_VAR_KW'):
                                 # Python 3.5 changes the stack position of *args. kwargs come
                                 # after *args whereas in earlier Pythons, *args is at the end
                                 # which simpilfiies things from our perspective.
                                 # Python 3.6+ replaces CALL_FUNCTION_VAR_KW with CALL_FUNCTION_EX
                                 # We will just swap the order to make it look like earlier Python 3.
-                                entry = table[key.type]
+                                entry = table[key.kind]
                                 kwarg_pos = entry[2][1]
                                 args_pos = kwarg_pos - 1
                                 # Put last node[args_pos] after subsequent kwargs
@@ -402,9 +465,11 @@ class SourceWalker(GenericASTTraversal, object):
                         is_code = hasattr(code_node, 'attr') and iscode(code_node.attr)
                         if (is_code and
                             (code_node.attr.co_flags & COMPILER_FLAG_BIT['COROUTINE'])):
-                            self.engine(('\n\n%|async def %c\n', -2), node)
+                            self.template_engine(('\n\n%|async def %c\n',
+                                                  -2), node)
                         else:
-                            self.engine(('\n\n%|def %c\n', -2), node)
+                            self.template_engine(('\n\n%|def %c\n', -2),
+                                                 node)
                         self.prune()
                     self.n_funcdef = n_funcdef
 
@@ -502,10 +567,10 @@ class SourceWalker(GenericASTTraversal, object):
         super(SourceWalker, self).preorder(node)
         self.set_pos_info(node)
 
-    def indentMore(self, indent=TAB):
+    def indent_more(self, indent=TAB):
         self.indent += indent
 
-    def indentLess(self, indent=TAB):
+    def indent_less(self, indent=TAB):
         self.indent = self.indent[:-len(indent)]
 
     def traverse(self, node, indent=None, isLambda=False):
@@ -580,6 +645,20 @@ class SourceWalker(GenericASTTraversal, object):
                     node == AST('return_stmt',
                                 [AST('ret_expr', [NONE]), Token('RETURN_VALUE')]))
 
+    # Python 3.x can have be dead code as a result of its optimization?
+    # So we'll add a # at the end of the return lambda so the rest is ignored
+    def n_return_lambda(self, node):
+        if 1 <= len(node) <= 2:
+            self.preorder(node[0])
+            self.write(' # Avoid dead code: ')
+            self.prune()
+        else:
+            # We can't comment out like above because there may be a trailing ')'
+            # that needs to be written
+            assert len(node) == 3 and node[2] == 'LAMBDA_MARKER'
+            self.preorder(node[0])
+            self.prune()
+
     def n_return_stmt(self, node):
         if self.params['isLambda']:
             self.preorder(node[0])
@@ -597,6 +676,7 @@ class SourceWalker(GenericASTTraversal, object):
 
     def n_return_if_stmt(self, node):
         if self.params['isLambda']:
+            self.write(' return ')
             self.preorder(node[0])
             self.prune()
         else:
@@ -653,12 +733,12 @@ class SourceWalker(GenericASTTraversal, object):
 
     def n_expr(self, node):
         p = self.prec
-        if node[0].type.startswith('binary_expr'):
+        if node[0].kind.startswith('binary_expr'):
             n = node[0][-1][0]
         else:
             n = node[0]
 
-        self.prec = PRECEDENCE.get(n.type, -2)
+        self.prec = PRECEDENCE.get(n.kind, -2)
         if n == 'LOAD_CONST' and repr(n.pattr)[0] == '-':
             self.prec = 6
 
@@ -741,9 +821,9 @@ class SourceWalker(GenericASTTraversal, object):
         self.prune()
 
     def n_delete_subscr(self, node):
-        if node[-2][0] == 'build_list' and node[-2][0][-1].type.startswith('BUILD_TUPLE'):
+        if node[-2][0] == 'build_list' and node[-2][0][-1].kind.startswith('BUILD_TUPLE'):
             if node[-2][0][-1] != 'BUILD_TUPLE_0':
-                node[-2][0].type = 'build_tuple2'
+                node[-2][0].kind = 'build_tuple2'
         self.default(node)
 
     n_store_subscr = n_binary_subscr = n_delete_subscr
@@ -752,9 +832,9 @@ class SourceWalker(GenericASTTraversal, object):
     def n_tryfinallystmt(self, node):
         if len(node[1][0]) == 1 and node[1][0][0] == 'stmt':
             if node[1][0][0][0] == 'trystmt':
-                node[1][0][0][0].type = 'tf_trystmt'
+                node[1][0][0][0].kind = 'tf_trystmt'
             if node[1][0][0][0] == 'tryelsestmt':
-                node[1][0][0][0].type = 'tf_tryelsestmt'
+                node[1][0][0][0].kind = 'tf_tryelsestmt'
         self.default(node)
 
     def n_exec_stmt(self, node):
@@ -779,26 +859,26 @@ class SourceWalker(GenericASTTraversal, object):
 
         if len(n) == 1 == len(n[0]) and n[0] == '_stmts':
             n = n[0][0][0]
-        elif n[0].type in ('lastc_stmt', 'lastl_stmt'):
+        elif n[0].kind in ('lastc_stmt', 'lastl_stmt'):
             n = n[0][0]
         else:
             if not preprocess:
                 self.default(node)
             return
 
-        if n.type in ('ifstmt', 'iflaststmt', 'iflaststmtl'):
-            node.type = 'ifelifstmt'
-            n.type = 'elifstmt'
-        elif n.type in ('ifelsestmtr',):
-            node.type = 'ifelifstmt'
-            n.type = 'elifelsestmtr'
-        elif n.type in ('ifelsestmt', 'ifelsestmtc', 'ifelsestmtl'):
-            node.type = 'ifelifstmt'
+        if n.kind in ('ifstmt', 'iflaststmt', 'iflaststmtl'):
+            node.kind = 'ifelifstmt'
+            n.kind = 'elifstmt'
+        elif n.kind in ('ifelsestmtr',):
+            node.kind = 'ifelifstmt'
+            n.kind = 'elifelsestmtr'
+        elif n.kind in ('ifelsestmt', 'ifelsestmtc', 'ifelsestmtl'):
+            node.kind = 'ifelifstmt'
             self.n_ifelsestmt(n, preprocess=True)
             if n == 'ifelifstmt':
-                n.type = 'elifelifstmt'
-            elif n.type in ('ifelsestmt', 'ifelsestmtc', 'ifelsestmtl'):
-                n.type = 'elifelsestmt'
+                n.kind = 'elifelifstmt'
+            elif n.kind in ('ifelsestmt', 'ifelsestmtc', 'ifelsestmtl'):
+                n.kind = 'elifelsestmt'
         if not preprocess:
             self.default(node)
 
@@ -807,7 +887,7 @@ class SourceWalker(GenericASTTraversal, object):
     def n_ifelsestmtr(self, node):
         if node[2] == 'COME_FROM':
             return_stmts_node = node[3]
-            node.type = 'ifelsestmtr2'
+            node.kind = 'ifelsestmtr2'
         else:
             return_stmts_node = node[2]
         if len(return_stmts_node) != 2:
@@ -823,9 +903,9 @@ class SourceWalker(GenericASTTraversal, object):
         self.write(self.indent, 'if ')
         self.preorder(node[0])
         self.println(':')
-        self.indentMore()
+        self.indent_more()
         self.preorder(node[1])
-        self.indentLess()
+        self.indent_less()
 
         if_ret_at_end = False
         if len(return_stmts_node[0]) >= 3:
@@ -838,27 +918,27 @@ class SourceWalker(GenericASTTraversal, object):
         for n in return_stmts_node[0]:
             if (n[0] == 'ifstmt' and n[0][1][0] == 'return_if_stmts'):
                 if prev_stmt_is_if_ret:
-                    n[0].type = 'elifstmt'
+                    n[0].kind = 'elifstmt'
                 prev_stmt_is_if_ret = True
             else:
                 prev_stmt_is_if_ret = False
                 if not past_else and not if_ret_at_end:
                     self.println(self.indent, 'else:')
-                    self.indentMore()
+                    self.indent_more()
                     past_else = True
             self.preorder(n)
         if not past_else or if_ret_at_end:
             self.println(self.indent, 'else:')
-            self.indentMore()
+            self.indent_more()
         self.preorder(return_stmts_node[1])
-        self.indentLess()
+        self.indent_less()
         self.prune()
     n_ifelsestmtr2 = n_ifelsestmtr
 
     def n_elifelsestmtr(self, node):
         if node[2] == 'COME_FROM':
             return_stmts_node = node[3]
-            node.type = 'elifelsestmtr2'
+            node.kind = 'elifelsestmtr2'
         else:
             return_stmts_node = node[2]
 
@@ -873,22 +953,22 @@ class SourceWalker(GenericASTTraversal, object):
         self.write(self.indent, 'elif ')
         self.preorder(node[0])
         self.println(':')
-        self.indentMore()
+        self.indent_more()
         self.preorder(node[1])
-        self.indentLess()
+        self.indent_less()
 
         for n in return_stmts_node[0]:
-            n[0].type = 'elifstmt'
+            n[0].kind = 'elifstmt'
             self.preorder(n)
         self.println(self.indent, 'else:')
-        self.indentMore()
+        self.indent_more()
         self.preorder(return_stmts_node[1])
-        self.indentLess()
+        self.indent_less()
         self.prune()
 
     def n_import_as(self, node):
         store_node = node[-1][-1]
-        assert store_node.type.startswith('STORE_')
+        assert store_node.kind.startswith('STORE_')
         iname = node[0].pattr  # import name
         sname = store_node.pattr # store_name
         if iname and iname == sname or iname.startswith(sname + '.'):
@@ -924,14 +1004,14 @@ class SourceWalker(GenericASTTraversal, object):
         func_name = code_node.attr.co_name
         self.write(func_name)
 
-        self.indentMore()
+        self.indent_more()
         self.make_function(node, isLambda=False, codeNode=code_node)
 
         if len(self.param_stack) > 1:
             self.write('\n\n')
         else:
             self.write('\n\n\n')
-        self.indentLess()
+        self.indent_less()
         self.prune() # stop recursing
 
     def make_function(self, node, isLambda, nested=1,
@@ -956,7 +1036,10 @@ class SourceWalker(GenericASTTraversal, object):
                 return
             n = node[-1]
         elif node[-1] == 'del_stmt':
-            n = node[-3] if node[-2] == 'JUMP_BACK' else node[-2]
+            if node[-2] == 'JUMP_BACK':
+                n = node[-3]
+            else:
+                n = node[-2]
 
         assert n == 'list_iter'
 
@@ -974,7 +1057,10 @@ class SourceWalker(GenericASTTraversal, object):
             list_iter = node[-1]
         else:
             expr = n[1]
-            list_iter = node[-3] if node[-2] == 'JUMP_BACK' else node[-2]
+            if node[-2] == 'JUMP_BACK':
+                list_iter = node[-3]
+            else:
+                list_iter = node[-2]
 
         assert expr == 'expr'
         assert list_iter == 'list_iter'
@@ -1002,7 +1088,7 @@ class SourceWalker(GenericASTTraversal, object):
         """
         p = self.prec
         self.prec = 27
-        if node[-1].type == 'list_iter':
+        if node[-1].kind == 'list_iter':
             n = node[-1]
         elif self.is_pypy and node[-1] == 'JUMP_BACK':
             n = node[-2]
@@ -1026,7 +1112,10 @@ class SourceWalker(GenericASTTraversal, object):
         self.write( '[ ')
 
         expr = n[0]
-        list_iter = node[-2] if self.is_pypy and node[-1] == 'JUMP_BACK' else node[-1]
+        if self.is_pypy and node[-1] == 'JUMP_BACK':
+            list_iter = node[-2]
+        else:
+            list_iter = node[-1]
 
         assert expr == 'expr'
         assert list_iter == 'list_iter'
@@ -1100,7 +1189,10 @@ class SourceWalker(GenericASTTraversal, object):
         self.write(' for ')
         self.preorder(ast[iter_index-1])
         self.write(' in ')
-        iter_expr = node[2] if node[2] == 'expr' else node[-3]
+        if node[2] == 'expr':
+            iter_expr = node[2]
+        else:
+            iter_expr = node[-3]
         assert iter_expr == 'expr'
         self.preorder(iter_expr)
         self.preorder(ast[iter_index])
@@ -1108,7 +1200,10 @@ class SourceWalker(GenericASTTraversal, object):
 
     def n_genexpr(self, node):
         self.write('(')
-        code_index = -6 if self.version > 3.2 else -5
+        if self.version > 3.2:
+            code_index = -6
+        else:
+            code_index = -5
         self.comprehension_walk(node, iter_index=3, code_index=code_index)
         self.write(')')
         self.prune()
@@ -1117,7 +1212,7 @@ class SourceWalker(GenericASTTraversal, object):
         self.write('{')
         if node[0] in ['LOAD_SETCOMP', 'LOAD_DICTCOMP']:
             self.comprehension_walk3(node, 1, 0)
-        elif node[0].type == 'load_closure' and self.version >= 3.0:
+        elif node[0].kind == 'load_closure' and self.version >= 3.0:
             self.setcomprehension_walk3(node, collection_index=4)
         else:
             self.comprehension_walk(node, iter_index=4)
@@ -1184,7 +1279,7 @@ class SourceWalker(GenericASTTraversal, object):
 
         # Python 2.7+ starts including set_comp_body
         # Python 3.5+ starts including setcomp_func
-        assert n.type in ('lc_body', 'comp_body', 'setcomp_func', 'set_comp_body'), ast
+        assert n.kind in ('lc_body', 'comp_body', 'setcomp_func', 'set_comp_body'), ast
         assert designator, "Couldn't find designator in list/set comprehension"
 
         self.preorder(n[0])
@@ -1234,7 +1329,7 @@ class SourceWalker(GenericASTTraversal, object):
                 n = n[3]
             elif n in ('list_if', 'list_if_not'):
                 # FIXME: just a guess
-                if n[0].type == 'expr':
+                if n[0].kind == 'expr':
                     list_if = n
                 else:
                     list_if = n[1]
@@ -1255,7 +1350,7 @@ class SourceWalker(GenericASTTraversal, object):
 
     def n_listcomp(self, node):
         self.write('[')
-        if node[0].type == 'load_closure':
+        if node[0].kind == 'load_closure':
             self.listcomprehension_walk2(node)
         else:
             self.comprehension_walk3(node, 1, 0)
@@ -1292,7 +1387,7 @@ class SourceWalker(GenericASTTraversal, object):
                 n = n[3]
             elif n in ('list_if', 'list_if_not', 'comp_if', 'comp_if_not'):
                 # FIXME: just a guess
-                if n[0].type == 'expr':
+                if n[0].kind == 'expr':
                     list_if = n
                 else:
                     list_if = n[1]
@@ -1349,7 +1444,10 @@ class SourceWalker(GenericASTTraversal, object):
                             break
                         pass
                     pass
-                subclass_info = node if node == 'classdefdeco2' else node[0]
+                if node == 'classdefdeco2':
+                    subclass_info = node
+                else:
+                    subclass_info = node[0]
             elif buildclass[1][0] == 'load_closure':
                 # Python 3 with closures not functions
                 load_closure = buildclass[1]
@@ -1376,7 +1474,10 @@ class SourceWalker(GenericASTTraversal, object):
                 subclass_code = buildclass[1][0].attr
                 subclass_info = node[0]
         else:
-            buildclass = node if (node == 'classdefdeco2') else node[0]
+            if node == 'classdefdeco2':
+                buildclass = node
+            else:
+                buildclass = node[0]
             build_list = buildclass[1][0]
             if hasattr(buildclass[-3][0], 'attr'):
                 subclass_code = buildclass[-3][0].attr
@@ -1402,9 +1503,9 @@ class SourceWalker(GenericASTTraversal, object):
         self.println(':')
 
         # class body
-        self.indentMore()
+        self.indent_more()
         self.build_class(subclass_code)
-        self.indentLess()
+        self.indent_less()
 
         self.currentclass = cclass
         if len(self.param_stack) > 1:
@@ -1438,10 +1539,10 @@ class SourceWalker(GenericASTTraversal, object):
 
     def print_super_classes3(self, node):
         n = len(node)-1
-        if node.type != 'expr':
-            assert node[n].type.startswith('CALL_FUNCTION')
+        if node.kind != 'expr':
+            assert node[n].kind.startswith('CALL_FUNCTION')
             for i in range(n-2, 0, -1):
-                if not node[i].type in ['expr', 'LOAD_CLASSNAME']:
+                if not node[i].kind in ['expr', 'LOAD_CLASSNAME']:
                     break
                 pass
 
@@ -1475,13 +1576,13 @@ class SourceWalker(GenericASTTraversal, object):
         p = self.prec
         self.prec = 100
 
-        self.indentMore(INDENT_PER_LEVEL)
+        self.indent_more(INDENT_PER_LEVEL)
         sep = INDENT_PER_LEVEL[:-1]
         self.write('{')
         line_number = self.line_number
 
         if self.version >= 3.0 and not self.is_pypy:
-            if node[0].type.startswith('kvlist'):
+            if node[0].kind.startswith('kvlist'):
                 # Python 3.5+ style key/value list in mapexpr
                 kv_node = node[0]
                 l = list(kv_node)
@@ -1504,11 +1605,11 @@ class SourceWalker(GenericASTTraversal, object):
                     i += 2
                     pass
                 pass
-            elif len(node) > 1 and node[1].type.startswith('kvlist'):
+            elif len(node) > 1 and node[1].kind.startswith('kvlist'):
                 # Python 3.0..3.4 style key/value list in mapexpr
                 kv_node = node[1]
                 l = list(kv_node)
-                if len(l) > 0 and l[0].type == 'kv3':
+                if len(l) > 0 and l[0].kind == 'kv3':
                     # Python 3.2 does this
                     kv_node = node[1][0]
                     l = list(kv_node)
@@ -1533,7 +1634,7 @@ class SourceWalker(GenericASTTraversal, object):
                     i += 3
                     pass
                 pass
-            elif node[-1].type.startswith('BUILD_CONST_KEY_MAP'):
+            elif node[-1].kind.startswith('BUILD_CONST_KEY_MAP'):
                 # Python 3.6+ style const map
                 keys = node[-2].pattr
                 values = node[:-2]
@@ -1558,7 +1659,7 @@ class SourceWalker(GenericASTTraversal, object):
             pass
         else:
             # Python 2 style kvlist
-            assert node[-1].type.startswith('kvlist')
+            assert node[-1].kind.startswith('kvlist')
             kv_node = node[-1] # goto kvlist
 
             first_time = True
@@ -1613,7 +1714,7 @@ class SourceWalker(GenericASTTraversal, object):
         if sep.startswith(",\n"):
             self.write(sep[1:])
         self.write('}')
-        self.indentLess(INDENT_PER_LEVEL)
+        self.indent_less(INDENT_PER_LEVEL)
         self.prec = p
         self.prune()
 
@@ -1624,7 +1725,7 @@ class SourceWalker(GenericASTTraversal, object):
         p = self.prec
         self.prec = 100
         lastnode = node.pop()
-        lastnodetype = lastnode.type
+        lastnodetype = lastnode.kind
 
         # If this build list is inside a CALL_FUNCTION_VAR,
         # then the first * has already been printed.
@@ -1664,7 +1765,7 @@ class SourceWalker(GenericASTTraversal, object):
             else:
                 flat_elems.append(elem)
 
-        self.indentMore(INDENT_PER_LEVEL)
+        self.indent_more(INDENT_PER_LEVEL)
         sep = ''
 
         for elem in flat_elems:
@@ -1689,12 +1790,12 @@ class SourceWalker(GenericASTTraversal, object):
         if lastnode.attr == 1 and lastnodetype.startswith('BUILD_TUPLE'):
             self.write(',')
         self.write(endchar)
-        self.indentLess(INDENT_PER_LEVEL)
+        self.indent_less(INDENT_PER_LEVEL)
         self.prec = p
         self.prune()
 
     def n_unpack(self, node):
-        if node[0].type.startswith('UNPACK_EX'):
+        if node[0].kind.startswith('UNPACK_EX'):
             # Python 3+
             before_count, after_count = node[0].attr
             for i in range(before_count+1):
@@ -1709,8 +1810,8 @@ class SourceWalker(GenericASTTraversal, object):
             self.prune()
             return
         for n in node[1:]:
-            if n[0].type == 'unpack':
-                n[0].type = 'unpack_w_parens'
+            if n[0].kind == 'unpack':
+                n[0].kind = 'unpack_w_parens'
         self.default(node)
 
     n_unpack_w_parens = n_unpack
@@ -1719,33 +1820,34 @@ class SourceWalker(GenericASTTraversal, object):
         # A horrible hack for Python 3.0 .. 3.2
         if 3.0 <= self.version <= 3.2 and len(node) == 2:
             if (node[0][0] == 'LOAD_FAST' and node[0][0].pattr == '__locals__' and
-                node[1][0].type == 'STORE_LOCALS'):
+                node[1][0].kind == 'STORE_LOCALS'):
                 self.prune()
         self.default(node)
 
     def n_assign2(self, node):
         for n in node[-2:]:
             if n[0] == 'unpack':
-                n[0].type = 'unpack_w_parens'
+                n[0].kind = 'unpack_w_parens'
         self.default(node)
 
     def n_assign3(self, node):
         for n in node[-3:]:
             if n[0] == 'unpack':
-                n[0].type = 'unpack_w_parens'
+                n[0].kind = 'unpack_w_parens'
         self.default(node)
 
     def n_except_cond2(self, node):
         if node[-2][0] == 'unpack':
-            node[-2][0].type = 'unpack_w_parens'
+            node[-2][0].kind = 'unpack_w_parens'
         self.default(node)
 
-    def engine(self, entry, startnode):
+    def template_engine(self, entry, startnode):
         """The format template interpetation engine.  See the comment at the
-        beginning of this module for the how we interpret format specifications such as
-        %c, %C, and so on.
+        beginning of this module for the how we interpret format
+        specifications such as %c, %C, and so on.
         """
-        # self.println("----> ", startnode.type, ', ', entry[0])
+
+        # self.println("----> ", startnode.kind, ', ', entry[0])
         fmt = entry[0]
         arg = 1
         i = 0
@@ -1763,24 +1865,23 @@ class SourceWalker(GenericASTTraversal, object):
             if   typ == '%':	self.write('%')
             elif typ == '+':
                 self.line_number += 1
-                self.indentMore()
+                self.indent_more()
             elif typ == '-':
                 self.line_number += 1
-                self.indentLess()
+                self.indent_less()
             elif typ == '|':
                 self.line_number += 1
                 self.write(self.indent)
             # Used mostly on the LHS of an assignment
             # BUILD_TUPLE_n is pretty printed and may take care of other uses.
             elif typ == ',':
-                if (node.type in ('unpack', 'unpack_w_parens') and
+                if (node.kind in ('unpack', 'unpack_w_parens') and
                     node[0].attr == 1):
                     self.write(',')
             elif typ == 'c':
-                if isinstance(entry[arg], int):
-                    entry_node = node[entry[arg]]
-                    self.preorder(entry_node)
-                    arg += 1
+                entry_node = node[entry[arg]]
+                self.preorder(entry_node)
+                arg += 1
             elif typ == 'p':
                 p = self.prec
                 (index, self.prec) = entry[arg]
@@ -1846,8 +1947,8 @@ class SourceWalker(GenericASTTraversal, object):
             key = key[i]
             pass
 
-        if key.type in table:
-            self.engine(table[key.type], node)
+        if key.kind in table:
+            self.template_engine(table[key.kind], node)
             self.prune()
 
     def customize(self, customize):
@@ -1871,7 +1972,7 @@ class SourceWalker(GenericASTTraversal, object):
                         'CALL_FUNCTION_VAR_KW', 'CALL_FUNCTION_KW'):
                 if v == 0:
                     str = '%c(%C' # '%C' is a dummy here ...
-                    p2 = (0, 0, None) # .. because of this
+                    p2 = (0, 0, None) # .. because of the None in this
                 else:
                     str = '%c(%C, '
                     p2 = (1, -2, ', ')
@@ -2066,10 +2167,17 @@ class SourceWalker(GenericASTTraversal, object):
         # assert isinstance(tokens[0], Token)
 
         if isLambda:
+            for t in tokens:
+                if t.kind == 'RETURN_END_IF':
+                    t.kind = 'RETURN_END_IF_LAMBDA'
+                elif t.kind == 'RETURN_VALUE':
+                    t.kind = 'RETURN_VALUE_LAMBDA'
             tokens.append(Token('LAMBDA_MARKER'))
             try:
                 ast = python_parser.parse(self.p, tokens, customize)
-            except (python_parser.ParserError, AssertionError) as e:
+            except python_parser.ParserError, e:
+                raise ParserError(e, tokens)
+            except AssertionError, e:
                 raise ParserError(e, tokens)
             maybe_show_ast(self.showast, ast)
             return ast
@@ -2080,10 +2188,10 @@ class SourceWalker(GenericASTTraversal, object):
         # than fight (with the grammar to not emit "return None").
         if self.hide_internal:
             if len(tokens) >= 2 and not noneInNames:
-                if tokens[-1].type == 'RETURN_VALUE':
+                if tokens[-1].kind in ('RETURN_VALUE', 'RETURN_VALUE_LAMBDA'):
                     # Python 3.4's classes can add a "return None" which is
                     # invalid syntax.
-                    if tokens[-2].type == 'LOAD_CONST':
+                    if tokens[-2].kind == 'LOAD_CONST':
                         if isTopLevel or tokens[-2].pattr is None:
                             del tokens[-2:]
                         else:
@@ -2096,7 +2204,9 @@ class SourceWalker(GenericASTTraversal, object):
         # Build AST from disassembly.
         try:
             ast = python_parser.parse(self.p, tokens, customize)
-        except (python_parser.ParserError, AssertionError) as e:
+        except python_parser.ParserError, e:
+            raise ParserError(e, tokens)
+        except AssertionError, e:
             raise ParserError(e, tokens)
 
         maybe_show_ast(self.showast, ast)
@@ -2125,7 +2235,7 @@ def deparse_code(version, co, out=sys.stdout, showasm=None, showast=False,
     debug_parser = dict(PARSER_DEFAULT_DEBUG)
     if showgrammar:
         debug_parser['reduce'] = showgrammar
-        debug_parser['errorstack'] = True
+        debug_parser['errorstack'] = 'full'
 
     #  Build AST from disassembly.
     linestarts = dict(scanner.opc.findlinestarts(co))
@@ -2174,10 +2284,10 @@ def deparse_code(version, co, out=sys.stdout, showasm=None, showast=False,
 if __name__ == '__main__':
     def deparse_test(co):
         "This is a docstring"
-        sys_version = sys.version_info.major + (sys.version_info.minor / 10.0)
+        sys_version = float(sys.version[0:3])
         deparsed = deparse_code(sys_version, co, showasm='after', showast=True)
         # deparsed = deparse_code(sys_version, co, showasm=None, showast=False,
         #                         showgrammar=True)
         print(deparsed.text)
         return
-    deparse_test(deparse_test.__code__)
+    deparse_test(deparse_test.func_code)

uncompyle6/show.py

@@ -12,7 +12,10 @@ def maybe_show_asm(showasm, tokens):
     :param tokens:  The asm tokens to show.
     """
     if showasm:
-        stream = showasm if hasattr(showasm, 'write') else sys.stdout
+        if hasattr(showasm, 'write'):
+            stream = showasm
+        else:
+            stream = sys.stdout
         for t in tokens:
             stream.write(str(t))
             stream.write('\n')
@@ -29,7 +32,10 @@ def maybe_show_ast(showast, ast):
     :param ast:     The ast to show.
     """
     if showast:
-        stream = showast if hasattr(showast, 'write') else sys.stdout
+        if hasattr(showast, 'write'):
+            stream = showast
+        else:
+            stream = sys.stdout
         stream.write(str(ast))
         stream.write('\n')
 
@@ -48,7 +54,10 @@ def maybe_show_ast_param_default(showast, name, default):
     :param default: The function parameter default.
     """
     if showast:
-        stream = showast if hasattr(showast, 'write') else sys.stdout
+        if hasattr(showast, 'write'):
+            stream = showast
+        else:
+            stream = sys.stdout
         stream.write('\n')
         stream.write('--' + name)
         stream.write('\n')

uncompyle6/verify.py

@@ -1,13 +1,11 @@
 #
 # (C) Copyright 2000-2002 by hartmut Goebel <h.goebel@crazy-compilers.com>
-# (C) Copyright 2015-2016 by Rocky Bernstein
+# (C) Copyright 2015-2017 by Rocky Bernstein
 #
 """
 byte-code verification
 """
 
-from __future__ import print_function
-
 import dis, operator
 
 import uncompyle6
@@ -45,7 +43,7 @@ BIN_OP_FUNCS = {
 'BINARY_OR': operator.or_,
 }
 
-JUMP_OPs = None
+JUMP_OPS = None
 
 # --- exceptions ---
 
@@ -227,8 +225,8 @@ def cmp_code_objects(version, is_pypy, code_obj1, code_obj2,
                 import uncompyle6.scanners.scanner36 as scan
                 scanner = scan.Scanner36()
 
-            global JUMP_OPs
-            JUMP_OPs = list(scan.JUMP_OPs) + ['JUMP_BACK']
+            global JUMP_OPS
+            JUMP_OPS = list(scan.JUMP_OPS) + ['JUMP_BACK']
 
             # use changed Token class
             # We (re)set this here to save exception handling,
@@ -244,18 +242,18 @@ def cmp_code_objects(version, is_pypy, code_obj1, code_obj2,
                 scanner.resetTokenClass() # restore Token class
 
             targets1 = dis.findlabels(code_obj1.co_code)
-            tokens1 = [t for t in tokens1 if t.type != 'COME_FROM']
-            tokens2 = [t for t in tokens2 if t.type != 'COME_FROM']
+            tokens1 = [t for t in tokens1 if t.kind != 'COME_FROM']
+            tokens2 = [t for t in tokens2 if t.kind != 'COME_FROM']
 
             i1 = 0; i2 = 0
             offset_map = {}; check_jumps = {}
             while i1 < len(tokens1):
                 if i2 >= len(tokens2):
                     if len(tokens1) == len(tokens2) + 2 \
-                          and tokens1[-1].type == 'RETURN_VALUE' \
-                          and tokens1[-2].type == 'LOAD_CONST' \
+                          and tokens1[-1].kind == 'RETURN_VALUE' \
+                          and tokens1[-2].kind == 'LOAD_CONST' \
                           and tokens1[-2].pattr is None \
-                          and tokens1[-3].type == 'RETURN_VALUE':
+                          and tokens1[-3].kind == 'RETURN_VALUE':
                         break
                     else:
                         raise CmpErrorCodeLen(name, tokens1, tokens2)
@@ -267,13 +265,13 @@ def cmp_code_objects(version, is_pypy, code_obj1, code_obj2,
                         raise CmpErrorCode(name, tokens1[idx1].offset, tokens1[idx1],
                                    tokens2[idx2], tokens1, tokens2)
 
-                if tokens1[i1].type != tokens2[i2].type:
-                    if tokens1[i1].type == 'LOAD_CONST' == tokens2[i2].type:
+                if tokens1[i1].kind != tokens2[i2].kind:
+                    if tokens1[i1].kind == 'LOAD_CONST' == tokens2[i2].kind:
                         i = 1
-                        while tokens1[i1+i].type == 'LOAD_CONST':
+                        while tokens1[i1+i].kind == 'LOAD_CONST':
                             i += 1
-                        if tokens1[i1+i].type.startswith(('BUILD_TUPLE', 'BUILD_LIST')) \
-                              and i == int(tokens1[i1+i].type.split('_')[-1]):
+                        if tokens1[i1+i].kind.startswith(('BUILD_TUPLE', 'BUILD_LIST')) \
+                              and i == int(tokens1[i1+i].kind.split('_')[-1]):
                             t = tuple([ elem.pattr for elem in tokens1[i1:i1+i] ])
                             if t != tokens2[i2].pattr:
                                 raise CmpErrorCode(name, tokens1[i1].offset, tokens1[i1],
@@ -281,60 +279,60 @@ def cmp_code_objects(version, is_pypy, code_obj1, code_obj2,
                             i1 += i + 1
                             i2 += 1
                             continue
-                        elif i == 2 and tokens1[i1+i].type == 'ROT_TWO' and tokens2[i2+1].type == 'UNPACK_SEQUENCE_2':
+                        elif i == 2 and tokens1[i1+i].kind == 'ROT_TWO' and tokens2[i2+1].kind == 'UNPACK_SEQUENCE_2':
                             i1 += 3
                             i2 += 2
                             continue
-                        elif i == 2 and tokens1[i1+i].type in BIN_OP_FUNCS:
-                            f = BIN_OP_FUNCS[tokens1[i1+i].type]
+                        elif i == 2 and tokens1[i1+i].kind in BIN_OP_FUNCS:
+                            f = BIN_OP_FUNCS[tokens1[i1+i].kind]
                             if f(tokens1[i1].pattr, tokens1[i1+1].pattr) == tokens2[i2].pattr:
                                 i1 += 3
                                 i2 += 1
                                 continue
-                    elif tokens1[i1].type == 'UNARY_NOT':
-                        if tokens2[i2].type == 'POP_JUMP_IF_TRUE':
-                            if tokens1[i1+1].type == 'POP_JUMP_IF_FALSE':
+                    elif tokens1[i1].kind == 'UNARY_NOT':
+                        if tokens2[i2].kind == 'POP_JUMP_IF_TRUE':
+                            if tokens1[i1+1].kind == 'POP_JUMP_IF_FALSE':
                                 i1 += 2
                                 i2 += 1
                                 continue
-                        elif tokens2[i2].type == 'POP_JUMP_IF_FALSE':
-                            if tokens1[i1+1].type == 'POP_JUMP_IF_TRUE':
+                        elif tokens2[i2].kind == 'POP_JUMP_IF_FALSE':
+                            if tokens1[i1+1].kind == 'POP_JUMP_IF_TRUE':
                                 i1 += 2
                                 i2 += 1
                                 continue
-                    elif tokens1[i1].type in ('JUMP_FORWARD', 'JUMP_BACK') \
-                          and tokens1[i1-1].type == 'RETURN_VALUE' \
-                          and tokens2[i2-1].type in ('RETURN_VALUE', 'RETURN_END_IF') \
+                    elif tokens1[i1].kind in ('JUMP_FORWARD', 'JUMP_BACK') \
+                          and tokens1[i1-1].kind == 'RETURN_VALUE' \
+                          and tokens2[i2-1].kind in ('RETURN_VALUE', 'RETURN_END_IF') \
                           and int(tokens1[i1].offset) not in targets1:
                         i1 += 1
                         continue
-                    elif tokens1[i1].type == 'JUMP_FORWARD' and tokens2[i2].type == 'JUMP_BACK' \
-                          and tokens1[i1+1].type == 'JUMP_BACK' and tokens2[i2+1].type == 'JUMP_BACK' \
+                    elif tokens1[i1].kind == 'JUMP_FORWARD' and tokens2[i2].kind == 'JUMP_BACK' \
+                          and tokens1[i1+1].kind == 'JUMP_BACK' and tokens2[i2+1].kind == 'JUMP_BACK' \
                           and int(tokens1[i1].pattr) == int(tokens1[i1].offset) + 3:
                         if int(tokens1[i1].pattr) == int(tokens1[i1+1].offset):
                             i1 += 2
                             i2 += 2
                             continue
-                    elif tokens1[i1].type == 'LOAD_NAME' and tokens2[i2].type == 'LOAD_CONST' \
+                    elif tokens1[i1].kind == 'LOAD_NAME' and tokens2[i2].kind == 'LOAD_CONST' \
                          and tokens1[i1].pattr == 'None' and tokens2[i2].pattr is None:
                         pass
-                    elif tokens1[i1].type == 'LOAD_GLOBAL' and tokens2[i2].type == 'LOAD_NAME' \
+                    elif tokens1[i1].kind == 'LOAD_GLOBAL' and tokens2[i2].kind == 'LOAD_NAME' \
                          and tokens1[i1].pattr == tokens2[i2].pattr:
                         pass
-                    elif tokens1[i1].type == 'LOAD_ASSERT' and tokens2[i2].type == 'LOAD_NAME' \
+                    elif tokens1[i1].kind == 'LOAD_ASSERT' and tokens2[i2].kind == 'LOAD_NAME' \
                          and tokens1[i1].pattr == tokens2[i2].pattr:
                         pass
-                    elif (tokens1[i1].type == 'RETURN_VALUE' and
-                          tokens2[i2].type == 'RETURN_END_IF'):
+                    elif (tokens1[i1].kind == 'RETURN_VALUE' and
+                          tokens2[i2].kind == 'RETURN_END_IF'):
                         pass
-                    elif (tokens1[i1].type == 'BUILD_TUPLE_0' and
+                    elif (tokens1[i1].kind == 'BUILD_TUPLE_0' and
                           tokens2[i2].pattr == ()):
                         pass
                     else:
                         raise CmpErrorCode(name, tokens1[i1].offset, tokens1[i1],
                                            tokens2[i2], tokens1, tokens2)
-                elif tokens1[i1].type in JUMP_OPs and tokens1[i1].pattr != tokens2[i2].pattr:
-                    if tokens1[i1].type == 'JUMP_BACK':
+                elif tokens1[i1].kind in JUMP_OPS and tokens1[i1].pattr != tokens2[i2].pattr:
+                    if tokens1[i1].kind == 'JUMP_BACK':
                         dest1 = int(tokens1[i1].pattr)
                         dest2 = int(tokens2[i2].pattr)
                         if offset_map[dest1] != dest2:
@@ -365,8 +363,10 @@ def cmp_code_objects(version, is_pypy, code_obj1, code_obj2,
         elif member == 'co_flags':
             flags1 = code_obj1.co_flags
             flags2 = code_obj2.co_flags
-            if is_pypy:
+            if is_pypy or version == 2.4:
                 # For PYPY for now we don't care about PYPY_SOURCE_IS_UTF8:
+                # Python 2.4 also sets this flag and I am not sure
+                # where or why
                 flags2 &= ~0x0100  # PYPY_SOURCE_IS_UTF8
             # We also don't care about COROUTINE or GENERATOR for now
             flags1 &= ~0x000000a0
@@ -375,6 +375,8 @@ def cmp_code_objects(version, is_pypy, code_obj1, code_obj2,
                 raise CmpErrorMember(name, 'co_flags',
                                      pretty_flags(flags1),
                                      pretty_flags(flags2))
+
+
         else:
             # all other members must be equal
             if getattr(code_obj1, member) != getattr(code_obj2, member):
@@ -385,28 +387,28 @@ def cmp_code_objects(version, is_pypy, code_obj1, code_obj2,
 class Token(scanner.Token):
     """Token class with changed semantics for 'cmp()'."""
     def __cmp__(self, o):
-        t = self.type # shortcut
-        if t == 'BUILD_TUPLE_0' and o.type == 'LOAD_CONST' and o.pattr == ():
+        t = self.kind # shortcut
+        if t == 'BUILD_TUPLE_0' and o.kind == 'LOAD_CONST' and o.pattr == ():
             return 0
-        if t == 'COME_FROM' == o.type:
+        if t == 'COME_FROM' == o.kind:
             return 0
-        if t == 'PRINT_ITEM_CONT' and o.type == 'PRINT_ITEM':
+        if t == 'PRINT_ITEM_CONT' and o.kind == 'PRINT_ITEM':
             return 0
-        if t == 'RETURN_VALUE' and o.type == 'RETURN_END_IF':
+        if t == 'RETURN_VALUE' and o.kind == 'RETURN_END_IF':
             return 0
-        if t == 'JUMP_IF_FALSE_OR_POP' and o.type == 'POP_JUMP_IF_FALSE':
+        if t == 'JUMP_IF_FALSE_OR_POP' and o.kind == 'POP_JUMP_IF_FALSE':
             return 0
-        if JUMP_OPs and t in JUMP_OPs:
+        if JUMP_OPS and t in JUMP_OPS:
             # ignore offset
-            return t == o.type
-        return (t ==  o.type) or self.pattr ==  o.pattr
+            return t == o.kind
+        return (t ==  o.kind) or self.pattr ==  o.pattr
 
     def __repr__(self):
-        return '%s %s (%s)' % (str(self.type), str(self.attr),
+        return '%s %s (%s)' % (str(self.kind), str(self.attr),
                        repr(self.pattr))
 
     def __str__(self):
-        return '%s\t%-17s %r' % (self.offset, self.type, self.pattr)
+        return '%s\t%-17s %r' % (self.offset, self.kind, self.pattr)
 
 def compare_code_with_srcfile(pyc_filename, src_filename, weak_verify=False):
     """Compare a .pyc with a source code file."""
@@ -418,8 +420,10 @@ def compare_code_with_srcfile(pyc_filename, src_filename, weak_verify=False):
         return msg
     try:
         code_obj2 = load_file(src_filename)
-    except SyntaxError as e:
+    except SyntaxError, e:
         # src_filename can be the first of a group sometimes
+        if version == 2.4:
+            print(pyc_filename)
         return str(e).replace(src_filename, pyc_filename)
     cmp_code_objects(version, is_pypy, code_obj1, code_obj2, ignore_code=weak_verify)
     return None
@@ -438,4 +442,4 @@ if __name__ == '__main__':
     t2 = Token('LOAD_CONST', -421, 'code_object _expandLang', 55)
     print(repr(t1))
     print(repr(t2))
-    print(t1.type ==  t2.type, t1.attr == t2.attr)
+    print(t1.kind ==  t2.kind, t1.attr == t2.attr)

uncompyle6/version.py

@@ -1,3 +1,3 @@
 # This file is suitable for sourcing inside bash as
 # well as importing into Python
-VERSION='2.11.5'
+VERSION='2.13.2'