Get ready for release 2.9.11

by removing a lot of the control-flow labels of 3.3+

.gitignore

@@ -16,3 +16,5 @@
 /unpyc
 __pycache__
 build
+/.venv*
+/.idea

.travis.yml

@@ -3,12 +3,7 @@ language: python
 sudo: false
 
 python:
-  - '3.5'
+  - '2.7' # this is a cheat here because travis doesn't do 2.4-2.6
-  - '2.7.12'
-  - '2.6'
-  - '3.3'
-  - '3.4'
-  - '3.2'
 
 install:
 - pip install -r requirements.txt

ChangeLog

@@ -1,10 +1,636 @@
+2017-05-06  rocky <rb@dustyfeet.com>
+
+	* uncompyle6/version.py: Get ready for release 2.9.11
+
+2017-05-06  rocky <rb@dustyfeet.com>
+
+	* test/Makefile, uncompyle6/scanners/scanner2.py,
+	uncompyle6/scanners/scanner3.py, uncompyle6/semantics/fragments.py: 
+	Sync with master
+
+2017-05-06  rocky <rb@dustyfeet.com>
+
+	* : commit 4a4782290490187ac2fcaaecd3ca808f933722b2 Author: rocky
+	<rb@dustyfeet.com> Date:   Sat May 6 05:25:56 2017 -0400
+
+2017-05-05  rocky <rb@dustyfeet.com>
+
+	* uncompyle6/parsers/parse32.py, uncompyle6/scanners/scanner3.py: 
+	Improve Python 3.2 decompilation ...  by removing a lot of the control-flow labels of 3.3+
+
+2017-05-05  rocky <rb@dustyfeet.com>
+
+	* .travis.yml: Try CI testing on Python 3.6
+
+2017-05-02  rocky <rb@dustyfeet.com>
+
+	* test/simple_source/bug35/01_map_unpack.py, uncompyle6/parser.py,
+	uncompyle6/parsers/parse3.py, uncompyle6/parsers/parse35.py,
+	uncompyle6/semantics/pysource.py: Bang more on BUIlD_MAP_UNPACK there are still bugs. Note: {**{'x': 1}, **{'y': 2}} and {{'x': 1}, **{'y': 2}} generate the same Python 3.5+ bytecode.
+
+2017-05-02  rocky <rb@dustyfeet.com>
+
+	* test/simple_source/bug35/01_map_unpack.py, uncompyle6/parser.py,
+	uncompyle6/parsers/parse3.py, uncompyle6/parsers/parse35.py: 
+	BUILD_MAP_UNPACK'ing of dictionaries in 3.5
+
+2017-05-01  rocky <rb@dustyfeet.com>
+
+	* uncompyle6/semantics/pysource.py: Remove extra unpack *. Issue #98
+
+2017-04-29  R. Bernstein <rocky@users.noreply.github.com>
+
+	* HISTORY.md: Update HISTORY.md
+
+2017-04-29  rocky <rb@dustyfeet.com>
+
+	* test/simple_source/bug35/01_map_unpack.py,
+	uncompyle6/parsers/parse35.py, uncompyle6/semantics/pysource.py: 
+	Handle BUILD_MAP_UNPACK in a build_list
+
+2017-04-27  rocky <rb@dustyfeet.com>
+
+	* uncompyle6/semantics/pysource.py: A hacky way to get
+	CALL_FUNCTION_EX_KW to work.
+
+2017-04-26  rocky <rb@dustyfeet.com>
+
+	* uncompyle6/semantics/pysource.py: remove debug code
+
+2017-04-25  rocky <rb@dustyfeet.com>
+
+	* test/simple_source/bug36/01_call_function.py,
+	uncompyle6/parsers/parse36.py, uncompyle6/scanners/scanner36.py,
+	uncompyle6/semantics/pysource.py: Python 3.6 CALL_FUNCTION_EX first
+	attempt
+
+2017-04-22  rocky <rb@dustyfeet.com>
+
+	* uncompyle6/parser.py, uncompyle6/parsers/parse34.py: Reduse scope
+	of LOAD_ASSERT as expr to 3.4+
+
+2017-04-22  rocky <rb@dustyfeet.com>
+
+	* uncompyle6/parser.py, uncompyle6/verify.py: LOAD_ASSERT can also
+	be an expr This may have the undesirable property that assert statements might
+	get tagged with equivalant low-level Python code that uses "raise
+	AssertionError", but so be it.  Fixes #103
+
+2017-04-22  R. Bernstein <rocky@users.noreply.github.com>
+
+	* HISTORY.md: Update HISTORY.md
+
+2017-04-22  R. Bernstein <rocky@users.noreply.github.com>
+
+	* HISTORY.md: Update HISTORY.md
+
+2017-04-22  rocky <rb@dustyfeet.com>
+
+	* HISTORY.md: History keeps gettting amended
+
+2017-04-22  rocky <rb@dustyfeet.com>
+
+	* README.rst: Document Python 3.x status
+
+2017-04-22  rocky <rb@dustyfeet.com>
+
+	* test/simple_source/bug35/03_async_await.py,
+	uncompyle6/parsers/parse35.py, uncompyle6/semantics/pysource.py: Add
+	await expr Fixes #111
+
+2017-04-22  rocky <rb@dustyfeet.com>
+
+	* : Update test
+
+2017-04-22  rocky <rb@dustyfeet.com>
+
+	* test/simple_source/bug33/02_pos_args.py,
+	uncompyle6/parsers/parse3.py, uncompyle6/semantics/make_function.py: 
+	3.3+ bug in handling single kwarg after * Towards fixing issue #110
+
+2017-04-20  rocky <rb@dustyfeet.com>
+
+	* test/simple_source/bug35/02_async_for.py,
+	uncompyle6/parsers/parse35.py: Add async for with pass statement Fixes #109
+
+2017-04-19  rocky <rb@dustyfeet.com>
+
+	* test/simple_source/bug35/03_while-if-break.py,
+	uncompyle6/parsers/parse3.py: 3.5 ifelsestmtl grammar bug.  Fixes #108
+
+2017-04-18  rocky <rb@dustyfeet.com>
+
+	* test/simple_source/bug35/03_async_await.py,
+	uncompyle6/parsers/parse35.py: Expand await stmt handling Fixes #107
+
+2017-04-18  rocky <rb@dustyfeet.com>
+
+	* test/simple_source/bug33/01_delete_deref.py,
+	uncompyle6/parsers/parse32.py, uncompyle6/semantics/pysource.py: Add
+	DELETE_DEREF grammar rule Fixes Issue #106
+
+2017-04-17  rocky <rb@dustyfeet.com>
+
+	* test/simple_source/bug36/01_extended_arg.py,
+	test/simple_source/bug36/01_if_file.py: Rename test case to
+	something more appropriate
+
+2017-04-17  rocky <rb@dustyfeet.com>
+
+	* test/simple_source/bug36/01_if_file.py: Fix botched test case Thanks to Zm908 for pointing this out
+
+2017-04-16  rocky <rb@dustyfeet.com>
+
+	* uncompyle6/parsers/parse3.py: Comment on what's up with last
+	change
+
+2017-04-16  rocky <rb@dustyfeet.com>
+
+	* test/simple_source/bug22/03_if1.py,
+	test/simple_source/bug31/02_ifelse_comprehension.py,
+	uncompyle6/parsers/parse3.py: Python 3.x ifelse in comprehension Fixes Issue #91
+
+2017-04-16  rocky <rb@dustyfeet.com>
+
+	* : Add 2.7 complex test
+
+2017-04-15  rocky <rb@dustyfeet.com>
+
+	* test/simple_source/bug35/01_map_unpack.py,
+	uncompyle6/semantics/pysource.py: Correct bug in 3.5+ build_list
+	with UNPACK
+
+2017-04-15  R. Bernstein <rocky@users.noreply.github.com>
+
+	* HOW-TO-REPORT-A-BUG.md: Update HOW-TO-REPORT-A-BUG.md
+
+2017-04-15  R. Bernstein <rocky@users.noreply.github.com>
+
+	* HOW-TO-REPORT-A-BUG.md: Update HOW-TO-REPORT-A-BUG.md
+
+2017-04-15  rocky <rb@dustyfeet.com>
+
+	* test/simple_source/bug36/01_if_file.py,
+	uncompyle6/parsers/parse36.py: 3.6 generates Wonky EXTENDED_ARG in
+	expression Fixes Issue #102
+
+2017-04-15  rocky <rb@dustyfeet.com>
+
+	* HOW-TO-REPORT-A-BUG.md, MANIFEST.in: Add how to report a bug Add test case for ... if 1 else ...
+
+2017-04-14  rocky <rb@dustyfeet.com>
+
+	* test/simple_source/bug35/01_map_unpack.py,
+	uncompyle6/parsers/parse3.py, uncompyle6/parsers/parse35.py,
+	uncompyle6/semantics/pysource.py: Python 3.5+ BUILD_UNMAP_PACK rules Towards addressing Issue #98
+
+2017-04-14  rocky <rb@dustyfeet.com>
+
+	* uncompyle6/scanners/scanner3.py: Reduce adding RETURN_END_IF in
+	3.5+ The whole control flow determination has to be redone in a less
+	haphazard way using real flow-control analysis. Hopefully that's on
+	the way.  In the meantime we have this hack.
+
+2017-04-14  rocky <rb@dustyfeet.com>
+
+	* : commit 7e8f7ba67431725fceec08344934c929a517efc5 Author: rocky
+	<rb@dustyfeet.com> Date:   Fri Apr 14 05:42:44 2017 -0400
+
+2017-04-14  rocky <rb@dustyfeet.com>
+
+	* test/simple_source/bug27+/03_if_1_else.py,
+	test/simple_source/bug27+/03_if_true_else.py: Better names for a
+	test
+
+2017-04-13  rocky <rb@dustyfeet.com>
+
+	* test/simple_source/bug27+/03_if_true_else.py,
+	uncompyle6/parser.py, uncompyle6/parsers/parse3.py,
+	uncompyle6/semantics/consts.py: Add if1else. Fixes #101
+
+2017-04-13  rocky <rb@dustyfeet.com>
+
+	* uncompyle6/parsers/parse3.py: In 3.x come_from should include
+	COME_FROM_EXCEPT
+
+2017-04-13  rocky <rb@dustyfeet.com>
+
+	* uncompyle6/parsers/parse35.py: Towards fixing issue #92
+
+2017-04-13  rocky <rb@dustyfeet.com>
+
+	* uncompyle6/parsers/parse23.py, uncompyle6/semantics/pysource.py: 
+	Add Python 2.3 rule for "if 1: ..." Fully fixes #97 for Python 2.3. Python 2.4 was fixed in a previous
+	commit.
+
+2017-04-12  rocky <rb@dustyfeet.com>
+
+	* uncompyle6/parsers/parse3.py,
+	uncompyle6/semantics/make_function.py: annotate args type need to be
+	expr's not constants
+
+2017-04-12  rocky <rb@dustyfeet.com>
+
+	* uncompyle6/parsers/parse24.py: Handle Python 2.4 "if 1...."
+
+2017-04-11  rocky <rb@dustyfeet.com>
+
+	* test/simple_source/bug31/04_def_annotate.py,
+	uncompyle6/semantics/fragments.py,
+	uncompyle6/semantics/make_function.py: Bang on 3.x annotations
+
+2017-04-11  rocky <rb@dustyfeet.com>
+
+	* test/simple_source/bug31/04_def_annotate.py,
+	uncompyle6/parsers/parse3.py, uncompyle6/semantics/pysource.py: 
+	Towards fixing annotated decorator functions...  and annotate functions
+
+2017-04-10  rocky <rb@dustyfeet.com>
+
+	* uncompyle6/parsers/parse2.py, uncompyle6/scanners/scanner27.py,
+	uncompyle6/semantics/check_ast.py, uncompyle6/semantics/pysource.py: 
+	Misc bugs parse2.py: restore accidently-removed while1stmt rule scanner27.py:
+	grammar typo check_ast: add while1else to list of looping constructs
+	pysource.py: CALL_FUNCTION_VAR_KW_ARGS with positional args rule is
+	different?
+
+2017-04-10  rocky <rb@dustyfeet.com>
+
+	* test/simple_source/stmts/02_while1else.py,
+	uncompyle6/parsers/parse2.py, uncompyle6/parsers/parse3.py,
+	uncompyle6/parsers/parse35.py: Add more while1else grammar rules Towards addressing issue #93
+
+2017-04-10  rocky <rb@dustyfeet.com>
+
+	* : commit b9703cf6b41138b717c282fc791c08d807692b07 Author: rocky
+	<rb@dustyfeet.com> Date:   Sun Apr 9 06:58:41 2017 -0400
+
+2017-04-09  rocky <rb@dustyfeet.com>
+
+	* test/simple_source/def/10_kw+pos_args-bug.py,
+	uncompyle6/parsers/parse3.py, uncompyle6/semantics/pysource.py: 
+	Another Python 3.5 FUNCTION_VAR bug Fixes #94
+
+2017-04-09  rocky <rb@dustyfeet.com>
+
+	* : commit 4199bc7f617e387fb03fc06939cd17366dc15c5e Author: rocky
+	<rb@dustyfeet.com> Date:   Sun Apr 9 05:30:45 2017 -0400
+
+2017-04-03  rocky <rb@dustyfeet.com>
+
+	* : commit 6773a66b99d07e48290a77dbbbe3c71cc39c31ba Author: rocky
+	<rb@dustyfeet.com> Date:   Mon Apr 3 06:53:12 2017 -0400
+
+2017-03-27  rocky <rb@dustyfeet.com>
+
+	* : commit a91cd716670be09d3cef34e1bb36a67f96f91712 Author: rocky
+	<rb@dustyfeet.com> Date:   Mon Mar 27 07:08:59 2017 -0400
+
+2017-03-19  rocky <rb@dustyfeet.com>
+
+	* __pkginfo__.py: Use more-recent xdis
+
+2017-03-15  rocky <rb@dustyfeet.com>
+
+	* HISTORY.md, test/simple_source/bug33/01_if_try_except.py: grammar
+	typo and add another test
+
+2017-03-12  rocky <rb@dustyfeet.com>
+
+	* uncompyle6/scanners/scanner3.py: Python 3.0 doesn't have
+	POP_JUMP_IF...
+
+2017-03-12  rocky <rb@dustyfeet.com>
+
+	* README.rst: Note problem in handling pathologically long lists
+
+2017-03-07  rocky <rb@dustyfeet.com>
+
+	* uncompyle6/scanners/scanner3.py: Small cleanup - remove
+	POP_JUMP_TF
+
+2017-03-05  rocky <rb@dustyfeet.com>
+
+	* pytest/test_grammar.py, uncompyle6/parsers/parse3.py,
+	uncompyle6/parsers/parse33.py, uncompyle6/scanners/scanner3.py: More
+	accurate ranges of try blocks in 3.x
+
+2017-03-05  rocky <rb@dustyfeet.com>
+
+	* test/simple_source/bug33/01_try_except.py: More accurate ranges of
+	try blocks in 3.x
+
+2017-03-04  R. Bernstein <rocky@users.noreply.github.com>
+
+	* : Merge pull request #84 from
+	moagstar/property_based_test_function_call Property based test function call
+
+2017-03-04  rocky <rb@dustyfeet.com>
+
+	* README.rst: README updates for 3.5 and 1.5
+
+2017-03-04  rocky <rb@dustyfeet.com>
+
+	* test/simple_source/bug32/01_named_and_kwargs.py,
+	uncompyle6/parsers/parse3.py: Bug found by hypothesis in creating
+	function calls
+
+2017-03-04  Daniel Bradburn <moagstar@gmail.com>
+
+	* pytest/test_function_call.py: marked all function call tests as
+	failing until they pass across all python versions
+
+2017-03-04  Daniel Bradburn <moagstar@gmail.com>
+
+	* pytest/test_function_call.py: added minimal examples for various
+	function call opcodes
+
+2017-03-04  Daniel Bradburn <moagstar@gmail.com>
+
+	* pytest/test_function_call.py: added property based test for
+	verifying uncompylation of function calls. A number of minimal
+	examples for the various function call opcodes have been generated
+	with the majority marked as expected failure until python 3.6 opcode
+	support is complete. I'm hoping this will make it easier to figure
+	out what needs to be done to support the new opcodes and changed
+	semntics for function calls
+
+2017-03-03  Daniel Bradburn <moagstar@gmail.com>
+
+	* pytest/test_function_call.py: reduced errors when generating
+	function call instances
+
+2017-03-03  Daniel Bradburn <moagstar@gmail.com>
+
+	* pytest/test_function_call.py: added test file for function calls
+
+2017-03-03  Daniel Bradburn <moagstar@gmail.com>
+
+	* .gitignore: added .idea to gitignore
+
+2017-03-03  Daniel Bradburn <moagstar@gmail.com>
+
+	* .gitignore: added .venv to gitignore
+
+2017-03-01  rocky <rb@dustyfeet.com>
+
+	* : commit 160ec0d9cc5fe347f6e8bdb69515a28c76cfb368 Author: rocky
+	<rb@dustyfeet.com> Date:   Wed Mar 1 05:50:31 2017 -0500
+
+2017-02-28  rocky <rb@dustyfeet.com>
+
+	* README.rst, uncompyle6/parser.py, uncompyle6/parsers/parse26.py: 
+	Python 2.6 a == b or c == d == 3 grammar bug
+
+2017-02-28  rocky <rb@dustyfeet.com>
+
+	* : 2.6 a == b or x == y == z bug
+
+2017-02-28  rocky <rb@dustyfeet.com>
+
+	* test/simple_source/bug26/03_double_equals.py,
+	uncompyle6/semantics/consts.py: Predidence of cmp_list: x == y == z The x, y, z should not have parenthesis around pairs of them (x ==
+	y) or (y == z)
+
+2017-02-28  rocky <rb@dustyfeet.com>
+
+	* uncompyle6/parser.py, uncompyle6/parsers/parse27.py: Python 2.7
+	check jump targets of "and"
+
+2017-02-25  rocky <rb@dustyfeet.com>
+
+	* : commit 1e3ea60055027dfd3f098661ac4f5979c5c48f7e Author: rocky
+	<rb@dustyfeet.com> Date:   Sat Feb 25 20:18:03 2017 -0500
+
+2017-02-25  rocky <rb@dustyfeet.com>
+
+	* uncompyle6/parser.py: Python <= 2.6 grammar fixes
+
+2017-02-25  rocky <rb@dustyfeet.com>
+
+	* : commit 2fbbc728b10f0d3a754165708584bd80d33bc7f9 Author: rocky
+	<rb@dustyfeet.com> Date:   Sat Feb 25 04:45:10 2017 -0500
+
+2017-02-24  rocky <rb@dustyfeet.com>
+
+	* test/simple_source/bug25/03_if_for.py,
+	uncompyle6/parsers/parse26.py: Python 2.6 control flow bug with
+	added COME_FROM
+
+2017-02-22  rocky <rb@dustyfeet.com>
+
+	* test/simple_source/bug25/02_try_else.py,
+	uncompyle6/parsers/parse25.py: Python 2.5 wasn't handling tryelse
+	properly
+
+2017-02-22  rocky <rb@dustyfeet.com>
+
+	* test/Makefile, test/simple_source/bug25/02_try_else.py,
+	uncompyle6/parsers/parse25.py: Python 2.5 was missing try else stmt
+
+2017-02-22  rocky <rb@dustyfeet.com>
+
+	* : commit b043f6bafc9b9ae26e64dc0f1441d7abae894c37 Author: rocky
+	<rb@dustyfeet.com> Date:   Mon Feb 20 09:22:01 2017 -0500
+
+2017-02-20  rocky <rb@dustyfeet.com>
+
+	* test/simple_source/bug33/02_while1.py: Add test for last while1
+	bug fix
+
+2017-02-20  rocky <rb@dustyfeet.com>
+
+	* uncompyle6/parsers/parse3.py, uncompyle6/parsers/parse35.py: 
+	Python 3.x needs more "while 1" grammar rules
+
+2017-02-20  rocky <rb@dustyfeet.com>
+
+	* uncompyle6/parsers/parse3.py, uncompyle6/parsers/parse35.py,
+	uncompyle6/scanners/scanner3.py: Some Python 3.4 bugss fixed by
+	using 3.5 rules
+
+2017-02-20  rocky <rb@dustyfeet.com>
+
+	* test/simple_source/exception/02_try_finally.py,
+	uncompyle6/parsers/parse3.py, uncompyle6/scanners/scanner3.py: More
+	COME_FROM's in Python 3...  Need this to find boundaries of simple if better
+
+2017-02-19  rocky <rb@dustyfeet.com>
+
+	* uncompyle6/parsers/parse26.py: Marginally better for Python 2.6
+	but...  control flow is still wrong.
+
+2017-02-10  rocky <rb@dustyfeet.com>
+
+	* : commit f9d47abb2be7c3839df06c0ed69d3d513694af4e Author: rocky
+	<rb@dustyfeet.com> Date:   Fri Feb 10 02:07:04 2017 -0500
+
+2017-02-10  rocky <rb@dustyfeet.com>
+
+	* test/simple_source/bug22/01_ops.py, test/test_pythonlib.py: Beef
+	up grammar coverage
+
+2017-02-10  rocky <rb@dustyfeet.com>
+
+	* test/Makefile: Group coverage Makefile targets
+
+2017-01-29  rocky <rb@dustyfeet.com>
+
+	* test/Makefile, test/simple_source/bug22/01_ops.py,
+	uncompyle6/parsers/parse25.py, uncompyle6/semantics/consts.py,
+	uncompyle6/semantics/pysource.py: Changes based on grammar coverage
+	info
+
+2017-01-29  rocky <rb@dustyfeet.com>
+
+	* test/Makefile, test/simple_source/bug22/01_ops.py,
+	uncompyle6/parsers/parse25.py, uncompyle6/semantics/consts.py,
+	uncompyle6/semantics/pysource.py: Changes based on coverage
+	information
+
+2017-01-29  rocky <rb@dustyfeet.com>
+
+	* : commit 9348411056cbe809e07c4ef341effa17bca90e2f Merge: 3dc766d
+	e71dd01 Author: R. Bernstein <rocky@users.noreply.github.com> Date:
+	Sun Jan 29 21:54:45 2017 -0500
+
+2017-01-29  rocky <rb@dustyfeet.com>
+
+	* test/Makefile, test/simple_source/bug22/01_ops.py,
+	test/test_pyenvlib.py, test/test_pythonlib.py,
+	uncompyle6/semantics/consts.py: Simplfy getting coverage consts.py: notes on versions use which ops
+
+2017-01-29  rocky <rb@dustyfeet.com>
+
+	* test/simple_source/bug22/01_ops.py, test/test_pyenvlib.py: Add
+	--coverage to test_pyenvlib and ...  improve grammar coverage on 2.7
+
+2017-01-29  rocky <rb@dustyfeet.com>
+
+	* : commit 3dc766d0a9537842470c7b4f79e8ccb3d5a46843 Author: rocky
+	<rb@dustyfeet.com> Date:   Sun Jan 29 07:34:49 2017 -0500
+
+2017-01-29  rocky <rb@dustyfeet.com>
+
+	* test/test_pythonlib.py: Add --coverage option. WOOT!
+
+2017-01-27  rocky <rb@dustyfeet.com>
+
+	* __pkginfo__.py: Bump min spark_parser version
+
+2017-01-24  rocky <rb@dustyfeet.com>
+
+	* uncompyle6/scanners/scanner2.py, uncompyle6/semantics/consts.py,
+	uncompyle6/semantics/pysource.py: More 2.6, 2.7 control flow Todo more COME_FROMs but now need to check targets better. In some
+	cases we're relying on grammar ambiguity to work out right and in
+	2.7 it doesn't
+
+2017-01-24  rocky <rb@dustyfeet.com>
+
+	* uncompyle6/parsers/parse2.py, uncompyle6/parsers/parse27.py,
+	uncompyle6/scanners/scanner2.py, uncompyle6/semantics/consts.py,
+	uncompyle6/semantics/pysource.py: More 2.6, 2.7 control-flow bugs Wasn't limiting exception clause to try finally. Probably still has
+	bugs in try-finally nesting Add another 2.6/2.7 COME_FROM to try to limit if/end scope better
+
+2017-01-23  rocky <rb@dustyfeet.com>
+
+	* uncompyle6/scanners/scanner2.py,
+	uncompyle6/scanners/scanner26.py, uncompyle6/verify.py: Improve
+	Python 2.6 & 2.7 verification
+
+2017-01-22  rocky <rb@dustyfeet.com>
+
+	* uncompyle6/parsers/parse30.py, uncompyle6/verify.py: Fix up Python
+	3.0 handling
+
+2017-01-21  rocky <rb@dustyfeet.com>
+
+	* : commit 545a46dffaa0fe2246dd7cc1b560c58db525c2b4 Author: rocky
+	<rb@dustyfeet.com> Date:   Sat Jan 21 06:24:31 2017 -0500
+
+2017-01-20  rocky <rb@dustyfeet.com>
+
+	* uncompyle6/parsers/parse3.py, uncompyle6/scanners/scanner3.py,
+	uncompyle6/semantics/pysource.py: Handle BUILD_CONST_KEY_MAP as a
+	varargs custom rules with BUILD_CONST_KEY_MAP are pinned to the specific
+	number of args seen.
+
+2017-01-19  R. Bernstein <rocky@users.noreply.github.com>
+
+	* : Merge pull request #81 from moagstar/BUILD_CONST_KEY_MAP fixed bug with BUILD_CONST_KEY_MAP
+
+2017-01-19  R. Bernstein <rocky@users.noreply.github.com>
+
+	* : Merge pull request #80 from moagstar/BUILD_CONST_KEY_MAP Build const key map
+
+2017-01-18  Daniel Bradburn <moagstar@gmail.com>
+
+	* uncompyle6/semantics/pysource.py: added generation of dict display
+	from BUILD_CONST_KEY_MAP
+
+2017-01-18  Daniel Bradburn <moagstar@gmail.com>
+
+	* pytest/test_build_const_key_map.py: fixed typo
+
+2017-01-18  Daniel Bradburn <moagstar@gmail.com>
+
+	* pytest/test_build_const_key_map.py: added some more test cases for
+	BUILD_CONST_KEY_MAP
+
+2017-01-17  Daniel Bradburn <moagstar@gmail.com>
+
+	* pytest/test_build_const_key_map.py: simplified test cases for
+	test_build_const_key_map
+
+2017-01-17  Daniel Bradburn <moagstar@gmail.com>
+
+	* pytest/test_build_const_key_map.py, pytest/validate.py: added
+	validation code for checking decompilation of an expression
+
+2017-01-15  rocky <rb@dustyfeet.com>
+
+	* uncompyle6/parsers/parse3.py, uncompyle6/scanners/scanner3.py: 
+	Handle 3.6 BUILD_CONST_KEYMAP
+
+2017-01-15  rocky <rb@dustyfeet.com>
+
+	* uncompyle6/scanners/scanner2.py: Python 2.1 doesn't have FOR_ITER
+	or GET_ITER...  adjust locgic for this fact
+
+2017-01-12  rocky <rb@dustyfeet.com>
+
+	* uncompyle6/__init__.py: sys.recursionlimit is optional, not
+	essential
+
 2017-01-11  rocky <rb@dustyfeet.com>
 
-	* uncompyle6/version.py: Get ready for release 2.10.9
+	* NEWS, uncompyle6/version.py: Get ready for release 2.9.9
+
+2017-01-11  rocky <rb@dustyfeet.com>
+
+	* : commit b131c20e99514d3a969a51e841d3a823017f1beb Author: rocky
+	<rb@dustyfeet.com> Date:   Wed Jan 11 21:32:26 2017 -0500
+
+2017-01-11  rocky <rb@dustyfeet.com>
+
+	* ChangeLog, NEWS: Get ready for release 2.10.9
+
+2017-01-11  rocky <rb@dustyfeet.com>
+
+	* uncompyle6/parsers/parse3.py, uncompyle6/scanner.py,
+	uncompyle6/semantics/make_function.py, uncompyle6/version.py: Merge
+	changes ...  * str() in Python 2.4 doesn't detect unicode.  * index() doesn't work on tuples * ifelse change
+
+2017-01-11  rocky <rb@dustyfeet.com>
+
+	* : commit 7ece296f7638e71fad1117b940f7ffddbe095b1f Merge: 78a5b62
+	5035d54 Author: R. Bernstein <rocky@users.noreply.github.com> Date:
+	Wed Jan 11 07:10:23 2017 -0500
 
 2017-01-11  R. Bernstein <rocky@users.noreply.github.com>
 
-	* : Merge pull request #79 from rocky/revert-78-patch-1 Revert "fix bug : not generate all files when use "-ro""
+	* uncompyle6/main.py: Revert "fix bug : not generate all files when
+	use "-ro""
 
 2017-01-11  R. Bernstein <rocky@users.noreply.github.com>
 
@@ -103,6 +729,16 @@
 	* uncompyle6/parsers/parse35.py, uncompyle6/scanners/scanner3.py: 
 	Python 3.5 continue detection bug
 
+2017-01-02  rocky <rb@dustyfeet.com>
+
+	* uncompyle6/verify.py: 2.4 verify hacks
+
+2017-01-02  rocky <rb@dustyfeet.com>
+
+	* : commit a7d93e88b4e0dfd6876a7a31bd201a0e40f24bea Merge: 9891494
+	136f42a Author: rocky <rb@dustyfeet.com> Date:   Mon Jan 2 05:39:13
+	2017 -0500
+
 2017-01-01  rocky <rb@dustyfeet.com>
 
 	* uncompyle6/scanners/scanner3.py: add come_from for setup_finally
@@ -117,6 +753,14 @@
 
 	* README.rst: Note how to verify correctness ...  with --verify, --weak-verify and cross checking with pycdc
 
+2016-12-31  rocky <rb@dustyfeet.com>
+
+	* uncompyle6/version.py: We are version 2.9.9
+
+2016-12-31  rocky <rb@dustyfeet.com>
+
+	* uncompyle6/main.py: 2.7->2.4 conversion
+
 2016-12-31  rocky <rb@dustyfeet.com>
 
 	* ChangeLog, NEWS, uncompyle6/version.py: Get ready for release
@@ -126,6 +770,13 @@
 
 	* uncompyle6/parsers/parse26.py: 2.x list_if may have a THEN in it
 
+2016-12-31  rocky <rb@dustyfeet.com>
+
+	* test/Makefile, uncompyle6/main.py, uncompyle6/parsers/parse26.py,
+	uncompyle6/scanners/scanner2.py, uncompyle6/scanners/scanner26.py,
+	uncompyle6/scanners/scanner3.py, uncompyle6/semantics/pysource.py,
+	uncompyle6/verify.py: Merge master branche Handle 2.2 list_if
+
 2016-12-31  rocky <rb@dustyfeet.com>
 
 	* uncompyle6/scanners/scanner3.py: Towards fixing a Python 3.3
@@ -154,10 +805,9 @@
 
 	* : Merge pull request #73 from rocky/then-crap Add THEN token to improve Python 2.2-2.6 control flow detection
 
-2016-12-28  rocky <rb@dustyfeet.com>
+2016-12-29  R. Bernstein <rocky@users.noreply.github.com>
 
-	* uncompyle6/parsers/parse3.py, uncompyle6/scanners/tok.py: Misc
+	* : Merge pull request #72 from rocky/master THEN psuedo-ops for Python 2.x
-	bugs
 
 2016-12-28  rocky <rb@dustyfeet.com>
 
@@ -184,8 +834,7 @@
 
 2016-12-27  rocky <rb@dustyfeet.com>
 
-	* uncompyle6/scanners/scanner2.py: Make 2.6 and 2.7 ingest more
+	* : Merge commit '9b1dd0f' into python-2.4
-	alike
 
 2016-12-26  rocky <rb@dustyfeet.com>
 
@@ -205,6 +854,11 @@
 	* uncompyle6/parsers/parse25.py: fix bug in using python2 AST rules
 	in python 2.5
 
+2016-12-26  rocky <rb@dustyfeet.com>
+
+	* uncompyle6/parsers/parse25.py: Bug in using python2 ast checking
+	in python 2.5
+
 2016-12-26  rocky <rb@dustyfeet.com>
 
 	* : commit f1a947f106b231fb1480ba301b15e3ceaf78c94f Author: rocky
@@ -217,15 +871,19 @@
 	uncompyle6/verify.py: Scanner call fixes. NAME_MODULE removal for
 	<=2.4
 
-2016-12-24  rocky <rb@dustyfeet.com>
+2016-12-25  rocky <rb@dustyfeet.com>
 
-	* uncompyle6/parsers/astnode.py, uncompyle6/parsers/parse2.py,
+	* uncompyle6/main.py, uncompyle6/scanners/scanner15.py,
-	uncompyle6/parsers/parse26.py, uncompyle6/parsers/parse3.py,
+	uncompyle6/scanners/scanner21.py, uncompyle6/scanners/scanner22.py,
-	uncompyle6/parsers/parse36.py, uncompyle6/scanners/scanner15.py,
+	uncompyle6/scanners/scanner23.py, uncompyle6/scanners/scanner24.py,
-	uncompyle6/scanners/scanner2.py, uncompyle6/scanners/scanner21.py,
+	uncompyle6/semantics/pysource.py, uncompyle6/verify.py: Removing
-	uncompyle6/scanners/scanner22.py,
+	NAME_MODULE, lint and bug fixes scanner*.py: show_asm param is optional verify.py: call correct
-	uncompyle6/semantics/fragments.py, uncompyle6/semantics/pysource.py: 
+	scanners main.py, verify.py: Use older Python print statements
-	Lint
+
+2016-12-25  rocky <rb@dustyfeet.com>
+
+	* : commit e3f4beeb74e33d5b404094765cc63040f62a0b41 Author: rocky
+	<rb@dustyfeet.com> Date:   Sat Dec 24 07:45:02 2016 -0500
 
 2016-12-24  rocky <rb@dustyfeet.com>
 
@@ -246,22 +904,34 @@
 
 2016-12-18  rocky <rb@dustyfeet.com>
 
-	* pytest/.gitignore, test/simple_source/bug25/02_try_else.py,
+	* : commit c7c0a989829a9a625333665516387c1177c611c2 Author: rocky
-	uncompyle6/parsers/parse25.py, uncompyle6/parsers/parse26.py: Python
+	<rb@dustyfeet.com> Date:   Sun Dec 18 00:56:07 2016 -0500
-	2.5 mistaken try/else
 
 2016-12-17  rocky <rb@dustyfeet.com>
 
 	* uncompyle6/scanners/scanner2.py, uncompyle6/scanners/scanner25.py: 
 	show-asm on python2.5 is optional make scanner2 look a little more like scanner3
 
+2016-12-17  rocky <rb@dustyfeet.com>
+
+	* uncompyle6/scanners/scanner2.py, uncompyle6/scanners/scanner25.py: 
+	show-asm on python2.5 is optional Make scanner2 a little more like scanner3.
+
+2016-12-17  rocky <rb@dustyfeet.com>
+
+	* uncompyle6/parser.py, uncompyle6/scanner.py,
+	uncompyle6/scanners/scanner2.py, uncompyle6/scanners/scanner3.py,
+	uncompyle6/semantics/fragments.py: Python 2.6/2.7 tolerance in
+	Python 2.4 branch
+
 2016-12-16  rocky <rb@dustyfeet.com>
 
 	* NEWS: Release 2.9.8 news
 
 2016-12-16  rocky <rb@dustyfeet.com>
 
-	* __pkginfo__.py, uncompyle6/version.py: Get ready for release 2.9.8
+	* : commit 13d5cd1a588b7f4f2c233c436ce6b0b39db9950e Author: rocky
+	<rb@dustyfeet.com> Date:   Fri Dec 16 22:42:46 2016 -0500
 
 2016-12-16  rocky <rb@dustyfeet.com>
 
@@ -329,15 +999,12 @@
 
 2016-12-04  rocky <rb@dustyfeet.com>
 
-	* ChangeLog, NEWS, uncompyle6/main.py, uncompyle6/parser.py,
+	* ChangeLog: Get ready for release 2.9.7
-	uncompyle6/parsers/parse26.py, uncompyle6/parsers/parse3.py,
+
-	uncompyle6/parsers/parse34.py, uncompyle6/parsers/parse35.py,
+2016-12-04  rocky <rb@dustyfeet.com>
-	uncompyle6/scanner.py, uncompyle6/scanners/scanner2.py,
+
-	uncompyle6/scanners/scanner23.py, uncompyle6/scanners/scanner24.py,
+	* : commit d22931cb49f0e28a0fbe48a7c1526b1f170a5b52 Author: rocky
-	uncompyle6/scanners/scanner3.py, uncompyle6/scanners/tok.py,
+	<rb@dustyfeet.com> Date:   Sun Dec 4 07:31:34 2016 -0500
-	uncompyle6/semantics/make_function.py,
-	uncompyle6/semantics/pysource.py, uncompyle6/verify.py,
-	uncompyle6/version.py: Get ready for release 2.9.7 Some of the many lint things. Linting is kind of stupid though.
 
 2016-11-28  rocky <rb@dustyfeet.com>
 
@@ -388,11 +1055,34 @@
 	* uncompyle6/semantics/pysource.py: Better line number tracking Indent Python 2 list comprehensions, albeit badly.  DRY code a
 	little via indent_if_source_nl
 
+2016-11-24  rocky <rb@dustyfeet.com>
+
+	* circle.yml: CircleCI build
+
+2016-11-24  rocky <rb@dustyfeet.com>
+
+	* uncompyle6/parsers/parse3.py: Remove dup Python 3 grammar rule
+
 2016-11-24  rocky <rb@dustyfeet.com>
 
 	* uncompyle6/parsers/parse3.py, uncompyle6/scanners/scanner2.py: 
 	<2.7 "if" detection and dup Python 3 grammar rule
 
+2016-11-24  rocky <rb@dustyfeet.com>
+
+	* test/test_pyenvlib.py, uncompyle6/linenumbers.py,
+	uncompyle6/main.py, uncompyle6/scanners/scanner2.py,
+	uncompyle6/semantics/make_function.py,
+	uncompyle6/semantics/pysource.py, uncompyle6/verify.py: Bug in 2.4
+	"if" dectection and...  Wrong language used in old-style exceptions: use "except Error,e"
+	not "except Error(e)""
+
+2016-11-24  rocky <rb@dustyfeet.com>
+
+	* __pkginfo__.py, pytest/test_grammar.py, uncompyle6/parser.py,
+	uncompyle6/parsers/parse26.py:  Python 2.6 grammary bug and..  __pkginfo.py__: Bump spark_parser
+	version for parse_flags 'dups'
+
 2016-11-23  rocky <rb@dustyfeet.com>
 
 	* __pkginfo__.py, pytest/test_grammar.py, uncompyle6/parser.py,
@@ -404,8 +1094,17 @@
 
 2016-11-23  rocky <rb@dustyfeet.com>
 
-	* : commit 6aa1531972de83ecab15b4c96b89c873ea5a7458 Author: rocky
+	* : commit 7133540c235e16f02d2db62cb903b70aa311de20 Author: rocky
-	<rb@dustyfeet.com> Date:   Wed Nov 23 00:48:38 2016 -0500
+	<rb@dustyfeet.com> Date:   Wed Nov 23 08:26:12 2016 -0500
+
+2016-11-23  rocky <rb@dustyfeet.com>
+
+	* : commit a9349b8f3d12b2aa0cd88286617c1af9cccad018 Author: rocky
+	<rb@dustyfeet.com> Date:   Tue Nov 22 17:49:47 2016 -0500
+
+2016-11-23  rocky <rb@dustyfeet.com>
+
+	* circle.yml: Circle CI uses 2.7.10 and 2.7.12 is not available
 
 2016-11-22  rocky <rb@dustyfeet.com>
 

HISTORY.md

@@ -30,7 +30,7 @@ another clever idea: using table-driven semantics routines, using
 format specifiers.
 
 The last mention of a release of SPARK from John is around 2002. As
-released, although the Early Algorithm parser was in good shape, this
+released, although the Earley Algorithm parser was in good shape, this
 code was woefully lacking as serious Python deparser.
 
 In the fall of 2000, Hartmut Goebel
@@ -44,7 +44,8 @@ it appears that Hartmut did most of the work to get this code to
 accept the full Python language. He added precedence to the table
 specifiers, support for multiple versions of Python, the
 pretty-printing of docstrings, lists, and hashes. He also wrote test and verification routines of
-deparsed bytecode, and used this in an extensive set of tests that he also wrote. He could verify against the entire Python library.
+deparsed bytecode, and used this in an extensive set of tests that he also wrote. He says he could verify against the 
+entire Python library. However I have subsequently found small and relatively obscure bugs in the decompilation code. 
 
 decompyle2.2 was packaged for Debian (sarge) by
 [Ben Burton around 2002](https://packages.qa.debian.org/d/decompyle.html). As
@@ -65,10 +66,12 @@ code to handle first Python 2.3 and then 2.4 bytecodes. Because of
 jump optimization introduced in the CPython bytecode compiler at that
 time, various JUMP instructions were classifed as going backwards, and
 COME FROM instructions were reintroduced.  See
-RELEASE-2.4-CHANGELOG.txt for more details here. There wasn't a public
+[RELEASE-2.4-CHANGELOG.txt](https://github.com/rocky/python-uncompyle6/blob/master/DECOMPYLE-2.4-CHANGELOG.txt) 
+for more details here. There wasn't a public
 release of RELEASE-2.4 and bytecodes other than Python 2.4 weren't
 supported. Dan says the Python 2.3 version could verify the entire
-python library.
+Python library. But given subsequent bugs found like simply
+recognizing complex-number constants in bytecode, decompilation wasn't perfect.
 
 Next we get to ["uncompyle" and
 PyPI](https://pypi.python.org/pypi/uncompyle/1.1) and the era of
@@ -95,17 +98,17 @@ so. Then hamled made a few commits earler on, while Eike Siewertsen
 made a few commits later on. But mostly wibiti, and Guenther
 Starnberger got the code to where uncompyle2 was around 2012.
 
-In uncompyle2 decompilation of python bytecode 2.5 & 2.6 is done by
+In `uncompyle`, decompilation of python bytecode 2.5 & 2.6 is done by
 transforming the byte code into a a pseudo 2.7 python bytecode and is
 based on code from Eloi Vanderbeken.
 
-This project, uncompyle6, abandons that approach for various
+This project, `uncompyle6`, abandons that approach for various
 reasons. However the main reason is that we need offsets in fragment
 deparsing to be exactly the same, and the transformation process can
-remove instructions.  Adding instructions with psuedo_offsets is
+remove instructions.  _Adding_ instructions with psuedo offsets is
 however okay.
 
-Uncompyle6, however owes its existence to the fork of uncompyle2 by
+`Uncompyle6` however owes its existence to the fork of `uncompyle2` by
 Myst herie (Mysterie) whose first commit picks up at
 2012. I chose this since it seemed to have been at that time the most
 actively, if briefly, worked on. Also starting around 2012 is Dark
@@ -115,9 +118,12 @@ I started working on this late 2015, mostly to add fragment support.
 In that, I decided to make this runnable on Python 3.2+ and Python 2.6+
 while, handling Python bytecodes from Python versions 2.5+ and
 3.2+. In doing so, it has been expedient to separate this into three
-projects: load loading and disassembly (xdis), parsing and tree
+projects:
-building (spark_parser), and grammar and semantic actions for
+
-decompiling (uncompyle6).
+* bytecode loading and disassembly ([xdis](https://pypi.python.org/pypi/xdis)),
+* parsing and tree building ([spark_parser](https://pypi.python.org/pypi/spark_parser)),
+* this project - grammar and semantic actions for decompiling
+  ([uncompyle6](https://pypi.python.org/pypi/spark_parser)).
 
 
 Over the many years, code styles and Python features have
@@ -135,23 +141,26 @@ Hartmut a decade an a half ago:
     NB. This is not a masterpiece of software, but became more like a hack.
     Probably a complete rewrite would be sensefull. hG/2000-12-27
 
-This project deparses using an Early-algorithm parse with lots of
+This project deparses using an Earley-algorithm parse with lots of
 massaging of tokens and the grammar in the scanner
-phase. Early-algorithm parsers are context free and tend to be linear
+phase. Earley-algorithm parsers are context free and tend to be linear
 if the grammar is LR or left recursive.
 
 Another approach that doesn't use grammars is to do something like
 simulate execution symbolically and build expression trees off of
-stack results. The two important projects that work this way are
+stack results. Control flow in that apprproach still needs to be
-[unpyc3](https://code.google.com/p/unpyc3/) and most especially
+handled somewhat ad hoc.  The two important projects that work this
-[pycdc](https://github.com/zrax/pycdc) The latter project is largely
+way are [unpyc3](https://code.google.com/p/unpyc3/) and most
-by Michael Hansen and Darryl Pogue. If they supported getting
+especially [pycdc](https://github.com/zrax/pycdc) The latter project
-source-code fragments and I could call it from Python, I'd probably
+is largely by Michael Hansen and Darryl Pogue. If they supported
-ditch this and use that. From what I've seen, the code runs blindingly
+getting source-code fragments, did a better job in supporting Python
-fast and spans all versions of Python.
+more fully, and had a way I could call it from Python, I'd probably
+would have ditched this and used that. The code runs blindingly fast
+and spans all versions of Python, although more recently Python 3
+support has been lagging.
 
 Tests for the project have been, or are being, culled from all of the
 projects mentioned.
 
-NB. If you find mistakes, want corrections, or want your name added (or removed),
+NB. If you find mistakes, want corrections, or want your name added
-please contact me.
+(or removed), please contact me.

HOW-TO-REPORT-A-BUG.md

@@ -0,0 +1,63 @@
+# How to report a Bug
+
+## The difficulty of the problem
+
+There is no Python decompiler yet, that I know about that will
+decompyle everything. This one probably does the
+best job of *any* Python decompiler. But it is a constant work in progress: Python keeps changing, and so does its code generation.
+
+I have found bugs in *every* Python decompiler I have tried. Even
+those where authors/maintainers claim that they have used it on 
+the entire Python standard library. And I don't mean that
+the program doesn't come out with the same Python source instructions,
+but that the program is *semantically* not equivalent.
+
+So it is likely you'll find a mistranslation in decompiling.
+
+## What to send (minimum requirements)
+
+The basic requirement is pretty simple:
+
+* Python bytecode
+* Source text
+
+## What to send (additional helpful information)
+
+Some kind folks also give the invocation they used and the output
+which usually includes an error message produced. This is helpful. I
+can figure out what OS you are running this on and what version of
+*uncomplye6* was used. Therefore, if you don't provide the input
+command and the output from that, please give:
+
+* _uncompile6_ version used
+* OS that you used this on
+* Python interpreter version used
+
+
+### But I don't *have* the source code!
+
+Sure, I get it. No problem. There is Python assembly code on parse
+errors, so simply by hand decompile that. To get a full disassembly, use pydisasm from the [xdis](https://pypi.python.org/pypi/xdis) package. Opcodes are described in the documentation for the [dis](https://docs.python.org/3.6/library/dis.html) module.
+
+### But I don't *have* the source code and am incapable of figuring how how to do a hand disassembly!
+
+Well, you could learn. No one is born into this world knowing how to disassemble Python bytecode. And as Richard Feynman once said, "What one fool can learn, so can another."
+
+## Narrowing the problem
+
+I don't need the entire source code base for which one file or module
+can't be decompiled. I just need that one file or module only. If
+there are several files, file a bug report for each file.
+
+Python modules can get quite large, and usually decompilation problems
+occur in a single function or maybe the main-line code but not any of
+the functions or classes. So please chop down the source code by
+removing those parts that do to decompile properly.
+
+By doing this, you'll probably have a better sense of what exactly is
+the problem. Perhaps you can find the boundary of what decompiles, and
+what doesn't. That is useful. Or maybe the same file will decompile
+properly on a neighboring version of Python. That is helpful too.
+
+In sum, the more you can isolate or narrow the problem, the more
+likley the problem will be fixed and fixed sooner.

MANIFEST.in

@@ -1,6 +1,7 @@
 include README.rst
 include ChangeLog
 include HISTORY.md
+include HOW-TO-REPORT-A-BUG.md
 include LICENSE
 include Makefile
 include requirements.txt

Makefile

@@ -37,7 +37,7 @@ check-3.0 check-3.1 check-3.2 check-3.5 check-3.6:
 	$(MAKE) -C test $@
 
 #:Tests for Python 2.6 (doesn't have pytest)
-check-2.6:
+check-2.4 check-2.5 check-2.6:
 	$(MAKE) -C test $@
 
 #:PyPy 2.6.1 or PyPy 5.0.1
@@ -59,7 +59,7 @@ clean: clean_pyc
 
 #: Create source (tarball) and wheel distribution
 dist:
-	$(PYTHON) ./setup.py sdist bdist_wheel
+	$(PYTHON) ./setup.py sdist bdist_egg
 
 #: Remove .pyc files
 clean_pyc:

NEWS

@@ -1,3 +1,15 @@
+uncompyle6 2.9.11 2016-04-06
+
+- Better support for Python 3.5+ BUILD_MAP_UNPACK
+- Start 3.6 CALL_FUNCTION_EX support
+- Many decompilation bug fixes. (Many more remain). See ChangeLog
+
+uncompyle6 2.9.10 2016-02-25
+
+- Python grammar rule fixes
+- Add ability to get grammar coverage on runs
+- Handle Python 3.6 opcode BUILD_CONST_KEYMAP
+
 uncompyle6 2.9.9 2016-12-16
 
 - Remaining Python 3.5 ops handled

README.rst

@@ -11,8 +11,8 @@ Introduction
 ------------
 
 *uncompyle6* translates Python bytecode back into equivalent Python
-source code. It accepts bytecodes from Python version 2.1 to 3.6 or
+source code. It accepts bytecodes from Python version 1.5, and 2.1 to
-so, including PyPy bytecode and Dropbox's Python 2.5 bytecode.
+3.6 or so, including PyPy bytecode and Dropbox's Python 2.5 bytecode.
 
 Why this?
 ---------
@@ -21,7 +21,8 @@ There were a number of decompyle, uncompile, uncompyle2, uncompyle3
 forks around. All of them came basically from the same code base, and
 almost all of them no were no longer actively maintained. Only one
 handled Python 3, and even there, only 3.2 or 3.3 depending on which
-code is used. This code pulls these together and moves forward. It
+code is used. This code pulls these together and moves forward. This
+project has the most complete support for Python 3.3 and above. It
 also addresses a number of open issues in the previous forks.
 
 What makes this different from other CPython bytecode decompilers?: its
@@ -46,7 +47,7 @@ Requirements
 This project requires Python 2.6 or later, PyPy 3-2.4, or PyPy-5.0.1.
 Python versions 2.4-2.7 are supported in the python-2.4 branch.
 The bytecode files it can read has been tested on Python bytecodes from
-versions 1.5, 2.1-2.7, and 3.2-3.6 and the above-mentioned PyPy versions.
+versions 1.5, 2.1-2.7, and 3.0-3.6 and the above-mentioned PyPy versions.
 
 Installation
 ------------
@@ -112,7 +113,7 @@ with handling control flow. All of the Python decompilers I have looked
 at have the same problem. In some cases we can detect an erroneous
 decompilation and report that.
 
-About 90% of the decompilation of Python standard library packages in
+Over 98% of the decompilation of Python standard library packages in
 Python 2.7.12 verifies correctly. Over 99% of Python 2.7 and 3.3-3.5
 "weakly" verify. Python 2.6 drops down to 96% weakly verifying.
 Other versions drop off in quality too.
@@ -140,11 +141,10 @@ and 2.0.
 
 In the Python 3 series, Python support is is strongest around 3.4 or
 3.3 and drops off as you move further away from those versions. Python
-3.5 largely works, but still has some bugs in it and is missing some
+3.6 changes things drastically by using word codes rather than byte
-opcodes.  Python 3.6 changes things drastically by using word codes
+codes. That has been addressed, but then it also changes function call
-rather than byte codes. That has been addressed, but then it also
+opcodes and its semantics and has more problems with control flow than
-changes function call opcodes and its semantics and has more problems
+3.5 has.
-with control flow than 3.5 has.
 
 Currently not all Python magic numbers are supported. Specifically in
 some versions of Python, notably Python 3.6, the magic number has
@@ -158,13 +158,16 @@ We also don't handle PJOrion_ obfuscated code. For that try: PJOrion
 Deobfuscator_ to unscramble the bytecode to get valid bytecode before
 trying this tool.
 
+Handling pathologically long lists of expressions or statements is
+slow.
+
 
 There is lots to do, so please dig in and help.
 
 See Also
 --------
 
-* https://github.com/zrax/pycdc : supports all versions of Python and is written in C++
+* https://github.com/zrax/pycdc : supports all versions of Python and is written in C++. Support for later Python 3 versions is a bit lacking though.
 * https://code.google.com/archive/p/unpyc3/ : supports Python 3.2 only. The above projects use a different decompiling technique what is used here.
 * https://github.com/figment/unpyc3/ : fork of above, but supports Python 3.3 only. Include some fixes like supporting function annotations
 * The HISTORY_ file.

__pkginfo__.py

@@ -9,7 +9,7 @@
 
 # Things that change more often go here.
 copyright   = """
-Copyright (C) 2015, 2016 Rocky Bernstein <rb@dustyfeet.com>.
+Copyright (C) 2015-2017 Rocky Bernstein <rb@dustyfeet.com>.
 """
 
 classifiers =  ['Development Status :: 5 - Production/Stable',
@@ -39,8 +39,8 @@ entry_points={
         'pydisassemble=uncompyle6.bin.pydisassemble:main',
     ]}
 ftp_url            = None
-install_requires   = ['spark-parser >= 1.5.1, < 1.6.0',
+install_requires   = ['spark-parser >= 1.6.0, < 1.7.0',
-                      'xdis >= 3.2.4, < 3.3.0']
+                      'xdis >= 3.3.0, < 3.4.0']
 license            = 'MIT'
 mailing_list       = 'python-debugger@googlegroups.com'
 modname            = 'uncompyle6'

circle.yml

@@ -10,4 +10,4 @@ dependencies:
         - pip install -r requirements-dev.txt
 test:
     override:
-        - python ./setup.py develop && make check-2.7
+        - python ./setup.py develop && make check-2.6

pytest/test_build_const_key_map.py

@@ -0,0 +1,21 @@
+import pytest
+# uncompyle6
+from uncompyle6 import PYTHON_VERSION
+from validate import validate_uncompyle
+
+
+@pytest.mark.skipif(PYTHON_VERSION < 3.6, reason='need at least python 3.6')
+@pytest.mark.parametrize('text', (
+    "{0.: 'a', -1: 'b'}",   # BUILD_MAP
+    "{'a':'b'}",            # BUILD_MAP
+    "{0: 1}",               # BUILD_MAP
+    "{b'0':1, b'2':3}",     # BUILD_CONST_KEY_MAP
+    "{0: 1, 2: 3}",         # BUILD_CONST_KEY_MAP
+    "{'a':'b','c':'d'}",    # BUILD_CONST_KEY_MAP
+    "{0: 1, 2: 3}",         # BUILD_CONST_KEY_MAP
+    "{'a': 1, 'b': 2}",     # BUILD_CONST_KEY_MAP
+    "{'a':'b','c':'d'}",    # BUILD_CONST_KEY_MAP
+    "{0.0:'b',0.1:'d'}",    # BUILD_CONST_KEY_MAP
+))
+def test_build_const_key_map(text):
+    validate_uncompyle(text)

pytest/test_fstring.py

@@ -1,150 +0,0 @@
-# std
-import os
-# test
-import pytest
-import hypothesis
-from hypothesis import strategies as st
-# uncompyle6
-from uncompyle6 import PYTHON_VERSION, deparse_code
-
-
-@st.composite
-def expressions(draw):
-    # todo : would be nice to generate expressions using hypothesis however
-    # this is pretty involved so for now just use a corpus of expressions
-    # from which to select.
-    return draw(st.sampled_from((
-        'abc',
-        'len(items)',
-        'x + 1',
-        'lineno',
-        'container',
-        'self.attribute',
-        'self.method()',
-        # These expressions are failing, I think these are control
-        # flow problems rather than problems with FORMAT_VALUE,
-        # however I need to confirm this...
-        #'sorted(items, key=lambda x: x.name)',
-        #'func(*args, **kwargs)',
-        #'text or default',
-        #'43 if life_the_universe and everything else None'
-    )))
-
-
-@st.composite
-def format_specifiers(draw):
-    """
-    Generate a valid format specifier using the rules:
-
-    format_spec ::=  [[fill]align][sign][#][0][width][,][.precision][type]
-    fill        ::=  <any character>
-    align       ::=  "<" | ">" | "=" | "^"
-    sign        ::=  "+" | "-" | " "
-    width       ::=  integer
-    precision   ::=  integer
-    type        ::=  "b" | "c" | "d" | "e" | "E" | "f" | "F" | "g" | "G" | "n" | "o" | "s" | "x" | "X" | "%"
-
-    See https://docs.python.org/2/library/string.html
-
-    :param draw: Let hypothesis draw from other strategies.
-
-    :return: An example format_specifier.
-    """
-    alphabet_strategy = st.characters(min_codepoint=ord('a'), max_codepoint=ord('z'))
-    fill = draw(st.one_of(alphabet_strategy, st.none()))
-    align = draw(st.sampled_from(list('<>=^')))
-    fill_align = (fill + align or '') if fill else ''
-
-    type_ = draw(st.sampled_from('bcdeEfFgGnosxX%'))
-    can_have_sign = type_ in 'deEfFgGnoxX%'
-    can_have_comma = type_ in 'deEfFgG%'
-    can_have_precision = type_ in 'fFgG'
-    can_have_pound = type_ in 'boxX%'
-    can_have_zero = type_ in 'oxX'
-
-    sign = draw(st.sampled_from(list('+- ') + [''])) if can_have_sign else ''
-    pound = draw(st.sampled_from(('#', '',))) if can_have_pound else ''
-    zero = draw(st.sampled_from(('0', '',))) if can_have_zero else ''
-
-    int_strategy = st.integers(min_value=1, max_value=1000)
-
-    width = draw(st.one_of(int_strategy, st.none()))
-    width = str(width) if width is not None else ''
-
-    comma = draw(st.sampled_from((',', '',))) if can_have_comma else ''
-    if can_have_precision:
-        precision = draw(st.one_of(int_strategy, st.none()))
-        precision = '.' + str(precision) if precision else ''
-    else:
-        precision = ''
-
-    return ''.join((fill_align, sign, pound, zero, width, comma, precision, type_,))
-
-
-@st.composite
-def fstrings(draw):
-    """
-    Generate a valid f-string.
-    See https://www.python.org/dev/peps/pep-0498/#specification
-
-    :param draw: Let hypothsis draw from other strategies.
-
-    :return: A valid f-string.
-    """
-    character_strategy = st.characters(
-        blacklist_characters='\r\n\'\\s{}',
-        min_codepoint=1,
-        max_codepoint=1000,
-    )
-    is_raw = draw(st.booleans())
-    integer_strategy = st.integers(min_value=0, max_value=3)
-    expression_count = draw(integer_strategy)
-    content = []
-    for _ in range(expression_count):
-        expression = draw(expressions())
-        conversion = draw(st.sampled_from(('', '!s', '!r', '!a',)))
-        has_specifier = draw(st.booleans())
-        specifier = ':' + draw(format_specifiers()) if has_specifier else ''
-        content.append('{{{}{}}}'.format(expression, conversion, specifier))
-        content.append(draw(st.text(character_strategy)))
-    content = ''.join(content)
-    return "f{}'{}'".format('r' if is_raw else '', content)
-
-
-@pytest.mark.skipif(PYTHON_VERSION < 3.6, reason='need at least python 3.6')
-@hypothesis.given(format_specifiers())
-def test_format_specifiers(format_specifier):
-    """Verify that format_specifiers generates valid specifiers"""
-    try:
-        exec('"{:' + format_specifier + '}".format(0)')
-    except ValueError as e:
-        if 'Unknown format code' not in str(e):
-            raise
-
-
-def run_test(text):
-    hypothesis.assume(len(text))
-    hypothesis.assume("f'{" in text)
-    expr = text + '\n'
-    code = compile(expr, '<string>', 'single')
-    deparsed = deparse_code(PYTHON_VERSION, code, compile_mode='single')
-    recompiled = compile(deparsed.text, '<string>', 'single')
-    if recompiled != code:
-        assert 'dis(' + deparsed.text.strip('\n') + ')' == 'dis(' + expr.strip('\n') + ')'
-
-
-@pytest.mark.skipif(PYTHON_VERSION < 3.6, reason='need at least python 3.6')
-@hypothesis.given(fstrings())
-def test_uncompyle_fstring(fstring):
-    """Verify uncompyling fstring bytecode"""
-    run_test(fstring)
-
-
-@pytest.mark.skipif(PYTHON_VERSION < 3.6, reason='need at least python 3.6')
-@pytest.mark.parametrize('fstring', [
-    "f'{abc}{abc!s}'",
-    "f'{abc}0'",
-])
-def test_uncompyle_direct(fstring):
-    """useful for debugging"""
-    run_test(fstring)

pytest/test_function_call.py

@@ -0,0 +1,128 @@
+# std
+import string
+# 3rd party
+from hypothesis import given, assume, strategies as st
+import pytest
+# uncompyle
+from validate import validate_uncompyle
+
+
+alpha = st.sampled_from(string.ascii_lowercase)
+numbers = st.sampled_from(string.digits)
+alphanum = st.sampled_from(string.ascii_lowercase + string.digits)
+expressions = st.sampled_from([x for x in string.ascii_lowercase + string.digits] + ['x+1'])
+
+
+@st.composite
+def function_calls(draw):
+    """
+    Strategy factory for generating function calls.
+
+    :param draw: Callable which draws examples from other strategies.
+
+    :return: The function call text.
+    """
+    list1 = st.lists(alpha, min_size=0, max_size=1)
+    list3 = st.lists(alpha, min_size=0, max_size=3)
+
+    positional_args = draw(list3)
+    named_args = [x + '=0' for x in draw(list3)]
+    star_args = ['*' + x for x in draw(list1)]
+    double_star_args = ['**' + x for x in draw(list1)]
+
+    arguments = positional_args + named_args + star_args + double_star_args
+    draw(st.randoms()).shuffle(arguments)
+    arguments = ','.join(arguments)
+
+    function_call = 'fn({arguments})'.format(arguments=arguments)
+    try:
+        # TODO: Figure out the exact rules for ordering of positional, named,
+        # star args, double star args and in which versions the various
+        # types of arguments are supported so we don't need to check that the
+        # expression compiles like this.
+        compile(function_call, '<string>', 'single')
+    except:
+        assume(False)
+    return function_call
+
+
+@pytest.mark.xfail()
+def test_CALL_FUNCTION():
+    validate_uncompyle("fn(w,m,f)")
+
+
+@pytest.mark.xfail()
+def test_BUILD_CONST_KEY_MAP_BUILD_MAP_UNPACK_WITH_CALL_BUILD_TUPLE_CALL_FUNCTION_EX():
+    validate_uncompyle("fn(w=0,m=0,**v)")
+
+
+@pytest.mark.xfail()
+def test_BUILD_MAP_BUILD_MAP_UNPACK_WITH_CALL_BUILD_TUPLE_CALL_FUNCTION_EX():
+    validate_uncompyle("fn(a=0,**g)")
+
+
+@pytest.mark.xfail()
+def test_CALL_FUNCTION_KW():
+    validate_uncompyle("fn(j=0)")
+
+
+@pytest.mark.xfail()
+def test_CALL_FUNCTION_EX():
+    validate_uncompyle("fn(*g,**j)")
+
+
+@pytest.mark.xfail()
+def test_BUILD_MAP_CALL_FUNCTION_EX():
+    validate_uncompyle("fn(*z,u=0)")
+
+
+@pytest.mark.xfail()
+def test_BUILD_TUPLE_CALL_FUNCTION_EX():
+    validate_uncompyle("fn(**a)")
+
+
+@pytest.mark.xfail()
+def test_BUILD_MAP_BUILD_TUPLE_BUILD_TUPLE_UNPACK_WITH_CALL_CALL_FUNCTION_EX():
+    validate_uncompyle("fn(b,b,b=0,*a)")
+
+
+@pytest.mark.xfail()
+def test_BUILD_TUPLE_BUILD_TUPLE_UNPACK_WITH_CALL_CALL_FUNCTION_EX():
+    validate_uncompyle("fn(*c,v)")
+
+
+@pytest.mark.xfail()
+def test_BUILD_CONST_KEY_MAP_CALL_FUNCTION_EX():
+    validate_uncompyle("fn(i=0,y=0,*p)")
+
+
+@pytest.mark.skip(reason='skipping property based test until all individual tests are passing')
+@given(function_calls())
+def test_function_call(function_call):
+    validate_uncompyle(function_call)
+
+
+examples = set()
+generate_examples = False
+
+
+@pytest.mark.skipif(not generate_examples, reason='not generating examples')
+@given(function_calls())
+def test_generate_hypothesis(function_call):
+    examples.add(function_call)
+
+
+@pytest.mark.skipif(not generate_examples, reason='not generating examples')
+def test_generate_examples():
+    import dis
+    example_opcodes = {}
+    for example in examples:
+        opcodes = tuple(sorted(set(
+            instruction.opname
+            for instruction in dis.Bytecode(example)
+            if instruction.opname not in ('LOAD_CONST', 'LOAD_NAME', 'RETURN_VALUE')
+        )))
+        example_opcodes[opcodes] = example
+    for k, v in example_opcodes.items():
+        print('def test_' + '_'.join(k) + '():\n    validate_uncompyle("' + v + '")\n\n')
+    return

pytest/test_grammar.py

@@ -40,7 +40,9 @@ def test_grammar():
     ignore_set = set(
             """
             JUMP_BACK CONTINUE RETURN_END_IF
-            COME_FROM COME_FROM_EXCEPT COME_FROM_LOOP COME_FROM_WITH
+            COME_FROM COME_FROM_EXCEPT
+            COME_FROM_EXCEPT_CLAUSE
+            COME_FROM_LOOP COME_FROM_WITH
             COME_FROM_FINALLY ELSE
             LOAD_GENEXPR LOAD_ASSERT LOAD_SETCOMP LOAD_DICTCOMP
             LAMBDA_MARKER RETURN_LAST

pytest/validate.py

@@ -0,0 +1,147 @@
+# future
+from __future__ import print_function
+# std
+import os
+import difflib
+import subprocess
+import tempfile
+import functools
+# compatability
+import six
+# uncompyle6 / xdis
+from uncompyle6 import PYTHON_VERSION, IS_PYPY, deparse_code
+# TODO : I think we can get xdis to support the dis api (python 3 version) by doing something like this there
+from xdis.bytecode import Bytecode
+from xdis.main import get_opcode
+opc = get_opcode(PYTHON_VERSION, IS_PYPY)
+Bytecode = functools.partial(Bytecode, opc=opc)
+
+
+def _dis_to_text(co):
+    return Bytecode(co).dis()
+
+
+def print_diff(original, uncompyled):
+    """
+    Try and display a pretty html line difference between the original and
+    uncompyled code and bytecode if elinks and BeautifulSoup are installed
+    otherwise just show the diff.
+
+    :param original: Text describing the original code object.
+    :param uncompyled: Text describing the uncompyled code object.
+    """
+    original_lines = original.split('\n')
+    uncompyled_lines = uncompyled.split('\n')
+    args = original_lines, uncompyled_lines, 'original', 'uncompyled'
+    try:
+        from bs4 import BeautifulSoup
+        diff = difflib.HtmlDiff().make_file(*args)
+        diff = BeautifulSoup(diff, "html.parser")
+        diff.select_one('table[summary="Legends"]').extract()
+    except ImportError:
+        print('\nTo display diff highlighting run:\n    pip install BeautifulSoup4')
+        diff = difflib.HtmlDiff().make_table(*args)
+
+    with tempfile.NamedTemporaryFile(delete=False) as f:
+        f.write(str(diff).encode('utf-8'))
+
+    try:
+        print()
+        html = subprocess.check_output([
+            'elinks',
+            '-dump',
+            '-no-references',
+            '-dump-color-mode',
+            '1',
+            f.name,
+        ]).decode('utf-8')
+        print(html)
+    except:
+        print('\nFor side by side diff install elinks')
+        diff = difflib.Differ().compare(original_lines, uncompyled_lines)
+        print('\n'.join(diff))
+    finally:
+        os.unlink(f.name)
+
+
+def are_instructions_equal(i1, i2):
+    """
+    Determine if two instructions are approximately equal,
+    ignoring certain fields which we allow to differ, namely:
+
+    * code objects are ignore (should probaby be checked) due to address
+    * line numbers
+
+    :param i1: left instruction to compare
+    :param i2: right instruction to compare
+
+    :return: True if the two instructions are approximately equal, otherwise False.
+    """
+    result = (1==1
+        and i1.opname == i2.opname
+        and i1.opcode == i2.opcode
+        and i1.arg == i2.arg
+        # ignore differences due to code objects
+        # TODO : Better way of ignoring address
+        and (i1.argval == i2.argval or '<code object' in str(i1.argval))
+        # TODO : Should probably recurse to check code objects
+        and (i1.argrepr == i2.argrepr or '<code object' in i1.argrepr)
+        and i1.offset == i2.offset
+        # ignore differences in line numbers
+        #and i1.starts_line
+        and i1.is_jump_target == i2.is_jump_target
+    )
+    return result
+
+
+def are_code_objects_equal(co1, co2):
+    """
+    Determine if two code objects are approximately equal,
+    see are_instructions_equal for more information.
+
+    :param i1: left code object to compare
+    :param i2: right code object to compare
+
+    :return: True if the two code objects are approximately equal, otherwise False.
+    """
+    instructions1 = Bytecode(co1)
+    instructions2 = Bytecode(co2)
+    for opcode1, opcode2 in zip(instructions1, instructions2):
+        if not are_instructions_equal(opcode1, opcode2):
+            return False
+    return True
+
+
+def validate_uncompyle(text, mode='exec'):
+    """
+    Validate decompilation of the given source code.
+
+    :param text: Source to validate decompilation of.
+    """
+    original_code = compile(text, '<string>', mode)
+    original_dis = _dis_to_text(original_code)
+    original_text = text
+
+    deparsed = deparse_code(PYTHON_VERSION, original_code,
+                            compile_mode=mode, out=six.StringIO())
+    uncompyled_text = deparsed.text
+    uncompyled_code = compile(uncompyled_text, '<string>', 'exec')
+
+    if not are_code_objects_equal(uncompyled_code, original_code):
+
+        uncompyled_dis = _dis_to_text(uncompyled_text)
+
+        def output(text, dis):
+            width = 60
+            return '\n\n'.join([
+                ' SOURCE CODE '.center(width, '#'),
+                text.strip(),
+                ' BYTECODE '.center(width, '#'),
+                dis
+            ])
+
+        original = output(original_text, original_dis)
+        uncompyled = output(uncompyled_text, uncompyled_dis)
+        print_diff(original, uncompyled)
+
+        assert 'original' == 'uncompyled'

requirements-dev.txt

@@ -1,3 +1,4 @@
 pytest
 flake8
-hypothesis
+hypothesis
+six

test/Makefile

@@ -3,7 +3,7 @@ PHONY=check clean dist distclean test test-unit test-functional rmChangeLog clea
 GIT2CL ?= git2cl
 PYTHON ?= python
 
-PYTHON_VERSION = $(shell $(PYTHON) -V | cut -d ' ' -f 2 | cut -d'.' -f1,2)
+PYTHON_VERSION = $(shell $(PYTHON) -V 2>&1 | cut -d ' ' -f 2 | cut -d'.' -f1,2)
 NATIVE_CHECK = check-$(PYTHON_VERSION)
 
 # Set COMPILE='--compile' to force compilation before check
@@ -20,7 +20,7 @@ check:
 	$(MAKE) check-$$PYTHON_VERSION
 
 #: Run working tests from Python 2.6 or 2.7
-check-2.6 check-2.7: check-bytecode-2 check-bytecode-3 check-bytecode-1  check-2.7-ok
+check-2.4 check-2.5 check-2.6 check-2.7: check-bytecode-2 check-bytecode-3 check-bytecode-1  check-native-short
 
 #: Run working tests from Python 3.0
 check-3.0: check-bytecode
@@ -68,7 +68,7 @@ check-bytecode-2:
 check-bytecode-3:
 	$(PYTHON) test_pythonlib.py --bytecode-3.0 \
        --bytecode-3.1 --bytecode-3.2 --bytecode-3.3 \
-       --bytecode-3.4 --bytecode-3.5 --bytecode-pypy3.2
+       --bytecode-3.4 --bytecode-3.5 --bytecode-3.6 --bytecode-pypy3.2
 
 #: Check deparsing bytecode that works running Python 2 and Python 3
 check-bytecode: check-bytecode-3
@@ -98,14 +98,6 @@ check-bytecode-2.4:
 check-bytecode-2.5:
 	$(PYTHON) test_pythonlib.py --bytecode-2.5
 
-#: Check deparsing Python 2.6
-check-bytecode-2.6:
-	$(PYTHON) test_pythonlib.py --bytecode-2.6 --weak-verify
-
-#: Check deparsing Python 2.7
-check-bytecode-2.7:
-	$(PYTHON) test_pythonlib.py --bytecode-2.7 --verify
-
 #: Check deparsing Python 3.0
 check-bytecode-3.0:
 	$(PYTHON) test_pythonlib.py --bytecode-3.0
@@ -134,9 +126,33 @@ check-bytecode-3.5:
 check-bytecode-3.6:
 	$(PYTHON) test_pythonlib.py --bytecode-3.6
 
+#: Get grammar coverage for Python 2.4
+grammar-coverage-2.4:
+	SPARK_PARSER_COVERAGE=/tmp/spark-grammar-24.cover $(PYTHON) test_pythonlib.py --bytecode-2.4
+	SPARK_PARSER_COVERAGE=/tmp/spark-grammar-24.cover $(PYTHON) test_pyenvlib.py --2.4.6
+
+#: Get grammar coverage for Python 2.5
+grammar-coverage-2.5:
+	SPARK_PARSER_COVERAGE=/tmp/spark-grammar-25.cover $(PYTHON) test_pythonlib.py --bytecode-2.5
+	SPARK_PARSER_COVERAGE=/tmp/spark-grammar-25.cover $(PYTHON) test_pyenvlib.py --2.5.6
+
+#: Get grammar coverage for Python 2.6
+grammar-coverage-2.6:
+	SPARK_PARSER_COVERAGE=/tmp/spark-grammar-26.cover $(PYTHON) test_pythonlib.py --bytecode-2.6
+	SPARK_PARSER_COVERAGE=/tmp/spark-grammar-26.cover $(PYTHON) test_pyenvlib.py --2.6.9
+
+#: Get grammar coverage for Python 2.7
+grammar-coverage-2.7:
+	SPARK_PARSER_COVERAGE=/tmp/spark-grammar-27.cover $(PYTHON) test_pythonlib.py --bytecode-2.7
+	SPARK_PARSER_COVERAGE=/tmp/spark-grammar-27.cover $(PYTHON) test_pyenvlib.py --2.7.13
+
 #: short tests for bytecodes only for this version of Python
 check-native-short:
-	$(PYTHON) test_pythonlib.py --bytecode-$(PYTHON_VERSION) --verify $(COMPILE)
+	$(PYTHON) test_pythonlib.py --bytecode-$(PYTHON_VERSION) --weak-verify $(COMPILE)
+
+#: Run longer Python 2.6's lib files known to be okay
+check-2.4-ok:
+	$(PYTHON) test_pythonlib.py --ok-2.4 --verify $(COMPILE)
 
 #: Run longer Python 2.6's lib files known to be okay
 check-2.6-ok:

test/simple_source/bug22/01_ops.py

@@ -0,0 +1,18 @@
+# Statements to beef up grammar coverage rules
+# Force "inplace" ops
+y = +10  # UNARY_POSITIVE
+y /= 1   # INPLACE_DIVIDE
+y %= 4   # INPLACE_MODULO
+y **= 1  # INPLACE POWER
+y >>= 2  # INPLACE_RSHIFT
+y <<= 2  # INPLACE_LSHIFT
+y //= 1  # INPLACE_TRUE_DIVIDE
+y &= 1   # INPLACE_AND
+y ^= 1   # INPLACE_XOR
+
+`y`      # UNARY_CONVERT  - No in Python 3.x
+
+# Beef up augassign and STORE_SLICE+3
+x = [1,2,3,4,5]
+x[0:1] = 1
+x[0:3] += 1, 2, 3

test/simple_source/bug22/03_if1.py

@@ -0,0 +1,7 @@
+# From https://github.com/ToontownInfinite /src/otp/avatar/LocalAvatar.py#L364
+if 1:
+    def jumpLandAnimFix(self, jumpTime):
+        return 5
+
+    def jumpLand(self):
+        return 6

test/simple_source/bug25/02_try_else.py

@@ -1,7 +1,6 @@
 # Python 2.5 bug
 # Was turning into tryelse when there in fact is no else.
 def options(self, section):
-    """Return a list of option names for the given section name."""
     try:
         opts = self._sections[section].copy()
     except KeyError:
@@ -10,3 +9,19 @@ def options(self, section):
     if '__name__' in opts:
         del opts['__name__']
     return opts.keys()
+
+# From python2.5/distutils/command/register.py
+def post_to_server(self, urllib2):
+    try:
+        result = 5
+    except urllib2.HTTPError, e:
+        result = e.code, e.msg
+    except urllib2.URLError, e:
+        result = 500
+    else:
+        if self.show_response:
+            result = 10
+        result = 200
+    if self.show_response:
+        result = 8
+    return result

test/simple_source/bug25/03_if_for.py

@@ -0,0 +1,7 @@
+# From Python 2.6. distutils/sysconfig.py
+def get_config_vars(_config_vars, args):
+    if _config_vars:
+        if args == 1:
+            if args < 8:
+                for key in ('LDFLAGS', 'BASECFLAGS'):
+                    _config_vars[key] = 4

test/simple_source/bug26/03_double_equals.py

@@ -0,0 +1,6 @@
+# From Python 2.7 parse_starttag HTMLParser.pyc
+attrvalue = [1,2]
+while attrvalue:
+    if attrvalue[:1] == 5 or \
+       attrvalue[:1] == 2 == attrvalue[-1:]:
+        attrvalue = 10

test/simple_source/bug27+/03_if_1_else.py

@@ -0,0 +1 @@
+1 if 1 else __file__

test/simple_source/bug31/02_ifelse_comprehension.py

@@ -0,0 +1,12 @@
+# Python 2.7 sqlalchemy-1.013/sql/crud.py
+def _extend_values_for_multiparams(compiler, stmt, c):
+    c(
+        [
+            (
+                (compiler() if compiler()
+                    else compiler())
+                if c in stmt else compiler(),
+            )
+        ]
+        for i in enumerate(stmt)
+    )

test/simple_source/bug31/04_def_annotate.py

@@ -17,3 +17,14 @@ def div(a: dict(type=float, help='the dividend'),
     ) -> dict(type=float, help='the result of dividing a by b'):
     """Divide a by b"""
     return a / b
+
+class TestSignatureObject(unittest.TestCase):
+    def test_signature_on_wkwonly(self):
+        def test(*, a:float, b:str) -> int:
+            pass
+
+class SupportsInt(_Protocol):
+
+    @abstractmethod
+    def __int__(self) -> int:
+        pass

test/simple_source/bug32/01_named_and_kwargs.py

@@ -18,3 +18,12 @@ def __init__(self, defaults=None, dict_type=_default_dict,
              default_section=DEFAULTSECT,
              interpolation=_UNSET):
     pass
+
+# Bug found by hypothesis in creating function calls
+# thanks to moagstar
+def fn(a, b, d):
+    return (a, b, d)
+
+b = {'b': 1,
+     'd': 2}
+fn(a=0, **b)

test/simple_source/bug32/01_try_except_raise.py

@@ -0,0 +1,9 @@
+# From 3.2 _abcoll.py
+def pop(self):
+    it = iter(self)
+    try:
+        value = next(it)
+    except StopIteration:
+        raise KeyError
+    self.discard(value)
+    return value

test/simple_source/bug32/03_if.py

@@ -0,0 +1,11 @@
+# From 3.2 shlex.py
+def _samefile(os, src, dst):
+    if hasattr(os.path, 'samefile'):
+        try:
+            return os.path.samefile(src, dst)
+        except OSError:
+            return False
+
+    # All other platforms: check for same pathname.
+    return (os.path.normcase(os.path.abspath(src)) ==
+            os.path.normcase(os.path.abspath(dst)))

test/simple_source/bug33/01_delete_deref.py

@@ -0,0 +1,4 @@
+def a():
+    del y
+    def b():
+        return y

test/simple_source/bug33/01_if_try_except.py

@@ -0,0 +1,16 @@
+# From 3.3.5 _osx_support.py
+def _get_system_version():
+    if __file__ is None:
+        try:
+            m = 5
+        except IOError:
+            pass
+        else:
+            try:
+                m = 10
+            finally:
+                m = 15
+            if m is not None:
+                m = 20
+
+    return m

test/simple_source/bug33/01_try_except.py

@@ -0,0 +1,16 @@
+# From 3.3.5 _osx_support.py
+def _get_system_version():
+    if __file__ is None:
+        try:
+            m = 5
+        except IOError:
+            pass
+        else:
+            try:
+                m = 10
+            finally:
+                m = 15
+            if m is not None:
+                m = 20
+
+    return m

test/simple_source/bug33/02_pos_args.py

@@ -1,3 +1,6 @@
 # From Python 3.3.6 hmac.py
 # Problem was getting wrong placement of positional args
 digest_cons = lambda d=b'': 5
+
+# Handle single kwarg
+lambda *, d=0: None

test/simple_source/bug33/02_while1.py

@@ -0,0 +1,14 @@
+# From Python 3.4 mailcap
+def readmailcapfile(caps):
+    while 1:
+        line = 'abc'
+        if line[0] == '#' or line == '':
+            continue
+        key, fields = (1,2)
+        if not (key and fields):
+            continue
+        if key in caps:
+            caps[key].append(fields)
+        else:
+            caps[key] = [fields]
+    return caps

test/simple_source/bug35/01_map_unpack.py

@@ -0,0 +1,9 @@
+# Python 3.5+ PEP 448 - Additional Unpacking Generalizations for dictionaries
+{**{}}
+{**{'a': 1, 'b': 2}}
+## {**{'x': 1}, **{'y': 2}}
+# {'c': 1, {'d': 2}, **{'e': 3}}
+[*[]]
+{**{0:0 for a in b}}
+## {**{}, **{}}
+## {**{}, **{}, **{}}

test/simple_source/bug35/02_async_for.py

@@ -0,0 +1,3 @@
+async def a(b, c):
+    async for b in c:
+        pass

test/simple_source/bug35/03_async_await.py

@@ -24,3 +24,9 @@ async def awith_test():
 async def awith_as_test():
     async with 1 as i:
         print(i)
+
+async def f(z):
+    await z
+
+async def g(z):
+    return await z

test/simple_source/bug35/03_while-if-break.py

@@ -5,3 +5,10 @@ def display_date(loop):
         if loop.time():
             break
         x = 5
+
+    # Another loop to test 3.5 ifelsestmtl grammar rule
+    while loop:
+        if x:
+            True
+        else:
+            True

test/simple_source/bug36/01_call_function.py

@@ -0,0 +1,5 @@
+# Python 3.6's changes for calling functions.
+# See https://github.com/rocky/python-uncompyle6/issues/58
+# CALL_FUNCTION_EX takes 2 to 3 arguments on the stack: the function, the tuple of positional arguments,
+# and optionally the dict of keyword arguments if bit 0 of oparg is 1.
+a(*[])

test/simple_source/bug36/01_extended_arg.py

@@ -0,0 +1,2 @@
+if __file__:
+    0+0+0+0+0+0+0+0+0+0+0+0+0+0+0+0+0+0+0+0+0+0+0+0+0+0+0+0+0+0+0+0+0+0+0+0+0+0+0+0+0+0+0+0+0+0+0+0+0+0+0+0+0+0+0+0+0+0+0+0+0+0+0

test/simple_source/def/10_kw+pos_args-bug.py

@@ -8,3 +8,20 @@ def __init__(self, defaults=None, dict_type=_default_dict,
              default_section=DEFAULTSECT,
              interpolation=_UNSET):
     pass
+
+# From 3.5 sqlalchemy/orm/__init__.py
+# Python 3.5 changes the stack position of where * args are (furthest down the stack)
+# Python 3.6+ replaces CALL_FUNCTION_VAR_KW with CALL_FUNCTION_EX
+def deferred(*columns, **kw):
+    return ColumnProperty(deferred=True, *columns, **kw)
+
+
+# From sqlalchemy/sql/selectable.py
+class GenerativeSelect():
+    def __init__(self,
+                 ClauseList,
+                 util,
+                 order_by=None):
+        self._order_by_clause = ClauseList(
+            *util.to_list(order_by),
+            _literal_as_text=5)

test/simple_source/exception/02_try_finally.py

@@ -0,0 +1,11 @@
+# From 2.6.9 cmd.py
+try:
+    if __file__:
+        x = 2
+    x = 3
+finally:
+    if x and __file__:
+        try:
+            x = 1
+        except:
+            pass

test/simple_source/stmts/02_while1else.py

@@ -0,0 +1,8 @@
+# From Python-3.5.2/Lib/multiprocessing/connection.py
+
+def PipeClient(address):
+    while 1:
+        z = 2
+    else:
+        raise
+    return

test/test_pyenvlib.py

@@ -19,8 +19,6 @@ Step 2: Run the test:
 	  test_pyenvlib --mylib --verify # decompile verify 'mylib'
 """
 
-from __future__ import print_function
-
 from uncompyle6 import main, PYTHON3
 import os, time, shutil
 from fnmatch import fnmatch
@@ -30,7 +28,7 @@ from fnmatch import fnmatch
 TEST_VERSIONS=('2.3.7', '2.4.6', '2.5.6', '2.6.9',
                'pypy-2.4.0', 'pypy-2.6.1',
                'pypy-5.0.1', 'pypy-5.3.1',
-               '2.7.10', '2.7.11', '2.7.12',
+               '2.7.10', '2.7.11', '2.7.12', '2.7.13',
                '3.0.1', '3.1.5', '3.2.6',
                '3.3.5', '3.3.6',
                '3.4.2', '3.5.1', '3.6.0')
@@ -106,28 +104,40 @@ def do_tests(src_dir, patterns, target_dir, start_with=None, do_verify=False):
 if __name__ == '__main__':
     import getopt, sys
 
-    do_verify = False
+    do_coverage = do_verify = False
     test_dirs = []
     start_with = None
 
     test_options_keys = list(test_options.keys())
     test_options_keys.sort()
     opts, args = getopt.getopt(sys.argv[1:], '',
-                               ['start-with=', 'verify', 'weak-verify', 'all', ] \
+                               ['start-with=', 'verify', 'weak-verify',
+                                'coverage', 'all', ] \
                                + test_options_keys )
+    vers = ''
     for opt, val in opts:
         if opt == '--verify':
             do_verify = True
         if opt == '--weak-verify':
             do_verify = 'weak'
+        if opt == '--coverage':
+            do_coverage = True
         elif opt == '--start-with':
             start_with = val
         elif opt[2:] in test_options_keys:
-            test_dirs.append(test_options[opt[2:]])
+            triple = test_options[opt[2:]]
+            vers = triple[-1]
+            test_dirs.append(triple)
         elif opt == '--all':
+            vers = 'all'
             for val in test_options_keys:
                 test_dirs.append(test_options[val])
 
+    if do_coverage:
+        os.environ['SPARK_PARSER_COVERAGE'] = (
+            '/tmp/spark-grammar-%s.cover' % vers
+            )
+
     for src_dir, pattern, target_dir in test_dirs:
         if os.path.exists(src_dir):
             target_dir = os.path.join(target_base, target_dir)

test/test_pythonlib.py

@@ -27,8 +27,6 @@ Step 2: Run the test:
   test_pythonlib.py --mylib --verify # decompile verify 'mylib'
 """
 
-from __future__ import print_function
-
 import getopt, os, py_compile, sys, shutil, tempfile, time
 
 from uncompyle6 import PYTHON_VERSION
@@ -127,8 +125,10 @@ def do_tests(src_dir, obj_patterns, target_dir, opts):
     if opts['do_compile']:
         compiled_version = opts['compiled_version']
         if compiled_version and PYTHON_VERSION != compiled_version:
-            print("Not compiling: desired Python version is %s but we are running %s" %
+            sys.stderr.write("Not compiling: "
-                  (compiled_version, PYTHON_VERSION), file=sys.stderr)
+                             "desired Python version is %s "
+                             "but we are running %s" %
+                             (compiled_version, PYTHON_VERSION))
         else:
             for root, dirs, basenames in os.walk(src_dir):
                 file_matches(files, root, basenames, PY)
@@ -146,8 +146,8 @@ def do_tests(src_dir, obj_patterns, target_dir, opts):
         file_matches(files, dirname, basenames, obj_patterns)
 
     if not files:
-        print("Didn't come up with any files to test! Try with --compile?",
+        sys.stderr.write("Didn't come up with any files to test! "
-              file=sys.stderr)
+                         "Try with --compile?")
         exit(1)
 
     os.chdir(cwd)
@@ -161,9 +161,9 @@ def do_tests(src_dir, obj_patterns, target_dir, opts):
         except ValueError:
             pass
 
-    print(time.ctime())
+    print time.ctime()
-    print('Source directory: ', src_dir)
+    print 'Source directory: ', src_dir
-    print('Output directory: ', target_dir)
+    print 'Output directory: ', target_dir
     try:
         _, _, failed_files, failed_verify = \
           main(src_dir, target_dir, files, [],
@@ -190,6 +190,7 @@ if __name__ == '__main__':
     test_options_keys.sort()
     opts, args = getopt.getopt(sys.argv[1:], '',
                                ['start-with=', 'verify', 'weak-verify', 'all', 'compile',
+                                'coverage',
                                 'no-rm'] \
                                + test_options_keys )
     if not opts: help()
@@ -198,7 +199,8 @@ if __name__ == '__main__':
         'do_compile': False,
         'do_verify': False,
         'start_with': None,
-        'rmtree' : True
+        'rmtree' : True,
+        'coverage' : False
         }
 
     for opt, val in opts:
@@ -217,24 +219,30 @@ if __name__ == '__main__':
         elif opt == '--all':
             for val in test_options_keys:
                 test_dirs.append(test_options[val])
+        elif opt == '--coverage':
+            test_opts['coverage'] = True
         else:
             help()
             pass
         pass
 
+    if test_opts['coverage']:
+        os.environ['SPARK_PARSER_COVERAGE'] = (
+            '/tmp/spark-grammar-python-lib%s.cover' % test_dirs[0][-1]
+            )
+
     last_compile_version = None
     for src_dir, pattern, target_dir, compiled_version in test_dirs:
         if os.path.isdir(src_dir):
             checked_dirs.append([src_dir, pattern, target_dir])
         else:
-            print("Can't find directory %s. Skipping" % src_dir,
+            sys.stderr.write("Can't find directory %s. Skipping" % src_dir)
-                  file=sys.stderr)
             continue
         last_compile_version = compiled_version
         pass
 
     if not checked_dirs:
-        print("No directories found to check", file=sys.stderr)
+        sys.stderr.write("No directories found to check\n")
         sys.exit(1)
 
     test_opts['compiled_version'] = last_compile_version

uncompyle6/__init__.py

@@ -41,7 +41,9 @@ PYTHON_VERSION_STR  = "%s.%s" % (sys.version_info[0], sys.version_info[1])
 
 IS_PYPY = '__pypy__' in sys.builtin_module_names
 
-sys.setrecursionlimit(5000)
+if hasattr(sys, 'setrecursionlimit'):
+    # pyston doesn't have setrecursionlimit
+    sys.setrecursionlimit(5000)
 
 import uncompyle6.semantics.pysource
 import uncompyle6.semantics.fragments

uncompyle6/bin/pydisassemble.py

@@ -3,7 +3,6 @@
 #
 # Copyright (c) 2015-2016 by Rocky Bernstein <rb@dustyfeet.com>
 #
-from __future__ import print_function
 import sys, os, getopt
 
 from uncompyle6.disas import disassemble_file
@@ -26,7 +25,7 @@ Options:
   -V | --version     show version and stop
   -h | --help        show this message
 
-""".format(program)
+""" % (program, program)
 
 PATTERNS = ('*.pyc', '*.pyo')
 
@@ -37,15 +36,15 @@ Type -h for for full help.""" % program
     native = True
 
     if len(sys.argv) == 1:
-        print("No file(s) given", file=sys.stderr)
+        sys.stderr.write("No file(s) given\n")
-        print(Usage_short, file=sys.stderr)
+        sys.stderr.write(Usage_short)
         sys.exit(1)
 
     try:
         opts, files = getopt.getopt(sys.argv[1:], 'hVU',
                                     ['help', 'version', 'uncompyle6'])
-    except getopt.GetoptError as e:
+    except getopt.GetoptError(e):
-        print('%s: %s' % (os.path.basename(sys.argv[0]), e),  file=sys.stderr)
+        sys.stderr.write('%s: %s' % (os.path.basename(sys.argv[0]), e))
         sys.exit(-1)
 
     for opt, val in opts:
@@ -59,15 +58,14 @@ Type -h for for full help.""" % program
             native = False
         else:
             print(opt)
-            print(Usage_short, file=sys.stderr)
+            sys.stderr.write(Usage_short)
             sys.exit(1)
 
     for file in files:
         if os.path.exists(files[0]):
             disassemble_file(file, sys.stdout, native)
         else:
-            print("Can't read %s - skipping" % files[0],
+            sys.stderr.write("Can't read %s - skipping\n" % files[0])
-                  file=sys.stderr)
             pass
         pass
     return

uncompyle6/bin/uncompile.py

@@ -4,7 +4,6 @@
 # Copyright (c) 2015-2016 by Rocky Bernstein
 # Copyright (c) 2000-2002 by hartmut Goebel <h.goebel@crazy-compilers.com>
 #
-from __future__ import print_function
 import sys, os, getopt, time
 
 program, ext = os.path.splitext(os.path.basename(__file__))
@@ -65,11 +64,11 @@ def usage():
 
 
 def main_bin():
-    if not (sys.version_info[0:2] in ((2, 6), (2, 7),
+    if not (sys.version_info[0:2] in ((2, 4), (2, 5), (2, 6), (2, 7),
-                                      (3, 1), (3, 2), (3, 3),
+                                      (3, 2), (3, 3),
                                       (3, 4), (3, 5), (3, 6))):
-        print('Error: %s requires Python 2.6-2.7, or 3.1-3.6' % program,
+        sys.stderr.write('Error: %s requires Python 2.4 2.5 2.6, 2.7, '
-              file=sys.stderr)
+                         '3.2, 3.3, 3.4, 3.5, or 3.6' % program)
         sys.exit(-1)
 
     do_verify = recurse_dirs = False
@@ -84,8 +83,8 @@ def main_bin():
         opts, files = getopt.getopt(sys.argv[1:], 'hagtdrVo:c:p:',
                                     'help asm grammar linemaps recurse timestamp tree '
                                     'verify version showgrammar'.split(' '))
-    except getopt.GetoptError as e:
+    except getopt.GetoptError(e):
-        print('%s: %s' % (os.path.basename(sys.argv[0]), e),  file=sys.stderr)
+        sys.stderr.write('%s: %s\n' % (os.path.basename(sys.argv[0]), e))
         sys.exit(-1)
 
     options = {}
@@ -119,7 +118,7 @@ def main_bin():
         elif opt in ('--recurse', '-r'):
             recurse_dirs = True
         else:
-            print(opt, file=sys.stderr)
+            sys.stderr.write(opt)
             usage()
 
     # expand directory if specified
@@ -144,7 +143,7 @@ def main_bin():
         files = [f[sb_len:] for f in files]
 
     if not files:
-        print("No files given", file=sys.stderr)
+        sys.stderr.write("No files given\n")
         usage()
 
     if outfile == '-':

uncompyle6/disas.py

@@ -16,8 +16,6 @@ Second, we need structured instruction information for the
 want to run on Python 2.7.
 """
 
-from __future__ import print_function
-
 import sys
 from collections import deque
 
@@ -37,10 +35,9 @@ def disco(version, co, out=None, is_pypy=False):
 
     # store final output stream for case of error
     real_out = out or sys.stdout
-    print('# Python %s' % version, file=real_out)
+    real_out.write('# Python %s\n' % version)
     if co.co_filename:
-        print('# Embedded file name: %s' % co.co_filename,
+        real_out.write('# Embedded file name: %s\n' % co.co_filename)
-              file=real_out)
 
     scanner = get_scanner(version, is_pypy=is_pypy)
 
@@ -52,16 +49,15 @@ def disco_loop(disasm, queue, real_out):
     while len(queue) > 0:
         co = queue.popleft()
         if co.co_name != '<module>':
-            print('\n# %s line %d of %s' %
+            real_out.write('\n# %s line %d of %s\n' %
-                      (co.co_name, co.co_firstlineno, co.co_filename),
+                      (co.co_name, co.co_firstlineno, co.co_filename))
-                      file=real_out)
         tokens, customize = disasm(co)
         for t in tokens:
             if iscode(t.pattr):
                 queue.append(t.pattr)
             elif iscode(t.attr):
                 queue.append(t.attr)
-            print(t, file=real_out)
+            real_out.write(t)
             pass
         pass
 

uncompyle6/linenumbers.py

@@ -10,7 +10,7 @@ def line_number_mapping(pyc_filename, src_filename):
      source_size) = load_module(pyc_filename)
     try:
         code2 = load_file(src_filename)
-    except SyntaxError as e:
+    except SyntaxError, e:
         return str(e)
 
     queue = deque([code1, code2])

uncompyle6/main.py

@@ -1,4 +1,3 @@
-from __future__ import print_function
 import datetime, os, subprocess, sys, tempfile
 
 from uncompyle6 import verify, IS_PYPY
@@ -22,31 +21,36 @@ def decompile(
 
     # store final output stream for case of error
     real_out = out or sys.stdout
-    co_pypy_str = 'PyPy ' if is_pypy else ''
+    if is_pypy:
-    run_pypy_str = 'PyPy ' if IS_PYPY else ''
+        co_pypy_str = 'PyPy '
-    print('# uncompyle6 version %s\n'
+    else:
-          '# %sPython bytecode %s%s\n# Decompiled from: %sPython %s' %
+        co_pypy_str = ''
-          (VERSION, co_pypy_str, bytecode_version,
-           " (%d)" % magic_int if magic_int else "",
-           run_pypy_str, '\n# '.join(sys.version.split('\n'))),
-           file=real_out)
-    if co.co_filename:
-        print('# Embedded file name: %s' % co.co_filename,
-              file=real_out)
-    if timestamp:
-        print('# Compiled at: %s' % datetime.datetime.fromtimestamp(timestamp),
-              file=real_out)
-    if source_size:
-        print('# Size of source mod 2**32: %d bytes' % source_size,
-               file=real_out)
 
-    try:
+    if IS_PYPY:
-        pysource.deparse_code(bytecode_version, co, out, showasm, showast,
+        run_pypy_str = 'PyPy '
-                              showgrammar, code_objects=code_objects,
+    else:
-                              is_pypy=is_pypy)
+        run_pypy_str = ''
-    except pysource.SourceWalkerError as e:
+
-        # deparsing failed
+    if magic_int:
-        raise pysource.SourceWalkerError(str(e))
+        m = str(magic_int)
+    else:
+        m = ""
+    real_out.write('# uncompyle6 version %s\n'
+          '# %sPython bytecode %s%s\n# Decompiled from: %sPython %s\n' %
+          (VERSION, co_pypy_str, bytecode_version,
+           " (%s)" % m, run_pypy_str,
+           '\n# '.join(sys.version.split('\n'))))
+    if co.co_filename:
+        real_out.write('# Embedded file name: %s\n' % co.co_filename)
+    if timestamp:
+        real_out.write('# Compiled at: %s\n' %
+                       datetime.datetime.fromtimestamp(timestamp))
+    if source_size:
+        real_out.write('# Size of source mod 2**32: %d bytes\n' % source_size)
+
+    pysource.deparse_code(bytecode_version, co, out, showasm, showast,
+                          showgrammar, code_objects=code_objects,
+                          is_pypy=is_pypy)
 
 # For compatiblity
 uncompyle = decompile
@@ -127,7 +131,10 @@ def main(in_base, out_base, files, codes, outfile=None,
                 junk, outfile = tempfile.mkstemp(suffix=".py",
                                              prefix=prefix)
                 # Unbuffer output if possible
-                buffering = -1 if sys.stdout.isatty() else 0
+                if sys.stdout.isatty():
+                    buffering = -1
+                else:
+                    buffering = 0
                 sys.stdout = os.fdopen(sys.stdout.fileno(), 'w', buffering)
                 tee = subprocess.Popen(["tee", outfile], stdin=subprocess.PIPE)
                 os.dup2(tee.stdin.fileno(), sys.stdout.fileno())
@@ -144,9 +151,9 @@ def main(in_base, out_base, files, codes, outfile=None,
         try:
             decompile_file(infile, outstream, showasm, showast, showgrammar)
             tot_files += 1
-        except (ValueError, SyntaxError, ParserError, pysource.SourceWalkerError) as e:
+        except (ValueError, SyntaxError, ParserError, pysource.SourceWalkerError):
             sys.stdout.write("\n")
-            sys.stderr.write("\n# file %s\n# %s\n" % (infile, e))
+            sys.stderr.write("# file %s\n" % (infile))
             failed_files += 1
         except KeyboardInterrupt:
             if outfile:
@@ -180,31 +187,35 @@ def main(in_base, out_base, files, codes, outfile=None,
                         msg = verify.compare_code_with_srcfile(infile, outfile, weak_verify=weak_verify)
                         if not outfile:
                             if not msg:
-                                print('\n# okay decompiling %s' % infile)
+                                print '\n# okay decompiling %s' % infile
                                 okay_files += 1
                             else:
-                                print('\n# %s\n\t%s', infile, msg)
+                                print '\n# %s\n\t%s', infile, msg
-                    except verify.VerifyCmpError as e:
+                    except verify.VerifyCmpError, e:
                         print(e)
                         verify_failed_files += 1
                         os.rename(outfile, outfile + '_unverified')
                         sys.stderr.write("### Error Verifying %s\n" % filename)
                         sys.stderr.write(str(e) + "\n")
                         if not outfile:
+                            sys.stder.write("### Error Verifiying %s" %
+                                            filename)
+                            sys.stderr.write(e)
                             if raise_on_error:
                                 raise
                             pass
                         pass
                 pass
             elif do_verify:
-                sys.stderr.write("\n### uncompile successful, but no file to compare against\n")
+                sys.stderr.write("\n### uncompile successful, "
+                                 "but no file to compare against")
                 pass
             else:
                 okay_files += 1
                 if not outfile:
                     mess = '\n# okay decompiling'
                     # mem_usage = __memUsage()
-                    print(mess, infile)
+                    print mess, infile
         if outfile:
             sys.stdout.write("%s\r" %
                              status_msg(do_verify, tot_files, okay_files, failed_files, verify_failed_files))

uncompyle6/parser.py

@@ -1,4 +1,4 @@
-#  Copyright (c) 2015-2016 Rocky Bernstein
+#  Copyright (c) 2015-2017 Rocky Bernstein
 #  Copyright (c) 2005 by Dan Pascu <dan@windowmaker.org>
 #  Copyright (c) 2000-2002 by hartmut Goebel <h.goebel@crazy-compilers.com>
 #  Copyright (c) 1999 John Aycock
@@ -6,8 +6,6 @@
 Common uncompyle parser routines.
 """
 
-from __future__ import print_function
-
 import sys
 
 from xdis.code import iscode
@@ -28,6 +26,22 @@ nop_func = lambda self, args: None
 
 class PythonParser(GenericASTBuilder):
 
+    def __init__(self, AST, start, debug):
+        super(PythonParser, self).__init__(AST, start, debug)
+        self.collect = [
+            'stmts', 'except_stmts', '_stmts', 'load_attrs',
+            'exprlist', 'kvlist', 'kwargs', 'come_froms', '_come_from',
+            # Python < 3
+            'print_items',
+            # PyPy:
+            'kvlist_n']
+
+    def ast_first_offset(self, ast):
+        if hasattr(ast, 'offset'):
+            return ast.offset
+        else:
+            return self.ast_first_offset(ast[0])
+
     def add_unique_rule(self, rule, opname, count, customize):
         """Add rule to grammar, but only if it hasn't been added previously
            opname and count are used in the customize() semantic the actions
@@ -76,7 +90,10 @@ class PythonParser(GenericASTBuilder):
         def fix(c):
             s = str(c)
             i = s.find('_')
-            return s if i == -1 else s[:i]
+            if i == -1:
+                return s
+            else:
+                return s[:i]
 
         prefix = ''
         if parent and tokens:
@@ -107,7 +124,10 @@ class PythonParser(GenericASTBuilder):
         err_token = instructions[index]
         print("Instruction context:")
         for i in range(start, finish):
-            indent = '   ' if i != index else '-> '
+            if i != index:
+                indent = '   '
+            else:
+                indent = '-> '
             print("%s%s" % (indent, instructions[i]))
         raise ParserError(err_token, err_token.offset)
 
@@ -115,11 +135,7 @@ class PythonParser(GenericASTBuilder):
         return token.type
 
     def nonterminal(self, nt, args):
-        collect = ('stmts', 'exprlist', 'kvlist', '_stmts', 'print_items', 'kwargs',
+        if nt in self.collect and len(args) > 1:
-                   # PYPY:
-                   'kvlist_n')
-
-        if nt in collect and len(args) > 1:
             #
             #  Collect iterated thingies together. That is rather than
             #  stmts -> stmts stmt -> stmts stmt -> ...
@@ -389,8 +405,7 @@ class PythonParser(GenericASTBuilder):
         import_cont ::= LOAD_CONST LOAD_CONST import_as_cont
         import_as_cont ::= IMPORT_FROM designator
 
-        load_attrs ::= LOAD_ATTR
+        load_attrs ::= LOAD_ATTR+
-        load_attrs ::= load_attrs LOAD_ATTR
         """
 
     def p_list_comprehension(self, args):
@@ -497,8 +512,12 @@ class PythonParser(GenericASTBuilder):
         expr ::= conditional
         conditional ::= expr jmp_false expr JUMP_FORWARD expr COME_FROM
         conditional ::= expr jmp_false expr JUMP_ABSOLUTE expr
+
         expr ::= conditionalnot
-        conditionalnot ::= expr jmp_true expr _jump expr COME_FROM
+        conditionalnot  ::= expr jmp_true expr _jump expr COME_FROM
+
+        expr            ::= conditionalTrue
+        conditionalTrue ::= expr JUMP_FORWARD expr COME_FROM
 
         ret_expr ::= expr
         ret_expr ::= ret_and

uncompyle6/parsers/parse2.py

@@ -12,8 +12,6 @@ If we succeed in creating a parse tree, then we have a Python program
 that a later phase can turn into a sequence of ASCII text.
 """
 
-from __future__ import print_function
-
 from uncompyle6.parser import PythonParser, PythonParserSingle, nop_func
 from uncompyle6.parsers.astnode import AST
 from spark_parser import DEFAULT_DEBUG as PARSER_DEFAULT_DEBUG
@@ -44,7 +42,8 @@ class Python2Parser(PythonParser):
         while1stmt     ::= SETUP_LOOP l_stmts     JUMP_BACK COME_FROM
         while1stmt     ::= SETUP_LOOP l_stmts     JUMP_BACK POP_BLOCK COME_FROM
 
-        while1elsestmt ::= SETUP_LOOP l_stmts JUMP_BACK else_suite COME_FROM
+        while1elsestmt ::= SETUP_LOOP l_stmts JUMP_BACK POP_BLOCK else_suite COME_FROM
+        while1elsestmt ::= SETUP_LOOP l_stmts JUMP_BACK           else_suite COME_FROM
 
         exec_stmt ::= expr exprlist DUP_TOP EXEC_STMT
         exec_stmt ::= expr exprlist EXEC_STMT
@@ -126,6 +125,7 @@ class Python2Parser(PythonParser):
         assert_expr_and ::= assert_expr jmp_false expr
 
         ifstmt ::= testexpr _ifstmts_jump
+        ifstmt ::= testexpr return_if_stmts COME_FROM
 
         testexpr ::= testfalse
         testexpr ::= testtrue
@@ -144,6 +144,8 @@ class Python2Parser(PythonParser):
 
         ifelsestmtr ::= testexpr return_if_stmts return_stmts
 
+        ifelsestmtr ::= testexpr return_if_stmts COME_FROM return_stmts
+
         ifelsestmtl ::= testexpr c_stmts_opt JUMP_BACK else_suitel
 
 

uncompyle6/parsers/parse23.py

@@ -1,4 +1,4 @@
-#  Copyright (c) 2016 Rocky Bernstein
+#  Copyright (c) 2016-2017 Rocky Bernstein
 #  Copyright (c) 2000-2002 by hartmut Goebel <hartmut@goebel.noris.de>
 #  Copyright (c) 1999 John Aycock
 
@@ -14,6 +14,17 @@ class Python23Parser(Python24Parser):
 
     def p_misc23(self, args):
         '''
+        # Python 2.4 only adds something like the below for if 1:
+        # However we will just treat it as a noop (which of course messes up
+        # simple verify of bytecode.
+        # See also below in reduce_is_invalid where we check that the JUMP_FORWARD
+        # target matches the COME_FROM target
+        stmt     ::= if1_stmt
+        if1_stmt ::= JUMP_FORWARD JUMP_IF_FALSE THEN POP_TOP COME_FROM
+                     stmts
+                     JUMP_FORWARD COME_FROM POP_TOP COME_FROM
+
+
         # Used to keep semantic positions the same across later versions
         # of Python
         _while1test ::= SETUP_LOOP JUMP_FORWARD JUMP_IF_FALSE POP_TOP COME_FROM
@@ -33,6 +44,23 @@ class Python23Parser(Python24Parser):
         lc_body ::= LOAD_FAST expr LIST_APPEND
         '''
 
+    def add_custom_rules(self, tokens, customize):
+        super(Python23Parser, self).add_custom_rules(tokens, customize)
+
+    def reduce_is_invalid(self, rule, ast, tokens, first, last):
+        invalid = super(Python24Parser,
+                        self).reduce_is_invalid(rule, ast,
+                                                tokens, first, last)
+        if invalid:
+            return invalid
+
+        # FiXME: this code never gets called...
+        lhs = rule[0]
+        if lhs == 'nop_stmt':
+            return not int(tokens[first].pattr) == tokens[last].offset
+
+        return False
+
 class Python23ParserSingle(Python23Parser, PythonParserSingle):
     pass
 

uncompyle6/parsers/parse24.py

@@ -14,6 +14,15 @@ class Python24Parser(Python25Parser):
 
     def p_misc24(self, args):
         '''
+        # Python 2.4 only adds something like the below for if 1:
+        # However we will just treat it as a noop (which of course messes up
+        # simple verify of bytecode.
+        # See also below in reduce_is_invalid where we check that the JUMP_FORWARD
+        # target matches the COME_FROM target
+        stmt     ::= nop_stmt
+        nop_stmt ::= JUMP_FORWARD POP_TOP COME_FROM
+
+
         # 2.5+ has two LOAD_CONSTs, one for the number '.'s in a relative import
         # keep positions similar to simplify semantic actions
 
@@ -37,6 +46,25 @@ class Python24Parser(Python25Parser):
         gen_comp_body ::= expr YIELD_VALUE
         '''
 
+    def add_custom_rules(self, tokens, customize):
+        super(Python24Parser, self).add_custom_rules(tokens, customize)
+        if self.version == 2.4:
+            self.check_reduce['nop_stmt'] = 'tokens'
+
+    def reduce_is_invalid(self, rule, ast, tokens, first, last):
+        invalid = super(Python24Parser,
+                        self).reduce_is_invalid(rule, ast,
+                                                tokens, first, last)
+        if invalid:
+            return invalid
+
+        # FiXME: this code never gets called...
+        lhs = rule[0]
+        if lhs == 'nop_stmt':
+            return not int(tokens[first].pattr) == tokens[last].offset
+
+        return False
+
 class Python24ParserSingle(Python24Parser, PythonParserSingle):
     pass
 

uncompyle6/parsers/parse25.py

@@ -1,4 +1,4 @@
-#  Copyright (c) 2016 Rocky Bernstein
+#  Copyright (c) 2016-2017 Rocky Bernstein
 """
 spark grammar differences over Python2.6 for Python 2.5.
 """
@@ -20,13 +20,18 @@ class Python25Parser(Python26Parser):
         return_if_stmt ::= ret_expr  RETURN_END_IF JUMP_BACK
 
         # Python 2.6 uses ROT_TWO instead of the STORE_xxx
+        # withas is allowed as a "from future" in 2.5
         setupwithas ::= DUP_TOP LOAD_ATTR store LOAD_ATTR CALL_FUNCTION_0
                         setup_finally
 
         store ::= STORE_FAST
         store ::= STORE_NAME
 
-        # Python 2.6 omits ths LOAD_FAST DELETE_FAST below
+        tryelsestmt    ::= SETUP_EXCEPT suite_stmts_opt POP_BLOCK
+                           try_middle else_suite COME_FROM
+
+        # Python 2.6 omits the LOAD_FAST DELETE_FAST below
+        # withas is allowed as a "from future" in 2.5
         withasstmt ::= expr setupwithas designator suite_stmts_opt
                        POP_BLOCK LOAD_CONST COME_FROM
                        with_cleanup
@@ -46,18 +51,6 @@ class Python25Parser(Python26Parser):
                                                 tokens, first, last)
         if invalid:
             return invalid
-        lhs = rule[0]
-        if lhs in ('tryelsestmt', ):
-            # The end of the else part of try/else come_from has to come
-            # from an END_FINALLY statement
-            if tokens[last-1].type.startswith('COME_FROM'):
-                end_finally_offset = int(tokens[last-1].pattr)
-                current  = first
-                while current < last:
-                    offset = tokens[current].offset
-                    if offset == end_finally_offset:
-                        return tokens[current].type != 'END_FINALLY'
-                    current += 1
         return False
 
 

uncompyle6/parsers/parse26.py

@@ -1,4 +1,4 @@
-#  Copyright (c) 2016 Rocky Bernstein
+#  Copyright (c) 2017 Rocky Bernstein
 """
 spark grammar differences over Python2 for Python 2.6.
 """
@@ -22,25 +22,22 @@ class Python26Parser(Python2Parser):
                          JUMP_IF_FALSE POP_TOP POP_TOP designator POP_TOP
 
         try_middle   ::= JUMP_FORWARD COME_FROM except_stmts
-                         come_from_pop END_FINALLY COME_FROM
+                         come_from_pop END_FINALLY come_froms
+
+        try_middle   ::= JUMP_FORWARD COME_FROM except_stmts END_FINALLY
+                         come_froms
 
         try_middle   ::= jmp_abs COME_FROM except_stmts
                          POP_TOP END_FINALLY
 
         try_middle   ::= jmp_abs COME_FROM except_stmts
-                         POP_TOP END_FINALLY
+                         END_FINALLY JUMP_FORWARD
-
-        trystmt      ::= SETUP_EXCEPT suite_stmts_opt POP_TOP
-                         try_middle
 
         # Sometimes we don't put in COME_FROM to the next statement
         # like we do in 2.7. Perhaps we should?
         trystmt      ::= SETUP_EXCEPT suite_stmts_opt POP_BLOCK
                          try_middle
 
-        trystmt      ::= SETUP_EXCEPT suite_stmts_opt POP_BLOCK
-                         try_middle come_froms
-
         tryelsestmt    ::= SETUP_EXCEPT suite_stmts_opt POP_BLOCK
                            try_middle else_suite COME_FROM
 
@@ -151,6 +148,8 @@ class Python26Parser(Python2Parser):
         iflaststmtl ::= testexpr c_stmts_opt JUMP_BACK come_from_pop
         iflaststmt  ::= testexpr c_stmts_opt JUMP_ABSOLUTE come_from_pop
 
+        lastc_stmt ::= iflaststmt COME_FROM
+
         while1stmt ::= SETUP_LOOP l_stmts_opt JUMP_BACK COME_FROM
 
         ifstmt         ::= testexpr_then _ifstmts_jump
@@ -158,7 +157,6 @@ class Python26Parser(Python2Parser):
         # Semantic actions want the else to be at position 3
         ifelsestmt     ::= testexpr      c_stmts_opt jf_cf_pop else_suite come_froms
         ifelsestmt     ::= testexpr_then c_stmts_opt jf_cf_pop else_suite come_froms
-        ifelsestmt     ::= testexpr      c_stmts_opt filler else_suitel come_froms POP_TOP
         ifelsestmt     ::= testexpr_then c_stmts_opt filler else_suitel come_froms POP_TOP
 
         # Semantic actions want else_suitel to be at index 3

uncompyle6/parsers/parse27.py

@@ -1,4 +1,4 @@
-#  Copyright (c) 2016 Rocky Bernstein
+#  Copyright (c) 2016-2017 Rocky Bernstein
 #  Copyright (c) 2005 by Dan Pascu <dan@windowmaker.org>
 #  Copyright (c) 2000-2002 by hartmut Goebel <hartmut@goebel.noris.de>
 
@@ -61,7 +61,7 @@ class Python27Parser(Python2Parser):
 
         ret_and  ::= expr JUMP_IF_FALSE_OR_POP ret_expr_or_cond COME_FROM
         ret_or   ::= expr JUMP_IF_TRUE_OR_POP ret_expr_or_cond COME_FROM
-        ret_cond ::= expr POP_JUMP_IF_FALSE expr RETURN_END_IF ret_expr_or_cond
+        ret_cond ::= expr POP_JUMP_IF_FALSE expr RETURN_END_IF COME_FROM ret_expr_or_cond
         ret_cond_not ::= expr POP_JUMP_IF_TRUE expr RETURN_END_IF ret_expr_or_cond
 
         or   ::= expr JUMP_IF_TRUE_OR_POP expr COME_FROM
@@ -96,6 +96,27 @@ class Python27Parser(Python2Parser):
         while1stmt ::= SETUP_LOOP return_stmts COME_FROM
         """
 
+    def add_custom_rules(self, tokens, customize):
+        super(Python27Parser, self).add_custom_rules(tokens, customize)
+        self.check_reduce['and'] = 'AST'
+        return
+
+    def reduce_is_invalid(self, rule, ast, tokens, first, last):
+        invalid = super(Python27Parser,
+                        self).reduce_is_invalid(rule, ast,
+                                                tokens, first, last)
+        if invalid:
+            return invalid
+        if rule == ('and', ('expr', 'jmp_false', 'expr', '\\e_come_from_opt')):
+            # Test that jmp_false jumps to the end of "and"
+            # or that it jumps to the same place as the end of "and"
+            jmp_false = ast[1][0]
+            jmp_target = jmp_false.offset + jmp_false.attr + 3
+            return not (jmp_target == tokens[last].offset or
+                        tokens[last].pattr == jmp_false.pattr)
+        return False
+
+
 class Python27ParserSingle(Python27Parser, PythonParserSingle):
     pass
 

uncompyle6/parsers/parse3.py

@@ -1,4 +1,4 @@
-#  Copyright (c) 2015, 2016 Rocky Bernstein
+#  Copyright (c) 2015-2017 Rocky Bernstein
 #  Copyright (c) 2005 by Dan Pascu <dan@windowmaker.org>
 #  Copyright (c) 2000-2002 by hartmut Goebel <h.goebel@crazy-compilers.com>
 #  Copyright (c) 1999 John Aycock
@@ -15,8 +15,6 @@ If we succeed in creating a parse tree, then we have a Python program
 that a later phase can turn into a sequence of ASCII text.
 """
 
-from __future__ import print_function
-
 from uncompyle6.parser import PythonParser, PythonParserSingle, nop_func
 from uncompyle6.parsers.astnode import AST
 from spark_parser import DEFAULT_DEBUG as PARSER_DEFAULT_DEBUG
@@ -44,6 +42,10 @@ class Python3Parser(PythonParser):
 
         list_for ::= expr FOR_ITER designator list_iter jb_or_c
 
+        # This is seen in PyPy, but possibly it appears on other Python 3?
+        list_if     ::= expr jmp_false list_iter COME_FROM
+        list_if_not ::= expr jmp_true list_iter COME_FROM
+
         jb_or_c ::= JUMP_BACK
         jb_or_c ::= CONTINUE
 
@@ -52,6 +54,9 @@ class Python3Parser(PythonParser):
         setcomp_func ::= BUILD_SET_0 LOAD_FAST FOR_ITER designator comp_iter
                 JUMP_BACK RETURN_VALUE RETURN_LAST
 
+        setcomp_func ::= BUILD_SET_0 LOAD_FAST FOR_ITER designator comp_iter
+                COME_FROM JUMP_BACK RETURN_VALUE RETURN_LAST
+
         comp_body ::= dict_comp_body
         comp_body ::= set_comp_body
         dict_comp_body ::= expr expr MAP_ADD
@@ -113,9 +118,11 @@ class Python3Parser(PythonParser):
         classdefdeco1 ::= expr classdefdeco1 CALL_FUNCTION_1
         classdefdeco1 ::= expr classdefdeco2 CALL_FUNCTION_1
 
-        assert ::= assert_expr jmp_true LOAD_ASSERT RAISE_VARARGS_1
+        assert ::= assert_expr jmp_true LOAD_ASSERT RAISE_VARARGS_1 COME_FROM
-        assert2 ::= assert_expr jmp_true LOAD_ASSERT expr CALL_FUNCTION_1 RAISE_VARARGS_1
+        assert2 ::= assert_expr jmp_true LOAD_ASSERT expr CALL_FUNCTION_1
-        assert2 ::= assert_expr jmp_true LOAD_ASSERT expr RAISE_VARARGS_2
+                    RAISE_VARARGS_1 COME_FROM
+        assert2 ::= assert_expr jmp_true LOAD_ASSERT expr
+                    RAISE_VARARGS_2 COME_FROM
 
         assert_expr ::= expr
         assert_expr ::= assert_expr_or
@@ -132,6 +139,7 @@ class Python3Parser(PythonParser):
 
         _ifstmts_jump ::= return_if_stmts
         _ifstmts_jump ::= c_stmts_opt JUMP_FORWARD COME_FROM
+        _ifstmts_jump ::= c_stmts_opt COME_FROM
 
         iflaststmt ::= testexpr c_stmts_opt JUMP_ABSOLUTE
 
@@ -139,22 +147,25 @@ class Python3Parser(PythonParser):
         iflaststmtl ::= testexpr c_stmts_opt JUMP_BACK COME_FROM_LOOP
 
         # These are used to keep AST indices the same
-        jf_else    ::= JUMP_FORWARD ELSE
+        jump_forward_else  ::= JUMP_FORWARD ELSE
-        ja_else    ::= JUMP_ABSOLUTE ELSE
+        jump_absolute_else ::= JUMP_ABSOLUTE ELSE
 
         # Note: in if/else kinds of statements, we err on the side
         # of missing "else" clauses. Therefore we include grammar
         # rules with and without ELSE.
 
-        ifelsestmt ::= testexpr c_stmts_opt JUMP_FORWARD else_suite COME_FROM
+        ifelsestmt ::= testexpr c_stmts_opt JUMP_FORWARD else_suite opt_come_from_except
-        ifelsestmt ::= testexpr c_stmts_opt jf_else else_suite _come_from
+        ifelsestmt ::= testexpr c_stmts_opt jump_forward_else else_suite _come_from
 
         ifelsestmtc ::= testexpr c_stmts_opt JUMP_ABSOLUTE else_suitec
-        ifelsestmtc ::= testexpr c_stmts_opt ja_else else_suitec
+        ifelsestmtc ::= testexpr c_stmts_opt jump_absolute_else else_suitec
 
         ifelsestmtr ::= testexpr return_if_stmts return_stmts
 
         ifelsestmtl ::= testexpr c_stmts_opt JUMP_BACK else_suitel
+        ifelsestmtl ::= testexpr c_stmts_opt cf_jump_back else_suitel
+
+        cf_jump_back ::= COME_FROM JUMP_BACK
 
         # FIXME: this feels like a hack. Is it just 1 or two
         # COME_FROMs?  the parsed tree for this and even with just the
@@ -169,14 +180,17 @@ class Python3Parser(PythonParser):
                            POP_BLOCK LOAD_CONST
                            come_from_or_finally suite_stmts_opt END_FINALLY
 
+        tryelsestmt    ::= SETUP_EXCEPT suite_stmts_opt POP_BLOCK
+                           try_middle else_suite come_from_except_clauses
+
         tryelsestmt    ::= SETUP_EXCEPT suite_stmts_opt POP_BLOCK
                            try_middle else_suite come_froms
 
         tryelsestmtc ::= SETUP_EXCEPT suite_stmts_opt POP_BLOCK
-                         try_middle else_suitec COME_FROM
+                         try_middle else_suitec come_from_except_clauses
 
         tryelsestmtl ::= SETUP_EXCEPT suite_stmts_opt POP_BLOCK
-                         try_middle else_suitel COME_FROM
+                         try_middle else_suitel come_from_except_clauses
 
         try_middle ::= jmp_abs COME_FROM except_stmts
                        END_FINALLY
@@ -243,7 +257,10 @@ class Python3Parser(PythonParser):
 
     def p_misc3(self, args):
         """
-        try_middle ::= JUMP_FORWARD COME_FROM_EXCEPT except_stmts END_FINALLY COME_FROM
+        try_middle ::= JUMP_FORWARD COME_FROM_EXCEPT except_stmts
+                       END_FINALLY COME_FROM
+        try_middle ::= JUMP_FORWARD COME_FROM_EXCEPT except_stmts
+                       END_FINALLY COME_FROM_EXCEPT_CLAUSE
 
         for_block ::= l_stmts_opt opt_come_from_loop JUMP_BACK
         for_block ::= l_stmts
@@ -258,12 +275,14 @@ class Python3Parser(PythonParser):
         stmt ::= funcdef_annotate
         funcdef_annotate ::= mkfunc_annotate designator
 
+        mkfuncdeco0 ::= mkfunc_annotate
+
         # This has the annotation value.
         # LOAD_NAME is used in an annotation type like
         # int, float, str
         annotate_arg    ::= LOAD_NAME
         # LOAD_CONST is used in an annotation string
-        annotate_arg    ::= LOAD_CONST
+        annotate_arg    ::= expr
 
         # This stores the tuple of parameter names
         # that have been annotated
@@ -274,10 +293,13 @@ class Python3Parser(PythonParser):
         """
         opt_come_from_except ::= COME_FROM_EXCEPT
         opt_come_from_except ::= come_froms
+        opt_come_from_except ::= come_from_except_clauses
 
-        come_froms ::= come_froms COME_FROM
+        come_froms ::= COME_FROM*
-        come_froms ::=
 
+        come_from_except_clauses ::= COME_FROM_EXCEPT_CLAUSE+
+
+        opt_come_from_loop ::= opt_come_from_loop COME_FROM_LOOP
         opt_come_from_loop ::= opt_come_from_loop COME_FROM_LOOP
         opt_come_from_loop ::=
 
@@ -329,6 +351,9 @@ class Python3Parser(PythonParser):
         forelselaststmtl  ::= SETUP_LOOP expr _for designator for_block POP_BLOCK else_suitel
                               COME_FROM_LOOP
 
+        whilestmt         ::= SETUP_LOOP testexpr l_stmts_opt COME_FROM JUMP_BACK POP_BLOCK
+                              COME_FROM_LOOP
+
         whilestmt         ::= SETUP_LOOP testexpr l_stmts_opt JUMP_BACK POP_BLOCK
                               COME_FROM_LOOP
 
@@ -356,6 +381,11 @@ class Python3Parser(PythonParser):
         while1stmt        ::= SETUP_LOOP l_stmts
         while1stmt        ::= SETUP_LOOP l_stmts COME_FROM_LOOP
 
+        while1stmt        ::= SETUP_LOOP l_stmts COME_FROM JUMP_BACK COME_FROM_LOOP
+
+        while1elsestmt    ::= SETUP_LOOP l_stmts JUMP_BACK
+                              else_suite COME_FROM_LOOP
+
         # FIXME: investigate - can code really produce a NOP?
         whileTruestmt     ::= SETUP_LOOP l_stmts_opt JUMP_BACK NOP
                               COME_FROM_LOOP
@@ -376,9 +406,12 @@ class Python3Parser(PythonParser):
 
     def p_expr3(self, args):
         """
-        conditional    ::= expr jmp_false expr jf_else expr COME_FROM
+        conditional    ::= expr jmp_false expr jump_forward_else expr COME_FROM
-        conditionalnot ::= expr jmp_true  expr jf_else expr COME_FROM
+        conditionalnot ::= expr jmp_true  expr jump_forward_else expr COME_FROM
 
+        # a JUMP_FORWARD to another JUMP_FORWARD can get turned into
+        # a JUMP_ABSOLUTE with no COME_FROM
+        conditional    ::= expr jmp_false expr jump_absolute_else expr
 
         expr ::= LOAD_CLASSNAME
 
@@ -436,9 +469,9 @@ class Python3Parser(PythonParser):
     def custom_classfunc_rule(self, opname, token, customize):
         """
         call_function ::= expr {expr}^n CALL_FUNCTION_n
-        call_function ::= expr {expr}^n CALL_FUNCTION_VAR_n POP_TOP
+        call_function ::= expr {expr}^n CALL_FUNCTION_VAR_n
-        call_function ::= expr {expr}^n CALL_FUNCTION_VAR_KW_n POP_TOP
+        call_function ::= expr {expr}^n CALL_FUNCTION_VAR_KW_n
-        call_function ::= expr {expr}^n CALL_FUNCTION_KW_n POP_TOP
+        call_function ::= expr {expr}^n CALL_FUNCTION_KW_n
 
         classdefdeco2 ::= LOAD_BUILD_CLASS mkfunc {expr}^n-1 CALL_FUNCTION_n
         """
@@ -446,31 +479,56 @@ class Python3Parser(PythonParser):
         # high byte number of positional parameters
         args_pos = token.attr & 0xff
         args_kw = (token.attr >> 8) & 0xff
+
+        # Additional exprs for * and ** args:
+        #  0 if neither
+        #  1 for CALL_FUNCTION_VAR or CALL_FUNCTION_KW
+        #  2 for * and ** args (CALL_FUNCTION_VAR_KW).
+        # Yes, this computation based on instruction name is a little bit hoaky.
         nak = ( len(opname)-len('CALL_FUNCTION') ) // 3
+
         token.type = self.call_fn_name(token)
+        uniq_param = args_kw + args_pos
+        if self.version == 3.5 and opname.startswith('CALL_FUNCTION_VAR'):
+            # Python 3.5 changes the stack position of where * args, the
+            # first LOAD_FAST, below are located.
+            # Python 3.6+ replaces CALL_FUNCTION_VAR_KW with CALL_FUNCTION_EX
+            if opname.endswith('KW'):
+                kw = 'LOAD_FAST '
+            else:
+                kw = ''
+            rule = ('call_function ::= expr expr ' +
+                    ('pos_arg ' * args_pos) +
+                    ('kwarg ' * args_kw) + kw + token.type)
+            self.add_unique_rule(rule, token.type, uniq_param, customize)
+
         rule = ('call_function ::= expr ' +
                 ('pos_arg ' * args_pos) +
                 ('kwarg ' * args_kw) +
                 'expr ' * nak + token.type)
-        self.add_unique_rule(rule, token.type, args_pos, customize)
+
+        self.add_unique_rule(rule, token.type, uniq_param, customize)
         if self.version >= 3.5:
             rule = ('async_call_function ::= expr ' +
                     ('pos_arg ' * args_pos) +
                     ('kwarg ' * args_kw) +
                     'expr ' * nak + token.type +
                     ' GET_AWAITABLE LOAD_CONST YIELD_FROM')
-            self.add_unique_rule(rule, token.type, args_pos, customize)
+            self.add_unique_rule(rule, token.type, uniq_param, customize)
-            self.add_unique_rule('expr ::= async_call_function', token.type, args_pos, customize)
+            self.add_unique_rule('expr ::= async_call_function', token.type, uniq_param, customize)
 
         rule = ('classdefdeco2 ::= LOAD_BUILD_CLASS mkfunc %s%s_%d'
                 %  (('expr ' * (args_pos-1)), opname, args_pos))
-        self.add_unique_rule(rule, token.type, args_pos, customize)
+        self.add_unique_rule(rule, token.type, uniq_param, customize)
 
     def add_make_function_rule(self, rule, opname, attr, customize):
         """Python 3.3 added a an addtional LOAD_CONST before MAKE_FUNCTION and
         this has an effect on many rules.
         """
-        new_rule = rule % (('LOAD_CONST ') * (1 if  self.version >= 3.3 else 0))
+        if self.version >= 3.3:
+            new_rule = rule % (('LOAD_CONST ') * 1)
+        else:
+            new_rule = rule % (('LOAD_CONST ') * 0)
         self.add_unique_rule(new_rule, opname, attr, customize)
 
     def add_custom_rules(self, tokens, customize):
@@ -607,15 +665,31 @@ class Python3Parser(PythonParser):
                     self.add_unique_rule(rule, 'kvlist_n', 1, customize)
                     rule = "mapexpr ::=  BUILD_MAP_n kvlist_n"
                 elif self.version >= 3.5:
-                    if  opname != 'BUILD_MAP_WITH_CALL':
+                    if opname != 'BUILD_MAP_WITH_CALL':
-                        rule = kvlist_n + ' ::= ' + 'expr ' * (token.attr*2)
+                        if opname == 'BUILD_MAP_UNPACK':
-                        self.add_unique_rule(rule, opname, token.attr, customize)
+                            rule = kvlist_n + ' ::= ' + 'expr ' * (token.attr*2)
-                        rule = "mapexpr ::=  %s %s" % (kvlist_n, opname)
+                            self.add_unique_rule(rule, opname, token.attr, customize)
+                            rule = 'dict ::= ' + 'expr ' * (token.attr*2)
+                            self.add_unique_rule(rule, opname, token.attr, customize)
+                            rule = 'mapexpr ::= ' + 'dict ' * token.attr
+                            self.add_unique_rule(rule, opname, token.attr, customize)
+                            rule = ('unmap_dict ::= ' +
+                                    ('mapexpr ' * token.attr) +
+                                    ' BUILD_MAP_UNPACK')
+                        else:
+                            rule = kvlist_n + ' ::= ' + 'expr ' * (token.attr*2)
+                            self.add_unique_rule(rule, opname, token.attr, customize)
+                            rule = "mapexpr ::=  %s %s" % (kvlist_n, opname)
                 else:
                     rule = kvlist_n + ' ::= ' + 'expr expr STORE_MAP ' * token.attr
                     self.add_unique_rule(rule, opname, token.attr, customize)
                     rule = "mapexpr ::=  %s %s" % (opname, kvlist_n)
                 self.add_unique_rule(rule, opname, token.attr, customize)
+            elif opname_base == 'BUILD_CONST_KEY_MAP':
+                # This is in 3.6+
+                kvlist_n = 'expr ' * (token.attr)
+                rule = "mapexpr ::= %sLOAD_CONST %s" % (kvlist_n, opname)
+                self.add_unique_rule(rule, opname, token.attr, customize)
             elif opname_base in ('UNPACK_EX',):
                 before_count, after_count = token.attr
                 rule = 'unpack ::= ' + opname + ' designator' * (before_count + after_count + 1)
@@ -632,7 +706,10 @@ class Python3Parser(PythonParser):
                 rule_pat = ("genexpr ::= %sload_genexpr %%s%s expr "
                             "GET_ITER CALL_FUNCTION_1" % ('pos_arg '* args_pos, opname))
                 self.add_make_function_rule(rule_pat, opname, token.attr, customize)
-                rule_pat = ('mklambda ::= %sLOAD_LAMBDA %%s%s' % ('pos_arg '* args_pos, opname))
+                rule_pat = ('mklambda ::= %s%sLOAD_LAMBDA %%s%s' %
+                            (('pos_arg '* args_pos),
+                            ('kwarg '* args_kw),
+                            opname))
                 self.add_make_function_rule(rule_pat, opname, token.attr, customize)
                 rule_pat  = ("listcomp ::= %sLOAD_LISTCOMP %%s%s expr "
                              "GET_ITER CALL_FUNCTION_1" % ('expr ' * args_pos, opname))
@@ -653,9 +730,6 @@ class Python3Parser(PythonParser):
                             ('pos_arg ' * args_pos, opname))
                 self.add_unique_rule(rule, opname, token.attr, customize)
                 if opname.startswith('MAKE_FUNCTION_A'):
-                    # rule = ('mkfunc2 ::= %s%sEXTENDED_ARG %s' %
-                    #         ('pos_arg ' * (args_pos), 'kwargs ' * (annotate_args-1), opname))
-                    self.add_unique_rule(rule, opname, token.attr, customize)
                     if self.version >= 3.3:
                         rule = ('mkfunc_annotate ::= %s%sannotate_tuple LOAD_CONST LOAD_CONST EXTENDED_ARG %s' %
                                 (('pos_arg ' * (args_pos)),
@@ -746,7 +820,8 @@ class Python3Parser(PythonParser):
         elif lhs == 'annotate_tuple':
             return not isinstance(tokens[first].attr, tuple)
         elif lhs == 'kwarg':
-            return not isinstance(tokens[first].attr, str)
+            return not (isinstance(tokens[first].attr, unicode) or
+                        isinstance(tokens[first].attr, str))
         elif lhs == 'while1elsestmt':
             # if SETUP_LOOP target spans the else part, then this is
             # not while1else. Also do for whileTrue?

uncompyle6/parsers/parse30.py

@@ -2,12 +2,11 @@
 """
 spark grammar differences over Python 3.1 for Python 3.0.
 """
-from __future__ import print_function
 
 from uncompyle6.parser import PythonParserSingle
-from uncompyle6.parsers.parse3 import Python3Parser
+from uncompyle6.parsers.parse31 import Python31Parser
 
-class Python30Parser(Python3Parser):
+class Python30Parser(Python31Parser):
 
     def p_30(self, args):
         """
@@ -44,5 +43,10 @@ class Python30Parser(Python3Parser):
         setup_finally ::= STORE_FAST SETUP_FINALLY LOAD_FAST DELETE_FAST
         """
 
+    def add_custom_rules(self, tokens, customize):
+        super(Python30Parser, self).add_custom_rules(tokens, customize)
+        return
+    pass
+
 class Python30ParserSingle(Python30Parser, PythonParserSingle):
     pass

uncompyle6/parsers/parse31.py

@@ -2,7 +2,6 @@
 """
 spark grammar differences over Python 3.2 for Python 3.1.
 """
-from __future__ import print_function
 
 from uncompyle6.parser import PythonParserSingle
 from uncompyle6.parsers.parse32 import Python32Parser

uncompyle6/parsers/parse32.py

@@ -2,8 +2,6 @@
 """
 spark grammar differences over Python 3 for Python 3.2.
 """
-from __future__ import print_function
-
 from uncompyle6.parser import PythonParserSingle
 from uncompyle6.parsers.parse3 import Python3Parser
 
@@ -20,10 +18,17 @@ class Python32Parser(Python3Parser):
         whileTruestmt     ::= SETUP_LOOP return_stmts
                               COME_FROM_LOOP
 
+        try_middle ::= JUMP_FORWARD COME_FROM_EXCEPT except_stmts
+                       END_FINALLY
+
         # Python 3.2+ has more loop optimization that removes
         # JUMP_FORWARD in some cases, and hence we also don't
         # see COME_FROM
         _ifstmts_jump ::= c_stmts_opt
+        _ifstmts_jump ::= c_stmts_opt JUMP_FORWARD _come_from
+
+        stmt           ::= del_deref_stmt
+        del_deref_stmt ::= DELETE_DEREF
         """
     pass
 

uncompyle6/parsers/parse33.py

@@ -2,7 +2,6 @@
 """
 spark grammar differences over Python 3.2 for Python 3.3.
 """
-from __future__ import print_function
 
 from uncompyle6.parser import PythonParserSingle
 from uncompyle6.parsers.parse32 import Python32Parser
@@ -20,7 +19,19 @@ class Python33Parser(Python32Parser):
 
         iflaststmt    ::= testexpr c_stmts_opt33
         c_stmts_opt33 ::= JUMP_BACK JUMP_ABSOLUTE c_stmts_opt
-        _ifstmts_jump ::= c_stmts_opt JUMP_FORWARD _come_from
+
+        # Python 3.5+ has jump optimization to remove the redundant
+        # jump_excepts. But in 3.3 we need them added
+
+        tryelsestmt ::= SETUP_EXCEPT suite_stmts_opt POP_BLOCK
+                        try_middle else_suite
+                        jump_excepts come_from_except_clauses
+
+        trystmt     ::= SETUP_EXCEPT suite_stmts_opt POP_BLOCK
+                        try_middle
+                        jump_excepts come_from_except_clauses
+
+        jump_excepts ::= jump_except+
         """
 
 class Python33ParserSingle(Python33Parser, PythonParserSingle):