Merge branch 'master' into python-2.4

More work is needed. Not done in Python 3 yet

ChangeLog

@@ -1,3 +1,692 @@
+2017-11-26  rocky <rb@dustyfeet.com>
+
+	* : commit bfac9a62602e7640d19c14ed2cb508be3396e101 Author: rocky
+	<rb@dustyfeet.com> Date:   Sun Nov 26 09:25:37 2017 -0500
+
+2017-11-26  rocky <rb@dustyfeet.com>
+
+	* uncompyle6/parsers/parse2.py, uncompyle6/parsers/parse3.py,
+	uncompyle6/semantics/pysource.py: Isolate build_set from build_list
+	in Python 2...  More work is needed. Not done in Python 3 yet
+
+2017-11-26  rocky <rb@dustyfeet.com>
+
+	* test/stdlib/runtests.sh: stdlib/runtess.sh python version
+	detection
+
+2017-11-26  rocky <rb@dustyfeet.com>
+
+	* pytest/test_grammar.py, test/simple_source/bug22/01_kv.py,
+	test/simple_source/bug32/05_while_true_break.py: Work around grammar
+	remove rule bug...  And reinstate source to a current 3.3 bug (which we don't detect).
+	But at least it is noted for future work.
+
+2017-11-26  rocky <rb@dustyfeet.com>
+
+	* uncompyle6/parser.py, uncompyle6/parsers/parse22.py,
+	uncompyle6/parsers/parse24.py, uncompyle6/parsers/parse25.py,
+	uncompyle6/parsers/parse26.py, uncompyle6/parsers/parse27.py,
+	uncompyle6/parsers/parse32.py: Isolaate kv, kv2 and kv3 better
+
+2017-11-26  rocky <rb@dustyfeet.com>
+
+	* uncompyle6/parser.py, uncompyle6/parsers/parse22.py,
+	uncompyle6/parsers/parse24.py, uncompyle6/parsers/parse25.py,
+	uncompyle6/parsers/parse26.py, uncompyle6/parsers/parse27.py,
+	uncompyle6/parsers/parse32.py: Isolate kv, kv2, and kdv3 better
+
+2017-11-26  rocky <rb@dustyfeet.com>
+
+	* uncompyle6/parser.py, uncompyle6/parsers/parse22.py: Localize kv
+
+2017-11-26  rocky <rb@dustyfeet.com>
+
+	* uncompyle6/parsers/parse21.py, uncompyle6/parsers/parse22.py: 
+	localize kv
+
+2017-11-26  rocky <rb@dustyfeet.com>
+
+	* : commit 0965e2cc967ab890dc789cdf840ba5a13dcde486 Author: rocky
+	<rb@dustyfeet.com> Date:   Sun Nov 26 01:26:57 2017 -0500
+
+2017-11-26  rocky <rb@dustyfeet.com>
+
+	* uncompyle6/parser.py, uncompyle6/parsers/parse26.py,
+	uncompyle6/parsers/parse32.py: remove comp_ifnot
+
+2017-11-25  rocky <rb@dustyfeet.com>
+
+	* : Another test
+
+2017-11-25  rocky <rb@dustyfeet.com>
+
+	* uncompyle6/semantics/make_function.py: Merge hell
+
+2017-11-25  rocky <rb@dustyfeet.com>
+
+	* : commit 8c0959de425caf50c1d9606b95679a39724d9f8f Author: rocky
+	<rb@dustyfeet.com> Date:   Sat Nov 25 23:11:27 2017 -0500
+
+2017-11-25  rocky <rb@dustyfeet.com>
+
+	* test/Makefile, test/simple_source/bug27+/05_for_try_except.py,
+	uncompyle6/parsers/parse27.py: 2.7 control-flow bug: except/pass in
+	loop
+
+2017-11-25  rocky <rb@dustyfeet.com>
+
+	* uncompyle6/parsers/parse2.py, uncompyle6/parsers/parse3.py,
+	uncompyle6/semantics/pysource.py: back off of build_slice{2,3}
+	->slice{2,3} There is another slice rule that interferes with this.
+
+2017-11-25  rocky <rb@dustyfeet.com>
+
+	* test/simple_source/slice/02_slice.py, uncompyle6/parser.py,
+	uncompyle6/parsers/parse2.py, uncompyle6/parsers/parse3.py,
+	uncompyle6/scanners/scanner2.py, uncompyle6/scanners/scanner26.py,
+	uncompyle6/semantics/pysource.py: localize 2 and 3 argument
+	BUILD_SLICE...  Nontermninal name matches AST anme now. Add test.
+
+2017-11-25  rocky <rb@dustyfeet.com>
+
+	* test/Makefile: Improve grammar-cover targets
+
+2017-11-25  rocky <rb@dustyfeet.com>
+
+	* pytest/test_grammar.py, test/Makefile, uncompyle6/parser.py,
+	uncompyle6/parsers/parse2.py, uncompyle6/semantics/fragments.py,
+	uncompyle6/semantics/make_function.py,
+	uncompyle6/semantics/pysource.py: remove unpack_list rule and ...  tidy things a bit more
+
+2017-11-24  rocky <rb@dustyfeet.com>
+
+	* test/Makefile: Regularze grammar coverage rules
+
+2017-11-24  rocky <rb@dustyfeet.com>
+
+	* uncompyle6/semantics/make_function.py: Merge hell
+
+2017-11-24  rocky <rb@dustyfeet.com>
+
+	* : commit c9f3838d04f812b693007d6a8542fbb3379f2634 Author: rocky
+	<rb@dustyfeet.com> Date:   Fri Nov 24 21:16:59 2017 -0500
+
+2017-11-24  rocky <rb@dustyfeet.com>
+
+	* test/simple_source/expression/01_float.py,
+	uncompyle6/semantics/pysource.py: Handle inf, +inf, -nan, and nan
+	constants
+
+2017-11-24  rocky <rb@dustyfeet.com>
+
+	* test/simple_source/expression/03_tuple_assign.py,
+	test/simple_source/stmts/04_withas.py,
+	uncompyle6/semantics/consts.py, uncompyle6/semantics/pysource.py: A
+	couple more bugs found running 2.7 stdlib tests
+
+2017-11-24  rocky <rb@dustyfeet.com>
+
+	* : One more (lambda) test
+
+2017-11-24  rocky <rb@dustyfeet.com>
+
+	* test/simple_source/expression/10_lambda.py,
+	uncompyle6/semantics/make_function.py: Fix bug where lambda has a
+	yield in it
+
+2017-11-24  rocky <rb@dustyfeet.com>
+
+	* uncompyle6/parser.py, uncompyle6/semantics/consts.py: cmp ->
+	compare ...  to better match the Python AST name Compare.  This forces us to
+	change compare -> compare_single Relates to Issue #29
+
+2017-11-24  rocky <rb@dustyfeet.com>
+
+	* test/simple_source/bug33/01_triple_compare.py,
+	uncompyle6/parser.py, uncompyle6/parsers/parse2.py,
+	uncompyle6/parsers/parse26.py, uncompyle6/parsers/parse27.py,
+	uncompyle6/parsers/parse3.py, uncompyle6/parsers/parse32.py,
+	uncompyle6/semantics/consts.py: cmp_list -> compare_chained ...  to better match the Python AST name Compare. Relates to Issue #29
+
+2017-11-23  rocky <rb@dustyfeet.com>
+
+	* test/stdlib/README.md, test/stdlib/compile-file.py,
+	test/stdlib/runtests.sh, uncompyle6/parsers/parse2.py: Start another
+	kind of testing...  And fix a bug found in that with 3-way equal
+
+2017-11-23  rocky <rb@dustyfeet.com>
+
+	* : commit 84632bdc78b79f15685c2f9c88489323a7cdc237 Author: rocky
+	<rb@dustyfeet.com> Date:   Thu Nov 23 16:44:07 2017 -0500
+
+2017-11-23  rocky <rb@dustyfeet.com>
+
+	* uncompyle6/parsers/parse3.py: Reduce 3.5 call_aysnc grammar rules
+
+2017-11-23  rocky <rb@dustyfeet.com>
+
+	* test/simple_source/bug32/03_if_try_raise.py,
+	uncompyle6/parsers/parse32.py: Trystmt grammar from 3.3 needed in
+	3.2 Add test to cover this
+
+2017-11-23  rocky <rb@dustyfeet.com>
+
+	* uncompyle6/parsers/parse32.py, uncompyle6/parsers/parse33.py,
+	uncompyle6/parsers/parse35.py: Add triple compare from 3.3 into 3.2
+
+2017-11-23  rocky <rb@dustyfeet.com>
+
+	* uncompyle6/parsers/parse32.py, uncompyle6/parsers/parse33.py: 
+	Improve try else in 3.2...  Grammar from 3.3 is relevant here
+
+2017-11-23  rocky <rb@dustyfeet.com>
+
+	* : commit 184f480bc860b878a13747a8d1d3d6b9c150975c Author: rocky
+	<rb@dustyfeet.com> Date:   Thu Nov 23 12:33:41 2017 -0500
+
+2017-11-23  rocky <rb@dustyfeet.com>
+
+	* uncompyle6/parsers/parse27.py: provisional 2.7 whileelselast rule
+
+2017-11-23  rocky <rb@dustyfeet.com>
+
+	* uncompyle6/parser.py, uncompyle6/parsers/parse26.py,
+	uncompyle6/parsers/parse27.py, uncompyle6/parsers/parse34.py: 
+	grammar reduction of while loops
+
+2017-11-23  rocky <rb@dustyfeet.com>
+
+	* test/simple_source/bug30/01_ops.py, uncompyle6/parser.py,
+	uncompyle6/parsers/parse2.py, uncompyle6/parsers/parse3.py,
+	uncompyle6/parsers/parse32.py: 3.x Grammar reduction of custom
+	rules...  by looking for token patterns GET_ITER CALL_FUNCTION_1
+
+2017-11-23  rocky <rb@dustyfeet.com>
+
+	* test/Makefile, test/simple_source/bug33/01_triple_compare.py,
+	uncompyle6/parser.py, uncompyle6/parsers/parse26.py,
+	uncompyle6/parsers/parse3.py, uncompyle6/parsers/parse32.py,
+	uncompyle6/parsers/parse33.py: Grammar coverage and pruning
+
+2017-11-22  rocky <rb@dustyfeet.com>
+
+	* uncompyle6/parser.py, uncompyle6/parsers/parse3.py: Isotate
+	conditionalnot
+
+2017-11-22  rocky <rb@dustyfeet.com>
+
+	* test/simple_source/expression/02_true_divide.py,
+	uncompyle6/parsers/parse2.py, uncompyle6/parsers/parse25.py,
+	uncompyle6/parsers/parse26.py, uncompyle6/parsers/parse27.py: 2.x
+	Grammar reduction/isolation
+
+2017-11-22  rocky <rb@dustyfeet.com>
+
+	* : commit b00c59bdd79fe965f7eb9b3b650f19b218930f8a Author: rocky
+	<rb@dustyfeet.com> Date:   Wed Nov 22 14:41:23 2017 -0500
+
+2017-11-22  rocky <rb@dustyfeet.com>
+
+	* uncompyle6/parsers/parse2.py, uncompyle6/parsers/parse3.py: Reduce
+	unecessary grammar rules in 2.x
+
+2017-11-22  rocky <rb@dustyfeet.com>
+
+	* test/simple_source/stmts/01_augmented_assign.py,
+	uncompyle6/parsers/parse2.py, uncompyle6/parsers/parse27.py: 
+	Increase grammar coverage
+
+2017-11-22  rocky <rb@dustyfeet.com>
+
+	* test/simple_source/stmts/01_augmented_assign.py,
+	uncompyle6/parsers/parse2.py, uncompyle6/parsers/parse23.py,
+	uncompyle6/parsers/parse27.py: More complete grammar coverage
+
+2017-11-22  rocky <rb@dustyfeet.com>
+
+	* admin-tools/setup-master.sh, admin-tools/setup-python-2.4.sh: 
+	Administrivia: add "git pull"s
+
+2017-11-22  rocky <rb@dustyfeet.com>
+
+	* admin-tools/setup-python-2.4.sh, test/Makefile: Merge hell
+
+2017-11-22  rocky <rb@dustyfeet.com>
+
+	* : commit a8e235de175bbc4e6bd87689c0c60d570dcb71b4 Author: rocky
+	<rb@dustyfeet.com> Date:   Sat Nov 18 11:02:54 2017 -0500
+
+2017-11-18  rocky <rb@dustyfeet.com>
+
+	* pytest/test_grammar.py, uncompyle6/parser.py,
+	uncompyle6/parsers/parse24.py, uncompyle6/semantics/pysource.py: 
+	Grammar cleanup: import_as_cont -> import_as
+
+2017-11-18  rocky <rb@dustyfeet.com>
+
+	* __pkginfo__.py, pytest/test_grammar.py: Track spark_parser changes
+
+2017-11-17  rocky <rb@dustyfeet.com>
+
+	* test/Makefile, uncompyle6/parsers/parse3.py: custom rule hacking
+	...  Reduce extraneous 3.x "load_list" and "load_closure" rules
+
+2017-11-17  rocky <rb@dustyfeet.com>
+
+	* uncompyle6/parsers/parse3.py, uncompyle6/parsers/parse33.py: 
+	Python 3 grammar clean up and reorganization
+
+2017-11-17  rocky <rb@dustyfeet.com>
+
+	* test/Makefile, uncompyle6/parser.py,
+	uncompyle6/parsers/parse2.py, uncompyle6/parsers/parse3.py,
+	uncompyle6/parsers/parse36.py: More grammar isolation. narrow custom
+	rules...  Specifically: _mklambda with load_closure is Python2 only Add
+	classdefdeco2 rule only when we have seen a LOAD_BUILD_CLASS
+
+2017-11-16  rocky <rb@dustyfeet.com>
+
+	* uncompyle6/parsers/parse34.py: Python 3.4 while grammar cleanup
+
+2017-11-16  rocky <rb@dustyfeet.com>
+
+	* uncompyle6/parser.py, uncompyle6/parsers/parse3.py: More grammar
+	cleanups
+
+2017-11-16  rocky <rb@dustyfeet.com>
+
+	* uncompyle6/semantics/consts.py: Tag %c nonterminals in more
+	template rules
+
+2017-11-16  rocky <rb@dustyfeet.com>
+
+	* uncompyle6/parsers/parse2.py: Bump copyright
+
+2017-11-16  rocky <rb@dustyfeet.com>
+
+	* uncompyle6/parser.py, uncompyle6/parsers/parse2.py,
+	uncompyle6/parsers/parse3.py: More grammar isolation: assert2 in
+	Python3
+
+2017-11-16  rocky <rb@dustyfeet.com>
+
+	* uncompyle6/parsers/parse25.py, uncompyle6/parsers/parse3.py: 
+	Isolate "assert2" rule
+
+2017-11-16  rocky <rb@dustyfeet.com>
+
+	* uncompyle6/parser.py, uncompyle6/parsers/parse23.py: Isolate
+	"and2" rule to 2.3
+
+2017-11-16  rocky <rb@dustyfeet.com>
+
+	* test/simple_source/stmts/05_with.py,
+	uncompyle6/parsers/parse25.py, uncompyle6/parsers/parse26.py: Python
+	2.5 "with"; Isolate 2.5-2.6 grammar better
+
+2017-11-16  rocky <rb@dustyfeet.com>
+
+	* test/Makefile, test/simple_source/stmts/05_with.py,
+	uncompyle6/parsers/parse25.py, uncompyle6/parsers/parse26.py: Python
+	2.5 "with". isolate 2.5-2.7 grammar better
+
+2017-11-16  rocky <rb@dustyfeet.com>
+
+	* : commit 953cf312dbfa8fa6f25b8f8304d0125fa4a7fa31 Author: rocky
+	<rb@dustyfeet.com> Date:   Thu Nov 16 01:29:49 2017 -0500
+
+2017-11-16  rocky <rb@dustyfeet.com>
+
+	* uncompyle6/parsers/parse26.py, uncompyle6/parsers/parse27.py: More
+	grammar cleanup
+
+2017-11-16  rocky <rb@dustyfeet.com>
+
+	* uncompyle6/scanners/scanner2.py: 2.7 continue-detection bug
+
+2017-11-15  rocky <rb@dustyfeet.com>
+
+	* uncompyle6/parsers/parse2.py, uncompyle6/parsers/parse27.py: More
+	2.7/2.7- grammer separation & cleanup
+
+2017-11-15  rocky <rb@dustyfeet.com>
+
+	* uncompyle6/parsers/parse2.py, uncompyle6/parsers/parse27.py: 
+	Grammar cleanup: separate some 2.7 from 2.7- rules
+
+2017-11-15  rocky <rb@dustyfeet.com>
+
+	* uncompyle6/parser.py: I said, remove dict/setcomp from <= 2.6!
+
+2017-11-15  rocky <rb@dustyfeet.com>
+
+	* uncompyle6/parsers/parse2.py, uncompyle6/parsers/parse27.py,
+	uncompyle6/parsers/parse3.py: grammar cleanup: dict/set
+	comprehensions 2.7+
+
+2017-11-15  rocky <rb@dustyfeet.com>
+
+	* HOW-TO-REPORT-A-BUG.md: Grammar typo
+
+2017-11-15  rocky <rb@dustyfeet.com>
+
+	* HOW-TO-REPORT-A-BUG.md: Grammar typo
+
+2017-11-15  rocky <rb@dustyfeet.com>
+
+	* : One more pypy test
+
+2017-11-15  rocky <rb@dustyfeet.com>
+
+	* : Keep bytecode_2.7 pypy free...  We need to add _another_ pypy directory
+
+2017-11-15  rocky <rb@dustyfeet.com>
+
+	* : del stmt testing on 2.7
+
+2017-11-15  rocky <rb@dustyfeet.com>
+
+	* test/Makefile, uncompyle6/parsers/parse3.py: Stronger 3.4 testing
+	...  parse3.py: remove unused grammar rule
+
+2017-11-15  rocky <rb@dustyfeet.com>
+
+	* test/Makefile, test/simple_source/stmts/10_del.py,
+	test/test_pyenvlib.py, uncompyle6/parsers/parse26.py,
+	uncompyle6/semantics/check_ast.py, uncompyle6/semantics/pysource.py: 
+	Profiling workarounds, more coverage ...  test/Makefile: more grammar checking. Update python versions
+	10_del.pyc add test of DEL_GLOBAL check_ast.py, pysource.py: Profileing workarounds
+
+2017-11-14  rocky <rb@dustyfeet.com>
+
+	* uncompyle6/parsers/parse2.py: Guard around null ast This can happen in profiling
+
+2017-11-13  rocky <rb@dustyfeet.com>
+
+	* HOW-TO-REPORT-A-BUG.md: bug report doc tweak
+
+2017-11-13  rocky <rb@dustyfeet.com>
+
+	* HOW-TO-REPORT-A-BUG.md: add note on illegal use
+
+2017-11-13  rocky <rb@dustyfeet.com>
+
+	* ChangeLog: Get ready for release python-2.4-
+
+2017-11-13  rocky <rb@dustyfeet.com>
+
+	* ChangeLog, uncompyle6/parser.py, uncompyle6/semantics/pysource.py: 
+	detected old-style Python 2.4 class better
+
+2017-11-13  rocky <rb@dustyfeet.com>
+
+	* uncompyle6/main.py, uncompyle6/parser.py,
+	uncompyle6/semantics/pysource.py: 2.4isms...  Need print without parens. Handle old-style classes more properly?
+
+2017-11-13  rocky <rb@dustyfeet.com>
+
+	* uncompyle6/scanners/scanner3.py,
+	uncompyle6/semantics/make_function.py: Get ready for release
+	python-2.4-2.13.3
+
+2017-11-13  rocky <rb@dustyfeet.com>
+
+	* : commit 35e4e034686065609b8b2e55cd37c9a391e16c00 Author: rocky
+	<rb@dustyfeet.com> Date:   Mon Nov 13 09:53:10 2017 -0500
+
+2017-11-13  rocky <rb@dustyfeet.com>
+
+	* ChangeLog, NEWS, uncompyle6/version.py: Get ready for release
+	2.13.3
+
+2017-11-12  rocky <rb@dustyfeet.com>
+
+	* test/Makefile: Back off --verify for --weak-verify
+
+2017-11-12  rocky <rb@dustyfeet.com>
+
+	* test/Makefile: Back off --verify for --weak-verify
+
+2017-11-12  rocky <rb@dustyfeet.com>
+
+	* : Reinstate previously failed tests 2.6, 3.5 and 3.6 decompilation has gotten better
+
+2017-11-10  rocky <rb@dustyfeet.com>
+
+	* __pkginfo__.py: Use newer xdis
+
+2017-11-09  rocky <rb@dustyfeet.com>
+
+	* uncompyle6/parsers/parse36.py, uncompyle6/semantics/pysource.py: 
+	Fix bug in return-optimized try stmt
+
+2017-11-09  rocky <rb@dustyfeet.com>
+
+	* HOW-TO-REPORT-A-BUG.md: More detail is needed in bug reporting...  sigh.
+
+2017-11-09  rocky <rb@dustyfeet.com>
+
+	* test/simple_source/bug35/04_importlist.py, uncompyle6/parser.py,
+	uncompyle6/semantics/consts.py: bug in 3.x importlists consts.py: add rule for importlists. imports weren't separated by ',
+	'.  parser.py: Make importlist a list type of node.  test/* add test for importlist
+
+2017-11-08  rocky <rb@dustyfeet.com>
+
+	* : commit e9b60ddbf020ee7f14d8d77c6f4a8588d2968377 Author: rocky
+	<rb@dustyfeet.com> Date:   Wed Nov 8 23:05:01 2017 -0500
+
+2017-11-08  rocky <rb@dustyfeet.com>
+
+	* HOW-TO-REPORT-A-BUG.md: more wordsmithing
+
+2017-11-08  rocky <rb@dustyfeet.com>
+
+	* HOW-TO-REPORT-A-BUG.md: more wordsmithing
+
+2017-11-08  rocky <rb@dustyfeet.com>
+
+	* HOW-TO-REPORT-A-BUG.md: more wordsmithing
+
+2017-11-08  rocky <rb@dustyfeet.com>
+
+	* HOW-TO-REPORT-A-BUG.md: Typo
+
+2017-11-08  rocky <rb@dustyfeet.com>
+
+	* HOW-TO-REPORT-A-BUG.md: Typo
+
+2017-11-08  rocky <rb@dustyfeet.com>
+
+	* HOW-TO-REPORT-A-BUG.md: Typo
+
+2017-11-08  rocky <rb@dustyfeet.com>
+
+	* HOW-TO-REPORT-A-BUG.md: Typo
+
+2017-11-08  rocky <rb@dustyfeet.com>
+
+	* HOW-TO-REPORT-A-BUG.md: Tweak how to report a bug.
+
+2017-11-08  rocky <rb@dustyfeet.com>
+
+	* uncompyle6/parser.py, uncompyle6/parsers/parse36.py,
+	uncompyle6/scanners/scanner3.py: Add 3.6+ grammar for except's
+	ending in RETURN...  Not totally out of the maze in 3.6 control flow...  There are still
+	problems with erroneous RETURN_VALUEs becoming RETURN_END_IF,
+
+2017-11-07  R. Bernstein <rocky@users.noreply.github.com>
+
+	* : Merge pull request #135 from rocky/3.6-instruction-refactor 3.6 instruction refactor
+
+2017-11-06  rocky <rb@dustyfeet.com>
+
+	* uncompyle6/scanners/scanner3.py: Small tweaks to sync up better
+	with scanner2.py
+
+2017-11-06  rocky <rb@dustyfeet.com>
+
+	* pytest/test_fjt.py: Remove parts of erroneous 2.7 test for now
+
+2017-11-06  rocky <rb@dustyfeet.com>
+
+	* pytest/test_fjt.py, uncompyle6/scanners/scanner3.py,
+	uncompyle6/scanners/scanner36.py: Fix 3.{3,4} pytest. Remove dup
+	find_jump_targets
+
+2017-11-06  rocky <rb@dustyfeet.com>
+
+	* Makefile, uncompyle6/scanners/scanner3.py,
+	uncompyle6/scanners/scanner36.py: Move refactored find-jump-targets
+	from 3.6 to 3.x
+
+2017-11-06  rocky <rb@dustyfeet.com>
+
+	* test/Makefile, uncompyle6/scanners/scanner3.py,
+	uncompyle6/scanners/scanner36.py: Move refactored ingest from 3.6 to
+	3.x...  We are getting away from working with bytecode in favor of working
+	with full-fledged structured instructions Up next: find_jump_targets()
+
+2017-11-06  rocky <rb@dustyfeet.com>
+
+	* uncompyle6/parsers/parse36.py: awith custom COME_FROMs ...  Now that jump branching has been properly fixed up for EXTENDED_ARG
+	instructions which are more prevalent with wordcode encoding.
+
+2017-11-06  rocky <rb@dustyfeet.com>
+
+	* : commit 9379922c89573972aa387e4f0b9abcba7358d1a3 Author: rocky
+	<rb@dustyfeet.com> Date:   Mon Nov 6 00:38:22 2017 -0500
+
+2017-11-06  rocky <rb@dustyfeet.com>
+
+	* uncompyle6/scanners/scanner36.py: Revert change that should have
+	been in a branch
+
+2017-11-06  rocky <rb@dustyfeet.com>
+
+	* uncompyle6/scanners/scanner2.py,
+	uncompyle6/scanners/scanner26.py, uncompyle6/scanners/scanner3.py,
+	uncompyle6/scanners/scanner36.py: xdis _disassemble->disassemble
+
+2017-11-04  rocky <rb@dustyfeet.com>
+
+	* uncompyle6/semantics/fragments.py,
+	uncompyle6/semantics/make_function.py,
+	uncompyle6/semantics/pysource.py: Add flag to tolerate deparse
+	errors...  and keep going. The fragment parser should ignore errors in nested
+	function definitions
+
+2017-11-04  rocky <rb@dustyfeet.com>
+
+	* uncompyle6/scanner.py, uncompyle6/semantics/fragments.py: Add
+	Python 3.6.3 scanner lookup
+
+2017-11-03  R. Bernstein <rocky@users.noreply.github.com>
+
+	* : Merge pull request #134 from mikemrm/master Corrected python3 import from queue
+
+2017-10-29  rocky <rb@dustyfeet.com>
+
+	* test/simple_source/bug36/10_extended_arg_loop.py,
+	uncompyle6/parsers/parse36.py, uncompyle6/scanners/scanner3.py: 
+	Python 3.6 control flow bug...  Much more is needed, but it's a start
+
+2017-10-29  rocky <rb@dustyfeet.com>
+
+	* uncompyle6/verify.py: In verify, JUMP_BACK is the same as
+	CONTINUE...  at least for now. See FIXME in verify
+
+2017-10-29  rocky <rb@dustyfeet.com>
+
+	* uncompyle6/scanner.py, uncompyle6/scanners/scanner2.py,
+	uncompyle6/scanners/scanner3.py, uncompyle6/scanners/scanner30.py: 
+	Python 3.6-inspired instruction size cleanup Revise and generalize for Python 3.6+ instructions vs < 3.6
+	instuctions.  Used more of the generalized methods in xdis and
+	remove some (but not all) of the magic numbers.  This is a lot of changes, but not all of the refactoring needed.
+	Much crap still remains. Also, there are still bugs in handling 3.6
+	bytecodes.
+
+2017-10-24  rocky <rb@dustyfeet.com>
+
+	* Makefile, __pkginfo__.py: Bump uncompyle. Pypy 5.8.0-beta
+	tolerance
+
+2017-10-13  rocky <rb@dustyfeet.com>
+
+	* test/Makefile, uncompyle6/semantics/consts.py: Tag more semantic
+	actions with nonterminals
+
+2017-10-13  rocky <rb@dustyfeet.com>
+
+	* uncompyle6/parser.py, uncompyle6/semantics/consts.py: More node
+	checking in tables
+
+2017-10-13  rocky <rb@dustyfeet.com>
+
+	* pytest/test_pysource.py, uncompyle6/parser.py,
+	uncompyle6/parsers/parse24.py, uncompyle6/semantics/consts.py,
+	uncompyle6/semantics/fragments.py, uncompyle6/semantics/pysource.py: 
+	Start allowing node names in template engine These are now used to assert we have the right node type.  Simplify import_from
+
+2017-10-13  rocky <rb@dustyfeet.com>
+
+	* HISTORY.md, uncompyle6/semantics/pysource.py: Small changes
+
+2017-10-12  rocky <rb@dustyfeet.com>
+
+	* admin-tools/make-dist-newer.sh, admin-tools/make-dist-older.sh: 
+	Administrivia - generalize shell code
+
+2017-10-12  rocky <rb@dustyfeet.com>
+
+	* : commit e42e3cc230237a448f48d10f3a6e11c8cda5e9c7 Author: rocky
+	<rb@dustyfeet.com> Date:   Thu Oct 12 07:29:52 2017 -0400
+
+2017-10-12  rocky <rb@dustyfeet.com>
+
+	* admin-tools/how-to-make-a-release.md: Update instructions
+
+2017-10-12  rocky <rb@dustyfeet.com>
+
+	* admin-tools/make-dist-newer.sh, admin-tools/make-dist-older.sh: 
+	Administrivia
+
+2017-10-12  rocky <rb@dustyfeet.com>
+
+	* admin-tools/make-dist-newer.sh, admin-tools/make-dist-older.sh: 
+	Merge conflicts
+
+2017-10-12  rocky <rb@dustyfeet.com>
+
+	* admin-tools/how-to-make-a-release.md: Minor
+
+2017-10-12  rocky <rb@dustyfeet.com>
+
+	* admin-tools/how-to-make-a-release.md,
+	admin-tools/pyenv-newer-versions, admin-tools/update-sources.sh: 
+	Sync
+
+2017-10-12  rocky <rb@dustyfeet.com>
+
+	* NEWS, uncompyle6/parser.py, uncompyle6/scanners/tok.py,
+	uncompyle6/semantics/consts.py,
+	uncompyle6/semantics/make_function.py: Update news
+
+2017-10-12  rocky <rb@dustyfeet.com>
+
+	* : commit b1e650a7bd79d58181557c95bc7ad64e1b8387e6 Merge: 491572e
+	717b22b Author: rocky <rb@dustyfeet.com> Date:   Thu Oct 12 06:52:24
+	2017 -0400
+
+2017-10-12  rocky <rb@dustyfeet.com>
+
+	* ChangeLog, Makefile, NEWS, admin-tools/how-to-make-a-release.txt,
+	uncompyle6/version.py: Get ready for release 2.13.2
+
+2017-10-12  rocky <rb@dustyfeet.com>
+
+	* ChangeLog, Makefile, NEWS, admin-tools/how-to-make-a-release.txt,
+	uncompyle6/version.py: Get ready for release 2.13.2
+
 2017-10-11  rocky <rb@dustyfeet.com>
 
 	* admin-tools/check-newer-versions.sh,
@@ -8,6 +697,21 @@
 	uncompyle6/parsers/parse37.py, uncompyle6/scanners/scanner37.py,
 	uncompyle6/version.py: More administrivia
 
+2017-10-11  rocky <rb@dustyfeet.com>
+
+	* admin-tools/check-older-versions.sh,
+	admin-tools/make-dist-newer.sh, admin-tools/make-dist-older.sh,
+	admin-tools/pyenv-newer-versions, admin-tools/pyenv-older-versions,
+	admin-tools/setup-master.sh: Administrivia
+
+2017-10-11  rocky <rb@dustyfeet.com>
+
+	* admin-tools/check-newer-versions.sh,
+	admin-tools/check-older-versions.sh,
+	admin-tools/how-to-make-a-release.txt,
+	admin-tools/make-dist-newer.sh, admin-tools/make-dist-older.sh: 
+	Adminstrivia
+
 2017-10-11  rocky <rb@dustyfeet.com>
 
 	* admin-tools/README.md, admin-tools/check-newer-versions.sh,
@@ -18,10 +722,33 @@
 	admin-tools/setup-master.sh, admin-tools/setup-python-2.4.sh: Some
 	admin tools I use
 
+2017-10-11  rocky <rb@dustyfeet.com>
+
+	* admin-tools/README.md, admin-tools/check-newer-versions.sh,
+	admin-tools/check-older-versions.sh,
+	admin-tools/how-to-make-a-release.txt,
+	admin-tools/make-dist-newer.sh, admin-tools/make-dist-older.sh,
+	admin-tools/pyenv-newer-versions, admin-tools/pyenv-older-versions,
+	admin-tools/setup-master.sh, admin-tools/setup-python-2.4.sh: Some
+	admin tools I use.
+
 2017-10-11  rocky <rb@dustyfeet.com>
 
 	* uncompyle6/parser.py: Remove creaping Python 2.6ism
 
+2017-10-11  rocky <rb@dustyfeet.com>
+
+	* setup.py: remove python_requires
+
+2017-10-11  rocky <rb@dustyfeet.com>
+
+	* uncompyle6/bin/uncompile.py: Program name was incorrect.  uncompile -> uncompyle6
+
+2017-10-11  rocky <rb@dustyfeet.com>
+
+	* Makefile, NEWS, setup.py, uncompyle6/version.py: Administrivia
+	woes
+
 2017-10-10  rocky <rb@dustyfeet.com>
 
 	* pytest/test_grammar.py: Sync with master
@@ -61,6 +788,57 @@
 	* : Merge commit '1d7a3c6444eab5a02d899f789f2a57cfdcbc5a84' into
 	python-2.4
 
+2017-10-10  rocky <rb@dustyfeet.com>
+
+	* ChangeLog, HOW-TO-REPORT-A-BUG.md, NEWS, test/Makefile,
+	uncompyle6/parser.py, uncompyle6/parsers/parse3.py,
+	uncompyle6/scanners/scanner3.py, uncompyle6/semantics/consts.py,
+	uncompyle6/semantics/pysource.py: Get ready for release 2.13.0
+
+2017-10-10  rocky <rb@dustyfeet.com>
+
+	* HOW-TO-REPORT-A-BUG.md, test/Makefile, uncompyle6/parser.py,
+	uncompyle6/parsers/parse3.py, uncompyle6/scanners/scanner3.py,
+	uncompyle6/semantics/consts.py, uncompyle6/semantics/pysource.py: 
+	Improve parse trace. lambda fixes yet again
+
+2017-10-10  rocky <rb@dustyfeet.com>
+
+	* test/simple_source/branching/02_ifelse_lambda.py,
+	uncompyle6/semantics/consts.py: Address dead code in lambda ifelse
+
+2017-10-10  rocky <rb@dustyfeet.com>
+
+	* uncompyle6/parsers/parse24.py, uncompyle6/scanners/scanner3.py: 
+	Misc bugs
+
+2017-10-10  rocky <rb@dustyfeet.com>
+
+	* uncompyle6/parsers/parse24.py, uncompyle6/scanners/scanner3.py: 
+	Misc bugs
+
+2017-10-10  R. Bernstein <rocky@users.noreply.github.com>
+
+	* : Merge pull request #131 from rocky/type2kind-rework Adjust for spark-parser 2.7.0 incompatibilities
+
+2017-10-10  rocky <rb@dustyfeet.com>
+
+	* __pkginfo__.py, pytest/test_grammar.py, pytest/test_pysource.py,
+	uncompyle6/parser.py, uncompyle6/parsers/astnode.py,
+	uncompyle6/parsers/parse2.py, uncompyle6/parsers/parse24.py,
+	uncompyle6/parsers/parse26.py, uncompyle6/parsers/parse27.py,
+	uncompyle6/parsers/parse3.py, uncompyle6/parsers/parse32.py,
+	uncompyle6/parsers/parse34.py, uncompyle6/parsers/parse35.py,
+	uncompyle6/parsers/parse36.py, uncompyle6/parsers/parse37.py,
+	uncompyle6/scanners/scanner22.py, uncompyle6/scanners/scanner26.py,
+	uncompyle6/scanners/scanner27.py, uncompyle6/scanners/scanner3.py,
+	uncompyle6/scanners/tok.py, uncompyle6/semantics/check_ast.py,
+	uncompyle6/semantics/fragments.py,
+	uncompyle6/semantics/make_function.py,
+	uncompyle6/semantics/pysource.py, uncompyle6/verify.py,
+	uncompyle6/version.py: Adjust for spark-parser 2.7.0
+	incompatabilities
+
 2017-10-05  rocky <rb@dustyfeet.com>
 
 	* test/simple_source/branching/02_ifelse_lambda.py: One more test
@@ -72,12 +850,41 @@
 	uncompyle6/parsers/parse3.py, uncompyle6/semantics/consts.py,
 	uncompyle6/semantics/pysource.py: Sync with master
 
+2017-10-05  rocky <rb@dustyfeet.com>
+
+	* : One more test
+
+2017-10-05  rocky <rb@dustyfeet.com>
+
+	* : commit b3359439f94c136619b198beaecbfce1b827d2db Author: rocky
+	<rb@dustyfeet.com> Date:   Thu Oct 5 11:00:55 2017 -0400
+
 2017-10-03  rocky <rb@dustyfeet.com>
 
 	* uncompyle6/parsers/parse2.py, uncompyle6/parsers/parse24.py,
 	uncompyle6/parsers/parse26.py: handle newer parser reduction
 	behavior
 
+2017-10-03  rocky <rb@dustyfeet.com>
+
+	* uncompyle6/parsers/parse2.py, uncompyle6/parsers/parse24.py,
+	uncompyle6/parsers/parse26.py: handle newer parser reduction
+	behavior
+
+2017-10-03  rocky <rb@dustyfeet.com>
+
+	* uncompyle6/semantics/pysource.py: Remove schumutz
+
+2017-10-03  rocky <rb@dustyfeet.com>
+
+	* uncompyle6/semantics/pysource.py: More table doc tweaks
+
+2017-10-03  rocky <rb@dustyfeet.com>
+
+	* uncompyle6/semantics/fragments.py,
+	uncompyle6/semantics/pysource.py: Go over table-semantics
+	description yet again
+
 2017-10-03  rocky <rb@dustyfeet.com>
 
 	* uncompyle6/semantics/fragments.py,
@@ -89,6 +896,11 @@
 	* uncompyle6/parsers/parse2.py, uncompyle6/parsers/parse3.py: Sync
 	with master
 
+2017-10-02  rocky <rb@dustyfeet.com>
+
+	* uncompyle6/parsers/parse2.py, uncompyle6/parsers/parse3.py: 
+	spark-parser induced changes...  reduce rules can be called without token streams.
+
 2017-09-30  rocky <rb@dustyfeet.com>
 
 	* uncompyle6/parser.py, uncompyle6/scanners/scanner2.py,

HISTORY.md

@@ -98,9 +98,20 @@ so. Then hamled made a few commits earler on, while Eike Siewertsen
 made a few commits later on. But mostly wibiti, and Guenther
 Starnberger got the code to where uncompyle2 was around 2012.
 
-In `uncompyle`, decompilation of python bytecode 2.5 & 2.6 is done by
-transforming the byte code into a pseudo-2.7 Python bytecode and is
-based on code from Eloi Vanderbeken.
+While John Aycock and Hartmut Goebel were well versed in compiler
+technology, those that have come afterwards don't seem to have been as
+facile in it.  Furthermore, documentation or guidance on how the
+decompiler code worked, comparison to a conventional compiler
+pipeline, how to add new constructs, or debug grammars was weak. Some
+of the grammar tracing and error reporting was a bit weak as well.
+
+Given this, perhaps it is not surprising that subsequent changes
+tended to shy away from using the built-in compiler technology
+mechanisms and addressed problems and extensions by some other means.
+
+Specifically, in `uncompyle`, decompilation of python bytecode 2.5 & 2.6
+is done by transforming the byte code into a pseudo-2.7 Python
+bytecode and is based on code from Eloi Vanderbeken.
 
 This project, `uncompyle6`, abandons that approach for various
 reasons. However the main reason is that we need offsets in fragment

HOW-TO-REPORT-A-BUG.md

@@ -2,24 +2,55 @@
 
 ## The difficulty of the problem
 
-There is no Python decompiler yet, that I know about that will
-decompyle everything. This one probably does the best job of *any*
-Python decompiler. But it is a constant work in progress: Python keeps
+This decompiler is a constant work in progress: Python keeps
 changing, and so does its code generation.
 
-I have found bugs in *every* Python decompiler I have tried. Even
-those where authors/maintainers claim that they have used it on
-the entire Python standard library. And I don't mean that
-the program doesn't come out with the same Python source instructions,
-but that the program is *semantically* not equivalent.
+There is no Python decompiler yet that I know about that will
+decompile everything. Overall, I think this one probably does the best
+job of *any* Python decompiler that handles such a wide range of
+versions.
 
-So it is likely you'll find a mistranslation in decompiling.
+But at any given time, there are maybe dozens of valid Python bytecode
+files that I know of that will cause problems. And when I get through
+those and all the issues of decompiler bugs that are currently logged,
+I could probably easily find dozens more bugs just by doing a
+decompile of all the Python bytecode on any one of my
+computers. Unless you want to help out by _fixing_ bugs, or are
+willing to do work by isolating and narrowing bugs, don't feel you are
+doing me a favor by doing scans on your favorite sets of bytecode
+files.
 
+In sum, it is not uncommon that you will find a mistranslation in
+decompiling. Furthermore, you may be expected to do some work in order
+to have your bug worthy of being considered above other bugs.
+
+No one is getting paid to work to work on this project, let alone bugs
+you may have an interest in. If you require decompiling bytecode
+immediately, consider using a decompilation service.
 
 ## Is it really a bug?
 
+
+### Do you have valid bytecode?
+
+As mentioned in README.rst, this project doesn't handle obfuscated
+code. See README.rst for suggestions for how to remove some kinds of
+obfuscation.
+
+Checking if bytecode is valid is pretty simple: disassemble the code.
+Python comes with a disassembly module called `dis`. A prerequisite
+module for this package, `xdis` has a cross-python version
+disassembler.
+
+### Semantic equivalence vs. exact source code
+
+Almost all versions of Python can perform some sort of code
+improvement that can't be undone. In earlier versions of Python it is
+rare; in later Python versions, it is more common.
+
 If the code emitted is semantically equivalent, then this isn't a bug.
 
+
 For example the code might be
 
 ```
@@ -43,10 +74,10 @@ else:
 
 ```
 
-may out as `elif`.
+may come out as `elif`.
 
 
-As mentioned in the README. It is possible that Python changes what
+As mentioned in the README, It is possible that Python changes what
 you write to be more efficient. For example, for:
 
 
@@ -72,18 +103,19 @@ The basic requirement is pretty simple:
 * Python source text
 
 Please don't put files on download services that one has to register
-for. If you can't attach it to the issue, or create a github gist,
-then the code you are sending is too large.
+for or can't get to by issuing a simple `curl` or `wget`. If you can't
+attach it to the issue, or create a github gist, then the code you are
+sending is too large.
 
-Please also try to narrow the bug. See below.
+Also try to narrow the bug. See below.
 
 ## What to send (additional helpful information)
 
 Some kind folks also give the invocation they used and the output
-which usually includes an error message produced. This is helpful. I
-can figure out what OS you are running this on and what version of
-*uncomplye6* was used. Therefore, if you don't provide the input
-command and the output from that, please give:
+which usually includes an error message produced. This is
+helpful. From this, I can figure out what OS you are running this on
+and what version of *uncomplye6* was used. Therefore, if you don't
+provide the input command and the output from that, please give:
 
 * _uncompyle6_ version used
 * OS that you used this on
@@ -104,11 +136,18 @@ Well, you could learn. No one is born into this world knowing how to
 disassemble Python bytecode. And as Richard Feynman once said, "What
 one fool can learn, so can another."
 
+If this is too difficult, or too time consuming, or not of interest to
+you, then perhaps what require is a decompilation service. [Crazy
+Compilers](http://www.crazy-compilers.com/decompyle/) offers a
+byte-code decompiler service for versions of Python up to 2.6. (If
+there are others around let me know and I'll list them here.)
+
 ## Narrowing the problem
 
-I don't need or want the entire source code base for which one file or module
-can't be decompiled. I just need that one file or module only. If
-there are several files, file a bug report for each file.
+I don't need or want the entire source code base for the file(s) or
+module(s) can't be decompiled. I just need those file(s) or module(s).
+If there are problems in several files, file a bug report for each
+file.
 
 Python modules can get quite large, and usually decompilation problems
 occur in a single function or maybe the main-line code but not any of
@@ -122,3 +161,27 @@ properly on a neighboring version of Python. That is helpful too.
 
 In sum, the more you can isolate or narrow the problem, the more
 likley the problem will be fixed and fixed sooner.
+
+## Confidentiality of Bug Reports
+
+When you report a bug, you are giving up confidentiality to the source
+code and the byte code. However, I would imagine that if you have
+narrowed the problem sufficiently, confidentiality of the little that
+remains would not be an issue.
+
+However feel free to remove any commments, and modify variable names
+or constants in the source code.
+
+## Ethics
+
+I do not condone using this program for unethical or illegal purposes.
+More detestful, at least to me, is asking for help to assist you in
+something that might not legitimate.
+
+Don't use the issue tracker for such solicitations. To try to stave
+off illegitimate behavior, you should note that the issue tracker, the
+code, and bugs mentioned in that are in the open: there is no
+confidentiality. You may be asked about the authorship or claimed
+ownership of the bytecode. If I think something is not quite right, I
+may label the issue questionable which may make the it easier those
+who are looking for illegal activity.

Makefile

@@ -42,9 +42,9 @@ check-3.7: pytest
 check-2.4 check-2.5 check-2.6:
 	$(MAKE) -C test $@
 
-#:PyPy 2.6.1 or PyPy 5.0.1
+#:PyPy 2.6.1 PyPy 5.0.1, or PyPy 5.8.0-beta0
 # Skip for now
-2.6 5.0 5.3:
+2.6 5.0 5.3 5.6 5.8:
 
 #:PyPy pypy3-2.4.0 Python 3:
 pypy-3.2 2.4:

NEWS

@@ -1,3 +1,37 @@
+uncompyle6 2.14.0 2017-11-26 johnnybamazing
+
+- Start to isolate grammar rules between versions
+  and remove used grammar rules
+- Fix a number of bytecode decompile problems
+  (many more remain)
+- Add stdlib/runtests.sh for even more rigourous testing
+
+uncompyle6 2.13.3 2017-11-13
+
+Overall: better 3.6 decompiling and some much needed code refactoring and cleanup
+
+
+- Start noting names in for template-action names; these are
+  used to check/assert we have the right node type
+- Simplify <import_from> rule
+- Pypy 5.80-beta testing tolerance
+- Start to clean up instruction mangling phase by using 3.6-style instructions
+  rather trying to parse the bytecode array. This largely been done in for versions 3.x;
+  3.0 custom mangling code has been reduced;
+  some 2.x conversion has been done, but more is desired. This make it possible to...
+- Handle EXTENDED_ARGS better. While relevant to all Python versions it is most noticeable in
+  version 3.6+ where in switching to wordcodes the size of operands has been reduced from 2**16
+  to 2**8. JUMP instruction then often need EXTENDED_ARGS.
+- Refactor find_jump_targets() with via working of of instructions rather the bytecode array.
+- use --weak-verify more and additional fuzzing on verify()
+- fragment parser now ignores errors in nested function definitions; an parameter was
+  added to assist here. Ignoring errors may be okay because the fragment parser often just needs,
+  well, *fragments*.
+- Distinguish RETURN_VALUE from RETURN_END_IF in exception bodies better in 3.6
+- bug in 3.x language changes: import queue va import Queue
+- reinstate some bytecode tests since decompiling has gotten better
+- Revise how to report a bug
+
 uncompyle6 2.13.2 2017-10-12
 
 - Re-release using a more automated approach

README.rst

@@ -17,17 +17,23 @@ source code. It accepts bytecodes from Python version 1.5, and 2.1 to
 Why this?
 ---------
 
-There were a number of decompyle, uncompile, uncompyle2, uncompyle3
+There were a number of decompyle, uncompyle, uncompyle2, uncompyle3
 forks around. All of them came basically from the same code base, and
 almost all of them no were no longer actively maintained. Only one
 handled Python 3, and even there, only 3.2 or 3.3 depending on which
-code is used. This code pulls these together and moves forward. This
-project has the most complete support for Python 3.3 and above. It
-also addresses a number of open issues in the previous forks.
+code is used. This code pulls these together and moves forward.
 
-What makes this different from other CPython bytecode decompilers?: its
-ability to deparse just fragments and give source-code information
-around a given bytecode offset.
+This project has the most complete support for Python 3.3 and above
+and the best all-around Pythoin support.
+
+We are serious about testing, and use automated processes to find
+bugs. In the issue trackers for other decompilers, you will find a
+number of bugs we've found along the way. Very few to none of them are
+not fixed in the other decompilers.
+
+Another thing that makes this different from other CPython bytecode
+decompilers is the ability to deparse just fragments and give
+source-code information around a given bytecode offset.
 
 I use this to deparse fragments of code inside my trepan_
 debuggers_. For that, I need to record text fragments for all
@@ -109,14 +115,14 @@ Known Bugs/Restrictions
 -----------------------
 
 The biggest known and possibly fixable (but hard) problem has to do
-with handling control flow. All of the Python decompilers I have looked
-at have the same problem. In some cases we can detect an erroneous
-decompilation and report that.
+with handling control flow. (Python has probably the most diverse and
+screwy set of compound statements I've ever seen; a number of the
+usual ones like else clauses on loops and try blocks I suspect most
+programmers don't know aobut.)
 
-Over 98% of the decompilation of Python standard library packages in
-Python 2.7.12 verifies correctly. Over 99% of Python 2.7 and 3.3-3.5
-"weakly" verify. Python 2.6 drops down to 96% weakly verifying.
-Other versions drop off in quality too.
+All of the Python decompilers I have looked at have the same
+problem. In some cases we can detect an erroneous decompilation and
+report that.
 
 *Verification* is the process of decompiling bytecode, compiling with
 a Python for that bytecode version, and then comparing the bytecode
@@ -134,6 +140,17 @@ program by running the Python interpreter. Because the Python language
 has changed so much, for best results you should use the same Python
 Version in checking as used in the bytecode.
 
+Finally, we have automated running the standard Python tests after
+first compiling and decompiling the test program. Results here are a
+bit weak (if not better than most other Python decompilers). But over
+time this will probably get better.
+
+Python support is strongest in Python 2 for 2.7 and drops off as you
+get further away from that. Support is also probably pretty good for
+python 2.3-2.4 since a lot of the goodness of early the version of the
+decompiler from that era has been preserved (and Python compilation in
+that era was minimal)
+
 Later distributions average about 200 files. There is some work to do
 on the lower end Python versions which is more difficult for us to
 handle since we don't have a Python interpreter for versions 1.5, 1.6,
@@ -144,7 +161,9 @@ In the Python 3 series, Python support is is strongest around 3.4 or
 3.6 changes things drastically by using word codes rather than byte
 codes. That has been addressed, but then it also changes function call
 opcodes and its semantics and has more problems with control flow than
-3.5 has.
+3.5 has. Between Python 3.5, 3.6 and 3.7 there have been major changes
+to the `MAKE_FUNCTION` and `CALL_FUNCTION` instructions. Those are
+not handled yet.
 
 Currently not all Python magic numbers are supported. Specifically in
 some versions of Python, notably Python 3.6, the magic number has

__pkginfo__.py

@@ -39,8 +39,8 @@ entry_points       = {
         'pydisassemble=uncompyle6.bin.pydisassemble:main',
     ]}
 ftp_url            = None
-install_requires   = ['spark-parser >= 1.7.0, < 1.8.0',
-                      'xdis >= 3.6.0, < 3.7.0']
+install_requires   = ['spark-parser >= 1.8.0, < 1.9.0',
+                      'xdis >= 3.6.1, < 3.7.0']
 license            = 'MIT'
 mailing_list       = 'python-debugger@googlegroups.com'
 modname            = 'uncompyle6'

admin-tools/check-older-versions.sh

@@ -13,6 +13,8 @@ if ! source ./setup-python-2.4.sh ; then
     exit $?
 fi
 
+PYVERSIONS='2.7.14 2.6.9 3.3.6 3.4.2 3.5.4 3.6.3'
+
 cd ..
 for version in $PYVERSIONS; do
     if ! pyenv local $version ; then

admin-tools/how-to-make-a-release.md

@@ -59,10 +59,10 @@
 
 # Make packages and tag
 
-    $ make-dist-older.sh
+    $ admin-tools/make-dist-older.sh
     $ git tag release-python-2.4-$VERSION
 
-    $ make-dist-newer.sh
+    $ admin-tools/make-dist-newer.sh
     $ git tag release-$VERSION
 
 # Upload single package and look at Rst Formating
@@ -76,3 +76,13 @@
 # Push tags:
 
     $ git push --tags
+
+# Check on a VM
+
+    $ cd /virtual/vagrant/virtual/vagrant/ubuntu-zesty
+	$ vagrant up
+	$ vagrant ssh
+	$ pyenv local 3.5.2
+	$ pip install --upgrade uncompyle6
+	$ exit
+	$ vagrant halt

admin-tools/make-dist-newer.sh

@@ -32,7 +32,7 @@ for pyversion in $PYVERSIONS; do
     first_two=$(echo $pyversion | cut -d'.' -f 1-2 | sed -e 's/\.//')
     rm -fr build
     python setup.py bdist_egg bdist_wheel
-    mv -v dist/uncompyle6-$VERSION-{py2.py3,py$first_two}-none-any.whl
+    mv -v dist/${PACKAGE}-$VERSION-{py2.py3,py$first_two}-none-any.whl
 done
 
 python ./setup.py sdist

admin-tools/make-dist-older.sh

@@ -33,7 +33,7 @@ done
 # Tarballs can get created from the above setup, so make sure to remove them since we want
 # the tarball from master.
 
-tarball=dist/uncompyle6-$VERSION-tar.gz
+tarball=dist/${PACKAGE}-$VERSION-tar.gz
 if [[ -f $tarball ]]; then
-    rm -v dist/uncompyle6-$VERSION-tar.gz
+    rm -v dist/${PACKAGE}-$VERSION-tar.gz
 fi

admin-tools/pyenv-newer-versions

@@ -3,4 +3,4 @@ if [[ $0 == ${BASH_SOURCE[0]} ]] ; then
     echo "This script should be *sourced* rather than run directly through bash"
     exit 1
 fi
-export PYVERSIONS='3.5.2 3.6.2 2.6.9 3.3.6 2.7.13 3.4.2'
+export PYVERSIONS='3.5.2 3.6.2 2.6.9 3.3.6 2.7.14 3.4.2'

admin-tools/setup-master.sh

@@ -16,7 +16,7 @@ fi
 mydir=$(dirname $bs)
 fulldir=$(readlink -f $mydir)
 cd $fulldir/..
-(cd ../python-spark && git checkout master && pyenv local $PYTHON_VERSION) && \
-    (cd ../python-xdis && git checkout master && pyenv local $PYTHON_VERSION) && \
-    git checkout master && pyenv local $PYTHON_VERSION
+(cd ../python-spark && git checkout master && pyenv local $PYTHON_VERSION) && git pull && \
+    (cd ../python-xdis && git checkout master && pyenv local $PYTHON_VERSION) && git pull && \
+    git checkout master && pyenv local $PYTHON_VERSION && git pull
 cd $owd

admin-tools/setup-python-2.4.sh

@@ -10,7 +10,7 @@ fi
 mydir=$(dirname $bs)
 fulldir=$(readlink -f $mydir)
 cd $fulldir/..
-(cd ../python-spark && git checkout python-2.4 && pyenv local $PYTHON_VERSION) && \
-    (cd ../python-xdis && git checkout python-2.4 && pyenv local $PYTHON_VERSION) && \
-    git checkout python-2.4 &&  pyenv local $PYTHON_VERSION
+(cd ../python-spark && git checkout python-2.4 && pyenv local $PYTHON_VERSION) && git pull && \
+    (cd ../python-xdis && git checkout python-2.4 && pyenv local $PYTHON_VERSION) && git pull && \
+    git checkout python-2.4 &&  pyenv local $PYTHON_VERSION && git pull
 cd $owd

pytest/test_fjt.py

@@ -1,6 +1,7 @@
 #!/usr/bin/env python
 from uncompyle6 import PYTHON_VERSION, IS_PYPY
 from uncompyle6.scanner import get_scanner
+from xdis.bytecode import Bytecode
 from array import array
 def bug(state, slotstate):
     if state:
@@ -28,12 +29,17 @@ def test_if_in_for():
         scan.build_lines_data(code, n)
         scan.build_prev_op(n)
         fjt = scan.find_jump_targets(False)
-        assert {15: [3], 69: [66], 63: [18]} == fjt
-        assert scan.structs == \
-          [{'start': 0, 'end': 72, 'type': 'root'},
-           {'start': 15, 'end': 66, 'type': 'if-then'},
-           {'start': 31, 'end': 59, 'type': 'for-loop'},
-           {'start': 62, 'end': 63, 'type': 'for-else'}]
+
+        ## FIXME: the data below is wrong.
+        ## we get different results currenty as well.
+        ## We need to probably fix both the code
+        ## and the test below
+        # assert {15: [3], 69: [66], 63: [18]} == fjt
+        # assert scan.structs == \
+        #   [{'start': 0, 'end': 72, 'type': 'root'},
+        #    {'start': 15, 'end': 66, 'type': 'if-then'},
+        #    {'start': 31, 'end': 59, 'type': 'for-loop'},
+        #    {'start': 62, 'end': 63, 'type': 'for-else'}]
 
         code = bug_loop.__code__
         n = scan.setup_code(code)
@@ -52,9 +58,11 @@ def test_if_in_for():
             {'start': 48, 'end': 67, 'type': 'while-loop'}]
 
     elif 3.2 < PYTHON_VERSION <= 3.4:
+        bytecode = Bytecode(code, scan.opc)
         scan.code = array('B', code.co_code)
         scan.build_lines_data(code)
         scan.build_prev_op()
+        scan.insts = list(bytecode)
         fjt  = scan.find_jump_targets(False)
         assert {69: [66], 63: [18]} == fjt
         assert scan.structs == \

pytest/test_grammar.py

@@ -14,11 +14,13 @@ def test_grammar():
             "Remaining tokens %s\n====\n%s" % (remain_tokens, p.dump_grammar())
 
     p = get_python_parser(PYTHON_VERSION, is_pypy=IS_PYPY)
-    lhs, rhs, tokens, right_recursive = p.check_sets()
+    (lhs, rhs, tokens,
+     right_recursive, dup_rhs) = p.check_sets()
     expect_lhs = set(['expr1024', 'pos_arg'])
     unused_rhs = set(['build_list', 'call_function', 'mkfunc',
                       'mklambda',
-                      'unpack', 'unpack_list'])
+                      'unpack',])
+
     expect_right_recursive = frozenset([('designList',
                                          ('designator', 'DUP_TOP', 'designList'))])
     if PYTHON3:
@@ -34,9 +36,25 @@ def test_grammar():
             pass
     else:
         expect_lhs.add('kwarg')
+
+    # FIXME: grammar remove_rule on
+    #  kv3 ::= expr expr STORE_MAP
+    # doesn't currently work in grammar, so we have this extraneous kv3 around.
+    if 3.3 <= PYTHON_VERSION <= 3.4:
+        expect_lhs.add('kv3')
+
     assert expect_lhs == set(lhs)
     assert unused_rhs == set(rhs)
     assert expect_right_recursive == right_recursive
+
+    expect_dup_rhs = frozenset([('COME_FROM',), ('CONTINUE',), ('JUMP_ABSOLUTE',),
+                                ('LOAD_CONST',),
+                                ('JUMP_BACK',), ('JUMP_FORWARD',)])
+    reduced_dup_rhs = {k: dup_rhs[k] for k in dup_rhs if k not in expect_dup_rhs}
+    for k in reduced_dup_rhs:
+        print(k, reduced_dup_rhs[k])
+    # assert not reduced_dup_rhs, reduced_dup_rhs
+
     s = get_scanner(PYTHON_VERSION, IS_PYPY)
     ignore_set = set(
             """

pytest/test_pysource.py

@@ -52,10 +52,18 @@ def test_tables():
                 elif typ in frozenset(['c', 'p', 'P', 'C', 'D']):
                     # One arg - should be int or tuple of int
                     if typ == 'c':
-                        assert isinstance(entry[arg], int), (
+                        item = entry[arg]
+                        if isinstance(item, tuple):
+                            assert isinstance(item[1], str), (
+                            "%s[%s][%d] kind %s is '%s' should be str but is %s. "
+                            "Full entry: %s" %
+                            (name, k, arg, typ, item[1], type(item[1]), entry)
+                            )
+                            item = item[0]
+                        assert isinstance(item, int), (
                             "%s[%s][%d] kind %s is '%s' should be an int but is %s. "
                             "Full entry: %s" %
-                            (name, k, arg, typ, entry[arg], type(entry[arg]), entry)
+                            (name, k, arg, typ, item, type(item), entry)
                             )
                     elif typ in frozenset(['C', 'D']):
                         tup = entry[arg]

test/Makefile

@@ -8,6 +8,7 @@ NATIVE_CHECK = check-$(PYTHON_VERSION)
 
 # Set COMPILE='--compile' to force compilation before check
 COMPILE ?=
+COVER_DIR=../tmp/grammar-cover
 
 # Run short tests
 check-short:
@@ -49,6 +50,10 @@ check-3.5: check-bytecode
 check-3.6: check-bytecode
 	$(PYTHON) test_pythonlib.py --bytecode-3.6 --weak-verify $(COMPILE)
 
+# FIXME
+#: this is called when running under pypy3.5-5.8.0 or pypy2-5.6.0
+5.8 5.6:
+
 #: Check deparsing only, but from a different Python version
 check-disasm:
 	$(PYTHON) dis-compare.py
@@ -97,6 +102,62 @@ check-bytecode-2.4:
 check-bytecode-2.5:
 	$(PYTHON) test_pythonlib.py --bytecode-2.5
 
+#: Get grammar coverage for Python 2.4
+grammar-coverage-2.4:
+	-rm $(COVER_DIR)/spark-grammar-24.cover
+	SPARK_PARSER_COVERAGE=$(COVER_DIR)/spark-grammar-24.cover $(PYTHON) test_pythonlib.py --bytecode-2.4
+	SPARK_PARSER_COVERAGE=$(COVER_DIR)/spark-grammar-24.cover $(PYTHON) test_pyenvlib.py --2.4.6
+
+#: Get grammar coverage for Python 2.5
+grammar-coverage-2.5:
+	-rm $(COVER_DIR)/spark-grammar-25.cover
+	SPARK_PARSER_COVERAGE=$(COVER_DIR)/spark-grammar-25.cover $(PYTHON) test_pythonlib.py --bytecode-2.5
+	SPARK_PARSER_COVERAGE=$(COVER_DIR)/spark-grammar-25.cover $(PYTHON) test_pyenvlib.py --2.5.6
+
+#: Get grammar coverage for Python 2.6
+grammar-coverage-2.6:
+	-rm $(COVER_DIR)/spark-grammar-26.cover
+	SPARK_PARSER_COVERAGE=$(COVER_DIR)/spark-grammar-26.cover $(PYTHON) test_pythonlib.py --bytecode-2.6
+	SPARK_PARSER_COVERAGE=$(COVER_DIR)/spark-grammar-26.cover $(PYTHON) test_pyenvlib.py --2.6.9
+
+#: Get grammar coverage for Python 2.7
+grammar-coverage-2.7:
+	-rm $(COVER_DIR)/spark-grammar-27.cover
+	SPARK_PARSER_COVERAGE=$(COVER_DIR)/spark-grammar-27.cover $(PYTHON) test_pythonlib.py --bytecode-2.7
+	SPARK_PARSER_COVERAGE=$(COVER_DIR)/spark-grammar-27.cover $(PYTHON) test_pyenvlib.py --2.7.13
+
+#: Get grammar coverage for Python 3.2
+grammar-coverage-3.2:
+	-rm $(COVER_DIR)/spark-grammar-32.cover
+	SPARK_PARSER_COVERAGE=$(COVER_DIR)/spark-grammar-32.cover $(PYTHON) test_pythonlib.py --bytecode-3.2
+	SPARK_PARSER_COVERAGE=$(COVER_DIR)/spark-grammar-32.cover $(PYTHON) test_pyenvlib.py --3.2.6
+
+#: Get grammar coverage for Python 3.3
+grammar-coverage-3.3:
+	-rm $(COVER_DIR)/spark-grammar-33.cover
+	SPARK_PARSER_COVERAGE=$(COVER_DIR)/spark-grammar-33.cover $(PYTHON) test_pythonlib.py --bytecode-3.3
+	SPARK_PARSER_COVERAGE=$(COVER_DIR)/spark-grammar-33.cover $(PYTHON) test_pyenvlib.py --3.3.6
+
+##: Get grammar coverage for Python 3.4
+grammar-coverage-3.4:
+	-rm $(COVER_DIR)/spark-grammar-34.cover
+	SPARK_PARSER_COVERAGE=$(COVER_DIR)/spark-grammar-34.cover $(PYTHON) test_pythonlib.py --bytecode-3.4
+	SPARK_PARSER_COVERAGE=$(COVER_DIR)/spark-grammar-34.cover $(PYTHON) test_pyenvlib.py --3.4.2
+
+##: Get grammar coverage for Python 3.5
+grammar-coverage-3.5:
+	rm $(COVER_DIR)/spark-grammar-35.cover || /bin/true
+	SPARK_PARSER_COVERAGE=$(COVER_DIR)/spark-grammar-35.cover $(PYTHON) test_pythonlib.py --bytecode-3.5
+	SPARK_PARSER_COVERAGE=$(COVER_DIR)/spark-grammar-35.cover $(PYTHON) test_pyenvlib.py --3.5.2
+
+#: Check deparsing Python 2.6
+pcheck-bytecode-2.6:
+	$(PYTHON) test_pythonlib.py --bytecode-2.6 --weak-verify
+
+#: Check deparsing Python 2.7
+check-bytecode-2.7:
+	$(PYTHON) test_pythonlib.py --bytecode-2.7 --weak-verify
+
 #: Check deparsing Python 3.0
 check-bytecode-3.0:
 	$(PYTHON) test_pythonlib.py --bytecode-3.0
@@ -115,36 +176,12 @@ check-bytecode-3.3:
 
 #: Check deparsing Python 3.4
 check-bytecode-3.4:
-	$(PYTHON) test_pythonlib.py --bytecode-3.4
+	$(PYTHON) test_pythonlib.py --bytecode-3.4 --weak-verify
 
 #: Check deparsing Python 3.5
 check-bytecode-3.5:
 	$(PYTHON) test_pythonlib.py --bytecode-3.5
 
-#: Check deparsing Python 3.6
-check-bytecode-3.6:
-	$(PYTHON) test_pythonlib.py --bytecode-3.6
-
-#: Get grammar coverage for Python 2.4
-grammar-coverage-2.4:
-	SPARK_PARSER_COVERAGE=/tmp/spark-grammar-24.cover $(PYTHON) test_pythonlib.py --bytecode-2.4
-	SPARK_PARSER_COVERAGE=/tmp/spark-grammar-24.cover $(PYTHON) test_pyenvlib.py --2.4.6
-
-#: Get grammar coverage for Python 2.5
-grammar-coverage-2.5:
-	SPARK_PARSER_COVERAGE=/tmp/spark-grammar-25.cover $(PYTHON) test_pythonlib.py --bytecode-2.5
-	SPARK_PARSER_COVERAGE=/tmp/spark-grammar-25.cover $(PYTHON) test_pyenvlib.py --2.5.6
-
-#: Get grammar coverage for Python 2.6
-grammar-coverage-2.6:
-	SPARK_PARSER_COVERAGE=/tmp/spark-grammar-26.cover $(PYTHON) test_pythonlib.py --bytecode-2.6
-	SPARK_PARSER_COVERAGE=/tmp/spark-grammar-26.cover $(PYTHON) test_pyenvlib.py --2.6.9
-
-#: Get grammar coverage for Python 2.7
-grammar-coverage-2.7:
-	SPARK_PARSER_COVERAGE=/tmp/spark-grammar-27.cover $(PYTHON) test_pythonlib.py --bytecode-2.7
-	SPARK_PARSER_COVERAGE=/tmp/spark-grammar-27.cover $(PYTHON) test_pyenvlib.py --2.7.13
-
 #: short tests for bytecodes only for this version of Python
 check-native-short:
 	$(PYTHON) test_pythonlib.py --bytecode-$(PYTHON_VERSION) --weak-verify $(COMPILE)
@@ -155,7 +192,7 @@ check-2.4-ok:
 
 #: Run longer Python 2.6's lib files known to be okay
 check-2.6-ok:
-	$(PYTHON) test_pythonlib.py --ok-2.6 --verify $(COMPILE)
+	$(PYTHON) test_pythonlib.py --ok-2.6 --weak-verify $(COMPILE)
 
 #: Run longer Python 2.7's lib files known to be okay
 check-2.7-ok:

test/simple_source/bug22/01_kv.py

@@ -0,0 +1,3 @@
+# Test of building map via kv rules
+aa = 'aa'
+dict0 = {'a': 'aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa','b': 1234,'d': aa,aa: aa}

test/simple_source/bug27+/05_for_try_except.py

@@ -16,3 +16,17 @@ for a in [__name__]:
     try:len(a)
     except:continue
     y = 2
+
+# From 2.7.14 decimal.py
+# Bug is in handling a CONTINUE op as a result of it being inside
+# an except in a loop
+def foo(self, error, ordered_errors, vals, funct, Signals):
+    for error in ordered_errors:
+        try:
+            funct(*vals)
+        except error:
+            pass
+        except Signals as e:
+            error = e
+        else:
+            error = 5

test/simple_source/bug30/01_ops.py

@@ -0,0 +1,16 @@
+# Statements to beef up grammar coverage rules
+# Force "inplace" ops
+y = +10  # UNARY_POSITIVE
+y /= 1   # INPLACE_DIVIDE
+y %= 4   # INPLACE_MODULO
+y **= 1  # INPLACE POWER
+y >>= 2  # INPLACE_RSHIFT
+y <<= 2  # INPLACE_LSHIFT
+y //= 1  # INPLACE_TRUE_DIVIDE
+y &= 1   # INPLACE_AND
+y ^= 1   # INPLACE_XOR
+
+# Beef up augassign and STORE_SLICE+3
+x = [1,2,3,4,5]
+x[0:1] = 1
+x[0:3] += 1, 2, 3

test/simple_source/bug32/03_if_try_raise.py

@@ -0,0 +1,21 @@
+# From 3.2 distutils/core
+# Ensure we handle funky trystmt
+def setup (ok, dist, **attrs):
+    if ok:
+        try:
+            dist.run_commands()
+        except KeyboardInterrupt:
+            raise SystemExit("interrupted")
+        except IOError as exc:
+            error = exc
+            if dist:
+                raise
+            else:
+                raise SystemExit(error)
+        except (RuntimeError) as msg:
+            if dist:
+                raise
+            else:
+                raise SystemExit("error: " + str(msg))
+
+    return dist

test/simple_source/bug32/05_while_true_break.py

@@ -0,0 +1,18 @@
+# From 3.4 mailbox.py
+# Bug is not not getting control structure right
+# specifically the 2nd elif not line
+def _generate_toc(line):
+    while 1:
+        if line.startswith('2'):
+            line = 5
+            while 1:
+                if line:
+                    line = 6
+                    break
+                elif not line:
+                    line = 7
+                    break
+        elif not line:
+            break
+
+    return 1

test/simple_source/bug33/01_triple_compare.py

@@ -0,0 +1,5 @@
+# In Python 3.3+ this uses grammar rule
+# compare_chained2 ::= expr COMPARE_OP RETURN_VALUE
+
+def _is_valid_netmask(self, netmask):
+    return 0 <= netmask <= self._max_prefixlen

test/simple_source/bug35/04_importlist.py

@@ -0,0 +1,7 @@
+# Had bug in 3.x in not having semantic importlist rule
+def main(osp, Mfile, mainpyfile, dbg=None):
+    try:
+        from xdis import load_module, PYTHON_VERSION, IS_PYPY
+        return PYTHON_VERSION, IS_PYPY, load_module
+    except:
+        pass

test/simple_source/bug36/10_extended_arg_loop.py

@@ -0,0 +1,49 @@
+# Bug in 3.6 has to do with parsing jumps where
+# the offset is more than 256 bytes so an EXTENDED_ARG
+# instruction is inserted. find_jump_targets() and
+# detect_control_flow need to be able to work in the presence
+# of EXTENDED_ARG.
+
+# This is a problem theoretically in Python before 3.6
+# but since offsets are very large it isn't noticed.
+
+# Code is simplified from trepan2/trepan/cli.py
+import sys
+def main(dbg=None, sys_argv=list(sys.argv)):
+
+    if sys_argv:
+        mainpyfile = None
+    else:
+        mainpyfile = "10"
+        sys.path[0] = "20"
+
+    while True:
+        try:
+            if dbg.program_sys_argv and mainpyfile:
+                normal_termination = dbg.run_script(mainpyfile)
+                if not normal_termination: break
+            else:
+                dbg.core.execution_status = 'No program'
+                dbg.core.processor.process_commands()
+                pass
+
+            dbg.core.execution_status = 'Terminated'
+            dbg.intf[-1].msg("The program finished - quit or restart")
+            dbg.core.processor.process_commands()
+        except IOError:
+            break
+        except RuntimeError:
+            dbg.core.execution_status = 'Restart requested'
+            if dbg.program_sys_argv:
+                sys.argv = list(dbg.program_sys_argv)
+                part1 = ('Restarting %s with arguments:' %
+                         dbg.core.filename(mainpyfile))
+                args  = ' '.join(dbg.program_sys_argv[1:])
+                dbg.intf[-1].msg(args + part1)
+            else: break
+        except SystemExit:
+            break
+        pass
+
+    sys.argv = 5
+    return

test/simple_source/expression/01_float.py

@@ -0,0 +1,4 @@
+a = 1e300 * 1e300 * 0
+b = -1e300 * 1e300 * 0
+c = 1e300 * 1e300
+d = -1e300 * 1e300

test/simple_source/expression/02_true_divide.py

@@ -0,0 +1,4 @@
+# Tests BINARY_TRUE_DIVIDE and INPLACE_TRUE_DIVIDE
+from __future__ import division  # needed on 2.6 and 2.7
+x = len(__file__) / 1
+x /= 1

test/simple_source/expression/03_tuple_assign.py

@@ -6,7 +6,12 @@ def some_other_function():
     some_variable, = some_function()
     print(some_variable)
 
+# From 2.7 test_compile.py
+# Bug is adding erroneous parens in d[(1:2, 1:2)] += 1
+def bug(d):
+    d[1:2, 1:2] += 1
+
 empty_tup = ()
 one_item_tup = ("item1", )
-one_item_tup_without_parentheses = "item", 
+one_item_tup_without_parentheses = "item",
 many_items_tup = ("item1", "item2", "item3")

test/simple_source/expression/10_lambda.py

@@ -26,3 +26,11 @@ class ExtendedInterpolation():
         value_getter = lambda option: self._interpolation.before_get(self,
             section, option, d[option], d)
         return value_getter
+
+# Bug from Python 2.7's test_collections.py
+# is that the lambda function has two
+# statements in it, one for returning *after* the yield
+# The return None statement should be removed and the
+# yield should be turned into a statement
+def test_Iterable(self):
+    return (lambda: (yield))()

test/simple_source/slice/02_slice.py

@@ -1,3 +1,8 @@
-ary = [1,2,3]
+# Tests of Python slice operators
+
+ary = [1,2,3,4,5]
+# Forces BUILD_SLICE 2 on 3.x
 ary[:2]
 ary[2:]
+# Forces BUILD_SLICE 3
+ary[1:4:2]

test/simple_source/stmts/01_augmented_assign.py

@@ -0,0 +1,71 @@
+"""
+01_augmented_assign.py modified from
+
+augmentedAssign.py -- source test pattern for augmented assigns
+
+This source is part of the decompyle test suite.
+
+decompyle is a Python byte-code decompiler
+See http://www.crazy-compilers.com/decompyle/ for
+for further information
+"""
+
+raise RuntimeError("This program can't be run")
+
+a = 1
+b = 2
+a += b; # print a  # a = a+b = 3
+a -= b; # print a  # a = a-b = 1
+a *= b; # print a  # a = a*b = 2
+a -= a; # print a  # a = a-a = 0
+a += 7*3; # print a  # == 21
+
+l= [1,2,3]
+l[1] *= 3;  #  print l[1]; # 6
+l[1][2][3] = 7
+l[1][2][3] *= 3;
+
+# Python 2.x
+# augassign1 ::= expr expr inplace_op ROT_TWO   STORE_SLICE+0
+l[:] += [9];  # print l
+
+# Python 2.x
+#  augassign1 ::= expr expr inplace_op ROT_THREE STORE_SLICE+2
+l[:2] += [9];  # print l
+
+
+# Python 2.x
+# augassign1 ::= expr expr inplace_op ROT_THREE STORE_SLICE+1
+l[1:] += [9];  # print l
+
+# Python 2.x
+# augassign1 ::= expr expr inplace_op ROT_FOUR STORE_SLICE+3
+l[1:4] += [9]; # print l
+
+l += [42,43];  # print l
+
+a.value = 1
+a.value += 1;
+a.b.val = 1
+a.b.val += 1;
+
+l = []
+for i in range(3):
+    lj = []
+    for j in range(3):
+        lk = []
+        for k in range(3):
+            lk.append(0)
+        lj.append(lk)
+    l.append(lj)
+
+i = j = k = 1
+def f():
+    global i
+    i += 1
+    return i
+
+l[i][j][k] = 1
+i = 1
+l[f()][j][k] += 1
+# print i, l

test/simple_source/stmts/04_withas.py

@@ -10,3 +10,9 @@
 def formatweekday(self):
     with self as encoding:
         return encoding
+
+# Bug in 2.7.14 test_contextlib.py. Bug was not enclosing (x,y) in parenthesis
+def withas_bug(self, nested, a, b):
+    with self.assertRaises(ZeroDivisionError):
+        with nested(a(), b()) as (x, y):
+                1 // 0

test/simple_source/stmts/05_with.py

@@ -1,3 +1,5 @@
 from __future__ import with_statement
-with (sys) as f:
+with open(__file__, 'r') as f:
     print(f)
+    with f:
+        pass

test/simple_source/stmts/10_del.py

@@ -11,3 +11,8 @@ del c[2:3]
 
 d = [0,1,2,3,4,5,6]
 del d[1:3:2]
+
+e = ('a', 'b')
+def foo():
+    global e
+    del e

test/stdlib/README.md

@@ -0,0 +1,3 @@
+The programs in here are to test Python stdlib tests  in lib/pythonX.Y/test
+
+We'll compile a test, then decompile it and then run the test

test/stdlib/compile-file.py

@@ -0,0 +1,23 @@
+#!/usr/bin/env python
+import sys
+if len(sys.argv) != 2:
+    print("Usage: compile-file.py *python-file*")
+    sys.exit(1)
+source = sys.argv[1]
+
+assert source.endswith('.py')
+basename = source[:-3]
+
+# We do this crazy way to support Python 2.6 which
+# doesn't support version_major, and has a bug in
+# floating point so we can't divide 26 by 10 and get
+# 2.6
+PY_VERSION = sys.version_info[0] + (sys.version_info[1] / 10.0)
+
+bytecode = "%s-%s.pyc" % (basename, PY_VERSION)
+
+import py_compile
+print("compiling %s to %s" % (source, bytecode))
+py_compile.compile(source, bytecode, 'exec')
+# import os
+# os.system("../bin/uncompyle6 %s" % bytecode)

test/stdlib/runtests.sh

@@ -0,0 +1,58 @@
+#!/bin/bash
+me=${BASH_SOURCE[0]}
+
+# Python version setup
+FULLVERSION=$(pyenv local)
+PYVERSION=${FULLVERSION%.*}
+MINOR=${FULLVERSION##?.?.}
+
+typeset -i STOP_ONERROR=1
+typeset -A SKIP_TESTS=( [test_aepack.py]=1 [audiotests.py]=1)
+
+# Test directory setup
+srcdir=$(dirname $me)
+cd $srcdir
+fulldir=$(pwd)
+TESTDIR=/tmp/test${PYVERSION}
+if [[ -e $TESTDIR ]] ; then
+    rm -fr $TESTDIR
+fi
+mkdir $TESTDIR || exit $?
+cp -r ~/.pyenv/versions/${PYVERSION}.${MINOR}/lib/python${PYVERSION}/test $TESTDIR
+cd $TESTDIR/test
+export PYTHONPATH=$TESTDIR
+
+# Run tests
+typeset -i i=0
+typeset -i allerrs=0
+for file in test_*.py; do
+    [[ -v SKIP_TESTS[$file] ]] && continue
+
+    # If the fails *before* decompiling, skip it!
+    if ! python $file >/dev/null 2>&1 ; then
+	continue
+    fi
+
+    ((i++))
+    # (( i > 40 )) && break
+    short_name=$(basename $file .py)
+    decompiled_file=$short_name-${PYVERSION}.pyc
+    $fulldir/compile-file.py $file && \
+    mv $file{,.orig} && \
+    $fulldir/../../bin/uncompyle6 $decompiled_file > $file
+    rc=$?
+    if (( rc == 0 )) ; then
+	echo ========== Running $file ===========
+	python $file
+	rc=$?
+    else
+	echo ======= Skipping $file due to compile/decompile errors ========
+    fi
+    (( rc != 0 && allerrs++ ))
+    if (( STOP_ONERROR && rc )) ; then
+	echo "** Ran $i tests before failure **"
+	exit $allerrs
+    fi
+done
+echo "Ran $i tests"
+exit $allerrs

test/test_pyenvlib.py

@@ -28,10 +28,11 @@ from fnmatch import fnmatch
 TEST_VERSIONS=('2.3.7', '2.4.6', '2.5.6', '2.6.9',
                'pypy-2.4.0', 'pypy-2.6.1',
                'pypy-5.0.1', 'pypy-5.3.1', 'pypy3.5-5.7.1-beta',
-               '2.7.10', '2.7.11', '2.7.12', '2.7.13',
+               '2.7.10', '2.7.11', '2.7.12', '2.7.13', '2.7.14',
                '3.0.1', '3.1.5', '3.2.6',
                '3.3.5', '3.3.6',
-               '3.4.2', '3.5.1', '3.6.0', 'native')
+               '3.4.2', '3.5.1', '3.5.2', '3.6.0', '3.6.3',
+               'native')
 
 target_base = '/tmp/py-dis/'
 lib_prefix = os.path.join(os.environ['HOME'], '.pyenv/versions')

uncompyle6/bin/uncompile.py

@@ -174,7 +174,7 @@ def main_bin():
         try:
             from Queue import Empty
         except ImportError:
-            from Queue import Empty
+            from queue import Empty
 
         fqueue = Queue(len(files)+numproc)
         for f in files:

uncompyle6/main.py

@@ -216,7 +216,7 @@ def main(in_base, out_base, files, codes, outfile=None,
                 if not current_outfile:
                     mess = '\n# okay decompiling'
                     # mem_usage = __memUsage()
-                    print(mess, infile)
+                    print mess, infile
         if current_outfile:
             sys.stdout.write("%s\r" %
                              status_msg(do_verify, tot_files, okay_files, failed_files, verify_failed_files))

uncompyle6/parser.py

@@ -3,7 +3,7 @@
 #  Copyright (c) 2000-2002 by hartmut Goebel <h.goebel@crazy-compilers.com>
 #  Copyright (c) 1999 John Aycock
 """
-Common parser routines.
+Common uncompyle6 parser routines.
 """
 
 import sys
@@ -28,13 +28,16 @@ class PythonParser(GenericASTBuilder):
 
     def __init__(self, AST, start, debug):
         super(PythonParser, self).__init__(AST, start, debug)
-        self.collect = [
+        # FIXME: customize per python parser version
+        nt_list = [
             'stmts', 'except_stmts', '_stmts', 'load_attrs',
             'exprlist', 'kvlist', 'kwargs', 'come_froms', '_come_from',
+            'importlist',
             # Python < 3
             'print_items',
             # PyPy:
             'kvlist_n']
+        self.collect = frozenset(nt_list)
 
     def ast_first_offset(self, ast):
         if hasattr(ast, 'offset'):
@@ -121,21 +124,25 @@ class PythonParser(GenericASTBuilder):
 
     def error(self, instructions, index):
         # Find the last line boundary
+        start, finish = -1, -1
         for start in range(index, -1, -1):
             if instructions[start].linestart:  break
             pass
         for finish in range(index+1, len(instructions)):
             if instructions[finish].linestart:  break
             pass
-        err_token = instructions[index]
-        print("Instruction context:")
-        for i in range(start, finish):
-            if i != index:
-                indent = '   '
-            else:
-                indent = '-> '
-            print("%s%s" % (indent, instructions[i]))
-        raise ParserError(err_token, err_token.offset)
+        if start > 0:
+            err_token = instructions[index]
+            print("Instruction context:")
+            for i in range(start, finish):
+                if i != index:
+                    indent = '   '
+                else:
+                    indent = '-> '
+                print "%s%s" % (indent, instructions[i])
+            raise ParserError(err_token, err_token.offset)
+        else:
+            raise ParserError(None, -1)
 
     def typestring(self, token):
         return token.kind
@@ -196,7 +203,6 @@ class PythonParser(GenericASTBuilder):
         c_stmts ::= continue_stmts
 
         lastc_stmt ::= iflaststmt
-        lastc_stmt ::= whileelselaststmt
         lastc_stmt ::= forelselaststmt
         lastc_stmt ::= ifelsestmtr
         lastc_stmt ::= ifelsestmtc
@@ -224,6 +230,8 @@ class PythonParser(GenericASTBuilder):
         suite_stmts ::= continue_stmts
 
         suite_stmts_opt ::= suite_stmts
+
+        # passtmt is needed for semantic actions to add "pass"
         suite_stmts_opt ::= passstmt
 
         else_suite ::= suite_stmts
@@ -232,7 +240,6 @@ class PythonParser(GenericASTBuilder):
         else_suitec ::= return_stmts
 
         stmt ::= assert
-        stmt ::= assert2
 
         stmt ::= classdef
         stmt ::= call_stmt
@@ -262,6 +269,10 @@ class PythonParser(GenericASTBuilder):
         return_stmt ::= ret_expr RETURN_VALUE
         return_stmt_lambda ::= ret_expr RETURN_VALUE_LAMBDA
 
+        # return_stmts are a sequence of statements that ends in a RETURN statement.
+        # In later Python versions with jump optimization, this can cause JUMPs
+        # that would normally appear to be omitted.
+
         return_stmts ::= return_stmt
         return_stmts ::= _stmts return_stmt
 
@@ -304,14 +315,6 @@ class PythonParser(GenericASTBuilder):
         come_from_opt ::= COME_FROM?
         """
 
-    def p_dictcomp(self, args):
-        '''
-        expr ::= dictcomp
-        stmt ::= dictcomp_func
-        dictcomp_func ::= BUILD_MAP_0 LOAD_FAST FOR_ITER designator
-                comp_iter JUMP_BACK RETURN_VALUE RETURN_LAST
-        '''
-
     def p_augmented_assign(self, args):
         '''
         stmt ::= augassign1
@@ -372,25 +375,6 @@ class PythonParser(GenericASTBuilder):
                 for_block POP_BLOCK else_suitel _come_from
         """
 
-    def p_whilestmt(self, args):
-        """
-        whilestmt  ::= SETUP_LOOP testexpr l_stmts_opt JUMP_BACK
-                       POP_BLOCK _come_from
-
-        whilestmt  ::= SETUP_LOOP testexpr return_stmts
-                       POP_BLOCK COME_FROM
-
-        whileelsestmt ::= SETUP_LOOP testexpr
-                l_stmts_opt JUMP_BACK
-                POP_BLOCK
-                else_suite COME_FROM
-
-        whileelselaststmt ::= SETUP_LOOP testexpr
-                l_stmts_opt JUMP_BACK
-                POP_BLOCK
-                else_suitec COME_FROM
-        """
-
     def p_import20(self, args):
         """
         stmt ::= importstmt
@@ -398,23 +382,18 @@ class PythonParser(GenericASTBuilder):
         stmt ::= importstar
         stmt ::= importmultiple
 
-        importlist2 ::= importlist2 import_as
-        importlist2 ::= import_as
-        import_as ::= IMPORT_NAME designator
-        import_as ::= IMPORT_NAME load_attrs designator
-        import_as ::= IMPORT_FROM designator
+        importlist ::= importlist import_as
+        importlist ::= import_as
+        import_as  ::= IMPORT_NAME designator
+        import_as  ::= IMPORT_FROM designator
 
         importstmt ::= LOAD_CONST LOAD_CONST import_as
         importstar ::= LOAD_CONST LOAD_CONST IMPORT_NAME IMPORT_STAR
-        importfrom ::= LOAD_CONST LOAD_CONST IMPORT_NAME importlist2 POP_TOP
+        importfrom ::= LOAD_CONST LOAD_CONST IMPORT_NAME importlist POP_TOP
         importmultiple ::= LOAD_CONST LOAD_CONST import_as imports_cont
 
-        imports_cont ::= imports_cont import_cont
-        imports_cont ::= import_cont
-        import_cont ::= LOAD_CONST LOAD_CONST import_as_cont
-        import_as_cont ::= IMPORT_FROM designator
-
-        load_attrs ::= LOAD_ATTR+
+        imports_cont ::= import_cont+
+        import_cont  ::= LOAD_CONST LOAD_CONST import_as
         """
 
     def p_list_comprehension(self, args):
@@ -440,15 +419,12 @@ class PythonParser(GenericASTBuilder):
         setcomp ::= LOAD_SETCOMP MAKE_FUNCTION_0 expr GET_ITER CALL_FUNCTION_1
 
         comp_iter ::= comp_if
-        comp_iter ::= comp_ifnot
         comp_iter ::= comp_for
         comp_iter ::= comp_body
         comp_body ::= gen_comp_body
         gen_comp_body ::= expr YIELD_VALUE POP_TOP
 
-        comp_if ::= expr jmp_false comp_iter
-        comp_ifnot ::= expr jmp_true comp_iter
-
+        comp_if  ::= expr jmp_false comp_iter
         comp_for ::= expr _for designator comp_iter JUMP_BACK
         """
 
@@ -463,10 +439,9 @@ class PythonParser(GenericASTBuilder):
         expr ::= load_attr
         expr ::= binary_expr
         expr ::= build_list
-        expr ::= cmp
+        expr ::= compare
         expr ::= mapexpr
         expr ::= and
-        expr ::= and2
         expr ::= or
         expr ::= unary_expr
         expr ::= call_function
@@ -474,31 +449,26 @@ class PythonParser(GenericASTBuilder):
         expr ::= binary_subscr
         expr ::= binary_subscr2
         expr ::= get_iter
-        expr ::= buildslice2
-        expr ::= buildslice3
         expr ::= yield
 
-        # Possibly Python < 2.3
-        # expr ::= SET_LINENO
-
         binary_expr ::= expr expr binary_op
-        binary_op ::= BINARY_ADD
-        binary_op ::= BINARY_MULTIPLY
-        binary_op ::= BINARY_AND
-        binary_op ::= BINARY_OR
-        binary_op ::= BINARY_XOR
-        binary_op ::= BINARY_SUBTRACT
-        binary_op ::= BINARY_TRUE_DIVIDE
-        binary_op ::= BINARY_FLOOR_DIVIDE
-        binary_op ::= BINARY_MODULO
-        binary_op ::= BINARY_LSHIFT
-        binary_op ::= BINARY_RSHIFT
-        binary_op ::= BINARY_POWER
+        binary_op   ::= BINARY_ADD
+        binary_op   ::= BINARY_MULTIPLY
+        binary_op   ::= BINARY_AND
+        binary_op   ::= BINARY_OR
+        binary_op   ::= BINARY_XOR
+        binary_op   ::= BINARY_SUBTRACT
+        binary_op   ::= BINARY_TRUE_DIVIDE
+        binary_op   ::= BINARY_FLOOR_DIVIDE
+        binary_op   ::= BINARY_MODULO
+        binary_op   ::= BINARY_LSHIFT
+        binary_op   ::= BINARY_RSHIFT
+        binary_op   ::= BINARY_POWER
 
-        unary_expr ::= expr unary_op
-        unary_op ::= UNARY_POSITIVE
-        unary_op ::= UNARY_NEGATIVE
-        unary_op ::= UNARY_INVERT
+        unary_expr  ::= expr unary_op
+        unary_op    ::= UNARY_POSITIVE
+        unary_op    ::= UNARY_NEGATIVE
+        unary_op    ::= UNARY_INVERT
 
         unary_not ::= expr UNARY_NOT
 
@@ -506,24 +476,13 @@ class PythonParser(GenericASTBuilder):
 
         load_attr ::= expr LOAD_ATTR
         get_iter ::= expr GET_ITER
-        buildslice3 ::= expr expr expr BUILD_SLICE_3
-        buildslice2 ::= expr expr BUILD_SLICE_2
 
         yield ::= expr YIELD_VALUE
 
-        _mklambda ::= load_closure mklambda
         _mklambda ::= mklambda
 
-        # "and" where the first part of the and is true,
-        # so there is only the 2nd part to evaluate
-        and2 ::= _jump jmp_false COME_FROM expr COME_FROM
-
         expr ::= conditional
         conditional ::= expr jmp_false expr JUMP_FORWARD expr COME_FROM
-        conditional ::= expr jmp_false expr JUMP_ABSOLUTE expr
-
-        expr ::= conditionalnot
-        conditionalnot  ::= expr jmp_true expr _jump expr COME_FROM
 
         expr            ::= conditionalTrue
         conditionalTrue ::= expr JUMP_FORWARD expr COME_FROM
@@ -542,32 +501,22 @@ class PythonParser(GenericASTBuilder):
         return_lambda ::= ret_expr RETURN_VALUE_LAMBDA LAMBDA_MARKER
         return_lambda ::= ret_expr RETURN_VALUE_LAMBDA
 
-        conditional_lambda ::= expr jmp_false return_if_stmt return_stmt LAMBDA_MARKER
+        # Doesn't seem to be used anymore, but other conditional_lambda's are
+        # conditional_lambda ::= expr jmp_false return_if_stmt return_stmt LAMBDA_MARKER
+
+        compare        ::= compare_chained
+        compare        ::= compare_single
+        compare_single ::= expr expr COMPARE_OP
+
+        # A compare_chained is two comparisions like x <= y <= z
+        compare_chained  ::= expr compare_chained1 ROT_TWO POP_TOP _come_from
+        compare_chained2 ::= expr COMPARE_OP JUMP_FORWARD
 
-        cmp ::= cmp_list
-        cmp ::= compare
-        compare ::= expr expr COMPARE_OP
-        cmp_list ::= expr cmp_list1 ROT_TWO POP_TOP
-                _come_from
-        cmp_list1 ::= expr DUP_TOP ROT_THREE
-                COMPARE_OP jmp_false
-                cmp_list1 _come_from
-        cmp_list1 ::= expr DUP_TOP ROT_THREE
-                COMPARE_OP jmp_false
-                cmp_list2 _come_from
-        cmp_list2 ::= expr COMPARE_OP JUMP_FORWARD
-        cmp_list2 ::= expr COMPARE_OP RETURN_VALUE
         mapexpr ::= BUILD_MAP kvlist
 
-        kvlist ::= kvlist kv
-        kvlist ::= kvlist kv2
-        kvlist ::= kvlist kv3
+        # Non-null kvlist items are broken out in the indiviual grammars
         kvlist ::=
 
-        kv ::= DUP_TOP expr ROT_TWO expr STORE_SUBSCR
-        kv2 ::= DUP_TOP expr expr ROT_THREE STORE_SUBSCR
-        kv3 ::= expr expr STORE_MAP
-
         exprlist ::= exprlist expr
         exprlist ::= expr
 
@@ -591,15 +540,14 @@ class PythonParser(GenericASTBuilder):
         ##   designLists ::=
         ## Will need to redo semantic actiion
 
-        designator ::= STORE_FAST
-        designator ::= STORE_NAME
-        designator ::= STORE_GLOBAL
-        designator ::= STORE_DEREF
-        designator ::= expr STORE_ATTR
-        designator ::= store_subscr
+        designator   ::= STORE_FAST
+        designator   ::= STORE_NAME
+        designator   ::= STORE_GLOBAL
+        designator   ::= STORE_DEREF
+        designator   ::= expr STORE_ATTR
+        designator   ::= store_subscr
         store_subscr ::= expr expr STORE_SUBSCR
-        designator ::= unpack
-        designator ::= unpack_list
+        designator   ::= unpack
         '''
 
 
@@ -785,4 +733,4 @@ if __name__ == '__main__':
         ast = python_parser(PYTHON_VERSION, co, showasm=True, is_pypy=IS_PYPY)
         print(ast)
         return
-    parse_test(parse_test.__code__)
+    # parse_test(parse_test.__code__)