Get ready for release 2.9.10

and some parser list nonterminal cleanup

ChangeLog

@@ -1,6 +1,523 @@
+2017-02-25  rocky <rb@dustyfeet.com>
+
+	* uncompyle6/version.py: Get ready for release 2.9.10
+
+2017-02-25  rocky <rb@dustyfeet.com>
+
+	* uncompyle6/parser.py, uncompyle6/parsers/parse26.py: Python 2.6
+	parsing bugs ..  and some parser list nonterminal cleanup
+
+2017-02-24  rocky <rb@dustyfeet.com>
+
+	* test/simple_source/bug25/03_if_for.py,
+	uncompyle6/parsers/parse26.py: Python 2.6 control flow bug with
+	added COME_FROM
+
+2017-02-22  rocky <rb@dustyfeet.com>
+
+	* test/simple_source/bug25/02_try_else.py,
+	uncompyle6/parsers/parse25.py: Python 2.5 wasn't handling tryelse
+	properly
+
+2017-02-20  rocky <rb@dustyfeet.com>
+
+	* : New test doesn't --verify correctly. Sigh.
+
+2017-02-20  rocky <rb@dustyfeet.com>
+
+	* test/simple_source/bug33/02_while1.py: Add test for last while1
+	bug fix
+
+2017-02-20  rocky <rb@dustyfeet.com>
+
+	* uncompyle6/parsers/parse3.py, uncompyle6/parsers/parse35.py: 
+	Python 3.x needs more "while 1" grammar rules
+
+2017-02-20  rocky <rb@dustyfeet.com>
+
+	* uncompyle6/parsers/parse3.py, uncompyle6/parsers/parse35.py,
+	uncompyle6/scanners/scanner3.py: Some Python 3.4 bugss fixed by
+	using 3.5 rules
+
+2017-02-20  rocky <rb@dustyfeet.com>
+
+	* test/simple_source/exception/02_try_finally.py,
+	uncompyle6/parsers/parse3.py, uncompyle6/scanners/scanner3.py: More
+	COME_FROM's in Python 3...  Need this to find boundaries of simple if better
+
+2017-02-19  rocky <rb@dustyfeet.com>
+
+	* uncompyle6/parsers/parse26.py: Marginally better for Python 2.6
+	but...  control flow is still wrong.
+
+2017-02-10  rocky <rb@dustyfeet.com>
+
+	* : commit f9d47abb2be7c3839df06c0ed69d3d513694af4e Author: rocky
+	<rb@dustyfeet.com> Date:   Fri Feb 10 02:07:04 2017 -0500
+
+2017-02-10  rocky <rb@dustyfeet.com>
+
+	* test/simple_source/bug22/01_ops.py, test/test_pythonlib.py: Beef
+	up grammar coverage
+
+2017-01-29  rocky <rb@dustyfeet.com>
+
+	* test/Makefile, test/simple_source/bug22/01_ops.py,
+	uncompyle6/parsers/parse25.py, uncompyle6/semantics/consts.py,
+	uncompyle6/semantics/pysource.py: Changes based on grammar coverage
+	info
+
+2017-01-29  R. Bernstein <rocky@users.noreply.github.com>
+
+	* : Merge pull request #83 from rocky/coverage Coverage
+
+2017-01-29  rocky <rb@dustyfeet.com>
+
+	* test/simple_source/bug22/01_ops.py, test/test_pyenvlib.py: Add
+	--coverage to test_pyenvlib and ...  improve grammar coverage on 2.7
+
+2017-01-29  rocky <rb@dustyfeet.com>
+
+	* : commit 3dc766d0a9537842470c7b4f79e8ccb3d5a46843 Author: rocky
+	<rb@dustyfeet.com> Date:   Sun Jan 29 07:34:49 2017 -0500
+
+2017-01-29  rocky <rb@dustyfeet.com>
+
+	* test/test_pythonlib.py: Add --coverage option. WOOT!
+
+2017-01-27  rocky <rb@dustyfeet.com>
+
+	* __pkginfo__.py: Bump min spark_parser version
+
+2017-01-24  rocky <rb@dustyfeet.com>
+
+	* uncompyle6/scanners/scanner2.py, uncompyle6/semantics/consts.py,
+	uncompyle6/semantics/pysource.py: More 2.6, 2.7 control flow Todo more COME_FROMs but now need to check targets better. In some
+	cases we're relying on grammar ambiguity to work out right and in
+	2.7 it doesn't
+
+2017-01-24  rocky <rb@dustyfeet.com>
+
+	* uncompyle6/parsers/parse2.py, uncompyle6/parsers/parse27.py,
+	uncompyle6/scanners/scanner2.py, uncompyle6/semantics/consts.py,
+	uncompyle6/semantics/pysource.py: More 2.6, 2.7 control-flow bugs Wasn't limiting exception clause to try finally. Probably still has
+	bugs in try-finally nesting Add another 2.6/2.7 COME_FROM to try to limit if/end scope better
+
+2017-01-23  rocky <rb@dustyfeet.com>
+
+	* uncompyle6/scanners/scanner2.py,
+	uncompyle6/scanners/scanner26.py, uncompyle6/verify.py: Improve
+	Python 2.6 & 2.7 verification
+
+2017-01-22  rocky <rb@dustyfeet.com>
+
+	* uncompyle6/parsers/parse30.py, uncompyle6/verify.py: Fix up Python
+	3.0 handling
+
+2017-01-21  rocky <rb@dustyfeet.com>
+
+	* : commit 545a46dffaa0fe2246dd7cc1b560c58db525c2b4 Author: rocky
+	<rb@dustyfeet.com> Date:   Sat Jan 21 06:24:31 2017 -0500
+
+2017-01-20  rocky <rb@dustyfeet.com>
+
+	* uncompyle6/parsers/parse3.py, uncompyle6/scanners/scanner3.py,
+	uncompyle6/semantics/pysource.py: Handle BUILD_CONST_KEY_MAP as a
+	varargs custom rules with BUILD_CONST_KEY_MAP are pinned to the specific
+	number of args seen.
+
+2017-01-19  R. Bernstein <rocky@users.noreply.github.com>
+
+	* : Merge pull request #81 from moagstar/BUILD_CONST_KEY_MAP fixed bug with BUILD_CONST_KEY_MAP
+
+2017-01-19  R. Bernstein <rocky@users.noreply.github.com>
+
+	* : Merge pull request #80 from moagstar/BUILD_CONST_KEY_MAP Build const key map
+
+2017-01-18  Daniel Bradburn <moagstar@gmail.com>
+
+	* uncompyle6/semantics/pysource.py: added generation of dict display
+	from BUILD_CONST_KEY_MAP
+
+2017-01-18  Daniel Bradburn <moagstar@gmail.com>
+
+	* pytest/test_build_const_key_map.py: fixed typo
+
+2017-01-18  Daniel Bradburn <moagstar@gmail.com>
+
+	* pytest/test_build_const_key_map.py: added some more test cases for
+	BUILD_CONST_KEY_MAP
+
+2017-01-17  Daniel Bradburn <moagstar@gmail.com>
+
+	* pytest/test_build_const_key_map.py: simplified test cases for
+	test_build_const_key_map
+
+2017-01-17  Daniel Bradburn <moagstar@gmail.com>
+
+	* pytest/test_build_const_key_map.py, pytest/validate.py: added
+	validation code for checking decompilation of an expression
+
+2017-01-15  rocky <rb@dustyfeet.com>
+
+	* uncompyle6/parsers/parse3.py, uncompyle6/scanners/scanner3.py: 
+	Handle 3.6 BUILD_CONST_KEYMAP
+
+2017-01-15  rocky <rb@dustyfeet.com>
+
+	* uncompyle6/scanners/scanner2.py: Python 2.1 doesn't have FOR_ITER
+	or GET_ITER...  adjust locgic for this fact
+
+2017-01-12  rocky <rb@dustyfeet.com>
+
+	* uncompyle6/__init__.py: sys.recursionlimit is optional, not
+	essential
+
+2017-01-11  rocky <rb@dustyfeet.com>
+
+	* : commit b131c20e99514d3a969a51e841d3a823017f1beb Author: rocky
+	<rb@dustyfeet.com> Date:   Wed Jan 11 21:32:26 2017 -0500
+
+2017-01-11  rocky <rb@dustyfeet.com>
+
+	* ChangeLog, NEWS: Get ready for release 2.10.9
+
+2017-01-11  R. Bernstein <rocky@users.noreply.github.com>
+
+	* : Merge pull request #79 from rocky/revert-78-patch-1 Revert "fix bug : not generate all files when use "-ro""
+
+2017-01-11  R. Bernstein <rocky@users.noreply.github.com>
+
+	* : Merge pull request #78 from jlugjb/patch-1 fix bug : not generate all files when use "-ro"
+
+2017-01-10  rocky <rb@dustyfeet.com>
+
+	* test/simple_source/bug35/03_double_star_unpack.py,
+	uncompyle6/parsers/parse3.py, uncompyle6/semantics/pysource.py: 
+	Improve BUILD_xxx_UNPACK slightly
+
+2017-01-09  rocky <rb@dustyfeet.com>
+
+	* test/simple_source/bug35/03_async_await.py,
+	uncompyle6/parsers/parse3.py, uncompyle6/semantics/pysource.py: Add
+	async_call_function for 3.5+
+
+2017-01-09  rocky <rb@dustyfeet.com>
+
+	* : Reinstate test
+
+2017-01-08  rocky <rb@dustyfeet.com>
+
+	* : Works now
+
+2017-01-08  rocky <rb@dustyfeet.com>
+
+	* uncompyle6/parsers/parse30.py, uncompyle6/scanners/scanner3.py: 
+	Python 3.0 decompile bugs
+
+2017-01-08  rocky <rb@dustyfeet.com>
+
+	* uncompyle6/scanners/scanner2.py, uncompyle6/scanners/scanner3.py,
+	uncompyle6/scanners/scanner30.py: Towards better 3.0 decompilation Sync scanner2 and scanner3 better
+
+2017-01-08  rocky <rb@dustyfeet.com>
+
+	* test/simple_source/bug35/03_while-if-break.py,
+	uncompyle6/parsers/parse35.py, uncompyle6/scanner.py,
+	uncompyle6/scanners/scanner3.py: Fix 3.5, 3.6 while true if/break
+	bug
+
+2017-01-08  rocky <rb@dustyfeet.com>
+
+	* uncompyle6/__init__.py, uncompyle6/main.py,
+	uncompyle6/semantics/consts.py, uncompyle6/semantics/fragments.py,
+	uncompyle6/semantics/pysource.py: Misc cleanups Favor "decompile" over "uncompyle" since "decompile" is in common
+	use Reduce size of pysource.py by splitting out constants
+
+2017-01-08  rocky <rb@dustyfeet.com>
+
+	* test/simple_source/bug35/03_async_await.py,
+	uncompyle6/parsers/parse35.py, uncompyle6/scanners/scanner3.py,
+	uncompyle6/semantics/pysource.py: Add 3.5+ async with/for ..  scanner3.py: 3.6 bytecode vs wordcode fix
+
+2017-01-07  rocky <rb@dustyfeet.com>
+
+	* test/simple_source/bug35/03_async_await.py,
+	uncompyle6/parsers/parse35.py, uncompyle6/semantics/pysource.py: 
+	Start to add 3.5+ await and async
+
+2017-01-07  rocky <rb@dustyfeet.com>
+
+	* test/simple_source/bug31/04_def_annotate.py,
+	uncompyle6/semantics/make_function.py: More Python 3 annotation bugs
+
+2017-01-07  rocky <rb@dustyfeet.com>
+
+	* uncompyle6/parsers/parse3.py, uncompyle6/parsers/parse31.py,
+	uncompyle6/parsers/parse32.py,
+	uncompyle6/semantics/make_function.py,
+	uncompyle6/semantics/pysource.py: Fix some errors in deparsing
+	Python 3 annotations
+
+2017-01-07  rocky <rb@dustyfeet.com>
+
+	* uncompyle6/semantics/make_function.py: Small Pyhton 3.x annotate
+	bug
+
+2017-01-03  rocky <rb@dustyfeet.com>
+
+	* README.rst: Note what's up with Python 3 decompile quality
+
+2017-01-03  rocky <rb@dustyfeet.com>
+
+	* uncompyle6/scanners/scanner3.py: 3.5 continue check is needed on
+	3.6
+
+2017-01-03  rocky <rb@dustyfeet.com>
+
+	* test/test_pyenvlib.py, uncompyle6/parsers/parse36.py,
+	uncompyle6/scanners/scanner3.py: Towards better 3.6 support
+
+2017-01-02  rocky <rb@dustyfeet.com>
+
+	* uncompyle6/parsers/parse35.py, uncompyle6/scanners/scanner3.py: 
+	Python 3.5 continue detection bug
+
+2017-01-01  rocky <rb@dustyfeet.com>
+
+	* uncompyle6/scanners/scanner3.py: add come_from for setup_finally
+	and setup_except
+
+2017-01-01  rocky <rb@dustyfeet.com>
+
+	* uncompyle6/parsers/parse35.py, uncompyle6/scanners/scanner3.py: 
+	Towards fixing Python 3.5 return bugs
+
+2017-01-01  rocky <rb@dustyfeet.com>
+
+	* README.rst: Note how to verify correctness ...  with --verify, --weak-verify and cross checking with pycdc
+
+2016-12-31  rocky <rb@dustyfeet.com>
+
+	* ChangeLog, NEWS, uncompyle6/version.py: Get ready for release
+	2.9.9
+
+2016-12-31  rocky <rb@dustyfeet.com>
+
+	* uncompyle6/parsers/parse26.py: 2.x list_if may have a THEN in it
+
+2016-12-31  rocky <rb@dustyfeet.com>
+
+	* uncompyle6/scanners/scanner3.py: Towards fixing a Python 3.3
+	return/continue bug
+
+2016-12-30  rocky <rb@dustyfeet.com>
+
+	* uncompyle6/main.py: On --verify if we can't unbuffer output, don't
+
+2016-12-29  rocky <rb@dustyfeet.com>
+
+	* uncompyle6/scanners/scanner2.py, uncompyle6/scanners/scanner3.py: 
+	dectect_structure() -> detect_control_flow()
+
+2016-12-29  rocky <rb@dustyfeet.com>
+
+	* uncompyle6/scanners/scanner2.py, uncompyle6/scanners/scanner3.py: 
+	DRY code and emitted Python 3 source * Python 3: break; continue -> break * Use variable in detect_structure for pre[rtarget] * Make Python 2 and Python 3 detect_structure more alie
+
+2016-12-29  rocky <rb@dustyfeet.com>
+
+	* uncompyle6/scanners/scanner3.py: More if/then detection in Python
+	3.x
+
+2016-12-29  R. Bernstein <rocky@users.noreply.github.com>
+
+	* : Merge pull request #73 from rocky/then-crap Add THEN token to improve Python 2.2-2.6 control flow detection
+
+2016-12-28  rocky <rb@dustyfeet.com>
+
+	* uncompyle6/parsers/parse3.py, uncompyle6/scanners/tok.py: Misc
+	bugs
+
+2016-12-28  rocky <rb@dustyfeet.com>
+
+	* : commit 723fa5dfed5bb198c66741c594e2c277ded88970 Author: rocky
+	<rb@dustyfeet.com> Date:   Wed Dec 28 18:57:09 2016 -0500
+
+2016-12-28  rocky <rb@dustyfeet.com>
+
+	* uncompyle6/parsers/parse3.py, uncompyle6/parsers/parse32.py,
+	uncompyle6/parsers/parse33.py: Towards fixing a 3.2 while true: ...
+	break bug
+
+2016-12-28  rocky <rb@dustyfeet.com>
+
+	* test/Makefile, uncompyle6/main.py, uncompyle6/parsers/parse26.py,
+	uncompyle6/verify.py: Bugs in Python 2.6- "and" and "lambda"
+	handling ..  and clean up verify output
+
+2016-12-27  rocky <rb@dustyfeet.com>
+
+	* uncompyle6/parsers/parse26.py, uncompyle6/scanners/scanner2.py,
+	uncompyle6/scanners/scanner26.py, uncompyle6/semantics/pysource.py: 
+	WIP : Add THEN to disambigute from "and"
+
+2016-12-27  rocky <rb@dustyfeet.com>
+
+	* uncompyle6/scanners/scanner2.py: Make 2.6 and 2.7 ingest more
+	alike
+
+2016-12-26  rocky <rb@dustyfeet.com>
+
+	* : Update 2.7 bytecode file for last fix
+
+2016-12-26  R. Bernstein <rocky@users.noreply.github.com>
+
+	* : Merge pull request #71 from jiangpengcheng/tupple_bug tupples which contain only 1 element need a comma
+
+2016-12-26  jiangpch <jiangpch@gohighsec.com>
+
+	* uncompyle6/semantics/pysource.py: tupples which contain only 1
+	element need a comma
+
+2016-12-26  rocky <rb@dustyfeet.com>
+
+	* uncompyle6/parsers/parse25.py: fix bug in using python2 AST rules
+	in python 2.5
+
+2016-12-26  rocky <rb@dustyfeet.com>
+
+	* : commit f1a947f106b231fb1480ba301b15e3ceaf78c94f Author: rocky
+	<rb@dustyfeet.com> Date:   Mon Dec 26 00:43:02 2016 -0500
+
+2016-12-25  rocky <rb@dustyfeet.com>
+
+	* uncompyle6/scanners/scanner23.py,
+	uncompyle6/scanners/scanner24.py, uncompyle6/semantics/pysource.py,
+	uncompyle6/verify.py: Scanner call fixes. NAME_MODULE removal for
+	<=2.4
+
+2016-12-24  rocky <rb@dustyfeet.com>
+
+	* uncompyle6/parsers/astnode.py, uncompyle6/parsers/parse2.py,
+	uncompyle6/parsers/parse26.py, uncompyle6/parsers/parse3.py,
+	uncompyle6/parsers/parse36.py, uncompyle6/scanners/scanner15.py,
+	uncompyle6/scanners/scanner2.py, uncompyle6/scanners/scanner21.py,
+	uncompyle6/scanners/scanner22.py,
+	uncompyle6/semantics/fragments.py, uncompyle6/semantics/pysource.py: 
+	Lint
+
+2016-12-24  rocky <rb@dustyfeet.com>
+
+	* uncompyle6/semantics/pysource.py: Remove stray debug hook
+
+2016-12-20  rocky <rb@dustyfeet.com>
+
+	* uncompyle6/semantics/pysource.py: Bang on 3.6
+	build_map_unpack_with_call Probably will fix better in the future.
+
+2016-12-18  rocky <rb@dustyfeet.com>
+
+	* uncompyle6/bin/pydisassemble.py, uncompyle6/bin/uncompile.py,
+	uncompyle6/parsers/parse2.py, uncompyle6/parsers/parse25.py,
+	uncompyle6/parsers/parse27.py, uncompyle6/parsers/parse3.py,
+	uncompyle6/scanners/scanner2.py, uncompyle6/scanners/scanner3.py: 
+	Python flake8 crap Was testing realgud's C-x!8 (goto flake8 warning/error)
+
+2016-12-18  rocky <rb@dustyfeet.com>
+
+	* pytest/.gitignore, test/simple_source/bug25/02_try_else.py,
+	uncompyle6/parsers/parse25.py, uncompyle6/parsers/parse26.py: Python
+	2.5 mistaken try/else
+
+2016-12-17  rocky <rb@dustyfeet.com>
+
+	* uncompyle6/scanners/scanner2.py, uncompyle6/scanners/scanner25.py: 
+	show-asm on python2.5 is optional make scanner2 look a little more like scanner3
+
+2016-12-16  rocky <rb@dustyfeet.com>
+
+	* NEWS: Release 2.9.8 news
+
+2016-12-16  rocky <rb@dustyfeet.com>
+
+	* __pkginfo__.py, uncompyle6/version.py: Get ready for release 2.9.8
+
+2016-12-16  rocky <rb@dustyfeet.com>
+
+	* test/simple_source/bug35/02_build_map_unpack_with_call.py,
+	uncompyle6/parsers/parse3.py, uncompyle6/parsers/parse35.py,
+	uncompyle6/parsers/parse36.py, uncompyle6/scanners/scanner3.py,
+	uncompyle6/semantics/pysource.py: Start to handle 3.5
+	build_map_unpack_with_call 3.6 also started but needs even more work
+
+2016-12-15  rocky <rb@dustyfeet.com>
+
+	* uncompyle6/scanner.py, uncompyle6/scanners/scanner3.py: Some
+	Python 3.6 bytecode->wordcode fixes
+
+2016-12-13  rocky <rb@dustyfeet.com>
+
+	* uncompyle6/semantics/fragments.py: Was passing wrong type
+
+2016-12-11  rocky <rb@dustyfeet.com>
+
+	* uncompyle6/parser.py: option -g: show start-end range when
+	possible
+
+2016-12-11  rocky <rb@dustyfeet.com>
+
+	* uncompyle6/semantics/make_function.py, uncompyle6/verify.py: two
+	misc changes - track print_docstring move to help (used in python 3.1) - verify: allow RETURN_VALUE to match RETURN_END_IF
+
+2016-12-10  rocky <rb@dustyfeet.com>
+
+	* .travis.yml, test/Makefile: 3.2 needs --weak-verify
+
+2016-12-10  rocky <rb@dustyfeet.com>
+
+	* .travis.yml: Try testing on 3.2
+
+2016-12-10  rocky <rb@dustyfeet.com>
+
+	* __pkginfo__.py, uncompyle6/bin/uncompile.py: Can run in Python 3.1
+	and Python 3.2
+
+2016-12-10  rocky <rb@dustyfeet.com>
+
+	* test/Makefile, uncompyle6/parsers/parse3.py,
+	uncompyle6/scanners/scanner3.py: Another python 3 ELSE fixes and ...  Makefile:   - test python 3.0 bytecode   - turn full --verify back on Python 3.x
+
+2016-12-10  rocky <rb@dustyfeet.com>
+
+	* uncompyle6/scanners/scanner3.py: Another faulty Python3 ELSE tag
+	remove
+
+2016-12-09  rocky <rb@dustyfeet.com>
+
+	* pytest/test_grammar.py: Grammar check: ELSE on RHS is ok.
+
+2016-12-09  rocky <rb@dustyfeet.com>
+
+	* uncompyle6/scanners/tok.py, uncompyle6/verify.py: Back of some of
+	the verification changes
+
+2016-12-09  rocky <rb@dustyfeet.com>
+
+	* : commit a5d2237435ee51e681c73db9e7ea379d56456205 Author: rocky
+	<rb@dustyfeet.com> Date:   Fri Dec 9 21:10:10 2016 -0500
+
 2016-12-04  rocky <rb@dustyfeet.com>
 
-	* uncompyle6/version.py: Get ready for release 2.9.7
+	* ChangeLog, NEWS, uncompyle6/main.py, uncompyle6/parser.py,
+	uncompyle6/parsers/parse26.py, uncompyle6/parsers/parse3.py,
+	uncompyle6/parsers/parse34.py, uncompyle6/parsers/parse35.py,
+	uncompyle6/scanner.py, uncompyle6/scanners/scanner2.py,
+	uncompyle6/scanners/scanner23.py, uncompyle6/scanners/scanner24.py,
+	uncompyle6/scanners/scanner3.py, uncompyle6/scanners/tok.py,
+	uncompyle6/semantics/make_function.py,
+	uncompyle6/semantics/pysource.py, uncompyle6/verify.py,
+	uncompyle6/version.py: Get ready for release 2.9.7 Some of the many lint things. Linting is kind of stupid though.
 
 2016-11-28  rocky <rb@dustyfeet.com>
 

HISTORY.md

@@ -30,7 +30,7 @@ another clever idea: using table-driven semantics routines, using
 format specifiers.
 
 The last mention of a release of SPARK from John is around 2002. As
-released, although the Early Algorithm parser was in good shape, this
+released, although the Earley Algorithm parser was in good shape, this
 code was woefully lacking as serious Python deparser.
 
 In the fall of 2000, Hartmut Goebel
@@ -135,9 +135,9 @@ Hartmut a decade an a half ago:
     NB. This is not a masterpiece of software, but became more like a hack.
     Probably a complete rewrite would be sensefull. hG/2000-12-27
 
-This project deparses using an Early-algorithm parse with lots of
+This project deparses using an Earley-algorithm parse with lots of
 massaging of tokens and the grammar in the scanner
-phase. Early-algorithm parsers are context free and tend to be linear
+phase. Earley-algorithm parsers are context free and tend to be linear
 if the grammar is LR or left recursive.
 
 Another approach that doesn't use grammars is to do something like

NEWS

@@ -1,3 +1,32 @@
+uncompyle6 2.9.10 2016-02-25
+
+- Python grammar rule fixes
+- Add ability to get grammar coverage on runs
+- Handle Python 3.6 opcode BUILD_CONST_KEYMAP
+
+uncompyle6 2.9.9 2016-12-16
+
+- Remaining Python 3.5 ops handled
+  (this also means more Python 3.6 ops are handled)
+- Python 3.5 and 3.6 async and await handled
+- Python 3.0 decompilation improved
+- Python 3 annotations fixed
+- Better control-flow detection
+- Code cleanups and misc bug fixes
+
+uncompyle6 2.9.8 2016-12-16
+
+- Better control-flow detection
+- pseudo instruction THEN in 2.x
+  to disambiguate if from and
+- fix bug in --verify option
+- DRY (a little) control-flow detection
+- fix syntax in tuples with one element
+- if AST rule inheritence in Python 2.5
+- NAME_MODULE removal for Python <= 2.4
+- verifycall fixes for Python <= 2.4
+- more Python lint
+
 uncompyle6 2.9.7 2016-12-16
 
 - Start to handle 3.5/3.6 build_map_unpack_with_call

README.rst

@@ -43,10 +43,10 @@ information.
 Requirements
 ------------
 
-This project requires Python 2.6 or later, PyPy 3-2.4, or PyPy-5.0.1. 
-Python versions 2.3-2.7 are supported in the python-2.4 branch.
+This project requires Python 2.6 or later, PyPy 3-2.4, or PyPy-5.0.1.
+Python versions 2.4-2.7 are supported in the python-2.4 branch.
 The bytecode files it can read has been tested on Python bytecodes from
-versions 2.1-2.7, and 3.2-3.6 and the above-mentioned PyPy versions.
+versions 1.5, 2.1-2.7, and 3.2-3.6 and the above-mentioned PyPy versions.
 
 Installation
 ------------
@@ -93,6 +93,16 @@ For usage help:
 
    $ uncompyle6 -h
 
+If you want strong verification of the correctness of the
+decompilation process, add the `--verify` option. But there are
+situations where this will indicate a failure, although the generated
+program is semantically equivalent. Using option `--weak-verify` will
+tell you if there is something definitely wrong. Generally, large
+swaths of code are decompiled correctly, if not the entire program.
+
+You can also cross compare the results with pycdc_ . Since they work
+differently, bugs here often aren't in that, and vice versa.
+
 
 Known Bugs/Restrictions
 -----------------------
@@ -128,10 +138,13 @@ on the lower end Python versions which is more difficult for us to
 handle since we don't have a Python interpreter for versions 1.5, 1.6,
 and 2.0.
 
-Python 3.0 support is weak; Python 3.5 largely works, but still has
-some bugs in it.  Python 3.6 changes things drastically by using word
-codes rather than byte codes. That has been addressed, but then it also
-changes function call opcodes and its semantics.
+In the Python 3 series, Python support is is strongest around 3.4 or
+3.3 and drops off as you move further away from those versions. Python
+3.5 largely works, but still has some bugs in it and is missing some
+opcodes.  Python 3.6 changes things drastically by using word codes
+rather than byte codes. That has been addressed, but then it also
+changes function call opcodes and its semantics and has more problems
+with control flow than 3.5 has.
 
 Currently not all Python magic numbers are supported. Specifically in
 some versions of Python, notably Python 3.6, the magic number has

__pkginfo__.py

@@ -9,7 +9,7 @@
 
 # Things that change more often go here.
 copyright   = """
-Copyright (C) 2015, 2016 Rocky Bernstein <rb@dustyfeet.com>.
+Copyright (C) 2015-2017 Rocky Bernstein <rb@dustyfeet.com>.
 """
 
 classifiers =  ['Development Status :: 5 - Production/Stable',
@@ -39,7 +39,7 @@ entry_points={
         'pydisassemble=uncompyle6.bin.pydisassemble:main',
     ]}
 ftp_url            = None
-install_requires   = ['spark-parser >= 1.5.1, < 1.6.0',
+install_requires   = ['spark-parser >= 1.6.0, < 1.7.0',
                       'xdis >= 3.2.4, < 3.3.0']
 license            = 'MIT'
 mailing_list       = 'python-debugger@googlegroups.com'

pytest/test_build_const_key_map.py

@@ -0,0 +1,21 @@
+import pytest
+# uncompyle6
+from uncompyle6 import PYTHON_VERSION
+from validate import validate_uncompyle
+
+
+@pytest.mark.skipif(PYTHON_VERSION < 3.6, reason='need at least python 3.6')
+@pytest.mark.parametrize('text', (
+    "{0.: 'a', -1: 'b'}",   # BUILD_MAP
+    "{'a':'b'}",            # BUILD_MAP
+    "{0: 1}",               # BUILD_MAP
+    "{b'0':1, b'2':3}",     # BUILD_CONST_KEY_MAP
+    "{0: 1, 2: 3}",         # BUILD_CONST_KEY_MAP
+    "{'a':'b','c':'d'}",    # BUILD_CONST_KEY_MAP
+    "{0: 1, 2: 3}",         # BUILD_CONST_KEY_MAP
+    "{'a': 1, 'b': 2}",     # BUILD_CONST_KEY_MAP
+    "{'a':'b','c':'d'}",    # BUILD_CONST_KEY_MAP
+    "{0.0:'b',0.1:'d'}",    # BUILD_CONST_KEY_MAP
+))
+def test_build_const_key_map(text):
+    validate_uncompyle(text)

pytest/validate.py

@@ -0,0 +1,143 @@
+# future
+from __future__ import print_function
+# std
+import os
+import dis
+import difflib
+import subprocess
+import tempfile
+# compatability
+import six
+# uncompyle6 / xdis
+from uncompyle6 import PYTHON_VERSION, deparse_code
+
+
+def _dis_to_text(co):
+    return dis.Bytecode(co).dis()
+
+
+def print_diff(original, uncompyled):
+    """
+    Try and display a pretty html line difference between the original and
+    uncompyled code and bytecode if elinks and BeautifulSoup are installed
+    otherwise just show the diff.
+
+    :param original: Text describing the original code object.
+    :param uncompyled: Text describing the uncompyled code object.
+    """
+    original_lines = original.split('\n')
+    uncompyled_lines = uncompyled.split('\n')
+    args = original_lines, uncompyled_lines, 'original', 'uncompyled'
+    try:
+        from bs4 import BeautifulSoup
+        diff = difflib.HtmlDiff().make_file(*args)
+        diff = BeautifulSoup(diff, "html.parser")
+        diff.select_one('table[summary="Legends"]').extract()
+    except ImportError:
+        print('\nTo display diff highlighting run:\n    pip install BeautifulSoup4')
+        diff = difflib.HtmlDiff().make_table(*args)
+
+    with tempfile.NamedTemporaryFile(delete=False) as f:
+        f.write(str(diff).encode('utf-8'))
+
+    try:
+        print()
+        html = subprocess.check_output([
+            'elinks',
+            '-dump',
+            '-no-references',
+            '-dump-color-mode',
+            '1',
+            f.name,
+        ]).decode('utf-8')
+        print(html)
+    except:
+        print('\nFor side by side diff install elinks')
+        diff = difflib.Differ().compare(original_lines, uncompyled_lines)
+        print('\n'.join(diff))
+    finally:
+        os.unlink(f.name)
+
+
+def are_instructions_equal(i1, i2):
+    """
+    Determine if two instructions are approximately equal,
+    ignoring certain fields which we allow to differ, namely:
+
+    * code objects are ignore (should probaby be checked) due to address
+    * line numbers
+
+    :param i1: left instruction to compare
+    :param i2: right instruction to compare
+
+    :return: True if the two instructions are approximately equal, otherwise False.
+    """
+    result = (1==1
+        and i1.opname == i2.opname
+        and i1.opcode == i2.opcode
+        and i1.arg == i2.arg
+        # ignore differences due to code objects
+        # TODO : Better way of ignoring address
+        and (i1.argval == i2.argval or '<code object' in str(i1.argval))
+        # TODO : Should probably recurse to check code objects
+        and (i1.argrepr == i2.argrepr or '<code object' in i1.argrepr)
+        and i1.offset == i2.offset
+        # ignore differences in line numbers
+        #and i1.starts_line
+        and i1.is_jump_target == i2.is_jump_target
+    )
+    return result
+
+
+def are_code_objects_equal(co1, co2):
+    """
+    Determine if two code objects are approximately equal,
+    see are_instructions_equal for more information.
+
+    :param i1: left code object to compare
+    :param i2: right code object to compare
+
+    :return: True if the two code objects are approximately equal, otherwise False.
+    """
+    # TODO : Use xdis for python2 compatability
+    instructions1 = dis.Bytecode(co1)
+    instructions2 = dis.Bytecode(co2)
+    for opcode1, opcode2 in zip(instructions1, instructions2):
+        if not are_instructions_equal(opcode1, opcode2):
+            return False
+    return True
+
+
+def validate_uncompyle(text, mode='exec'):
+    """
+    Validate decompilation of the given source code.
+
+    :param text: Source to validate decompilation of.
+    """
+    original_code = compile(text, '<string>', mode)
+    original_dis = _dis_to_text(original_code)
+    original_text = text
+
+    deparsed = deparse_code(PYTHON_VERSION, original_code,
+                            compile_mode=mode, out=six.StringIO())
+    uncompyled_text = deparsed.text
+    uncompyled_code = compile(uncompyled_text, '<string>', 'exec')
+
+    if not are_code_objects_equal(uncompyled_code, original_code):
+
+        uncompyled_dis = _dis_to_text(uncompyled_text)
+
+        def output(text, dis):
+            width = 60
+            return '\n\n'.join([
+                ' SOURCE CODE '.center(width, '#'),
+                text.strip(),
+                ' BYTECODE '.center(width, '#'),
+                dis
+            ])
+
+        original = output(original_text, original_dis)
+        uncompyled = output(uncompyled_text, uncompyled_dis)
+        print_diff(original, uncompyled)
+
+        assert 'original' == 'uncompyled'

requirements-dev.txt

@@ -1,3 +1,4 @@
 pytest
 flake8
-hypothesis
+hypothesis
+six

test/Makefile

@@ -98,9 +98,24 @@ check-bytecode-2.4:
 check-bytecode-2.5:
 	$(PYTHON) test_pythonlib.py --bytecode-2.5
 
+#: Get grammar coverage for Python 2.5
+grammar-coverage-2.5:
+	SPARK_PARSER_COVERAGE=/tmp/spark-grammar-25.cover $(PYTHON) test_pythonlib.py --bytecode-2.5
+	SPARK_PARSER_COVERAGE=/tmp/spark-grammar-25.cover $(PYTHON) test_pyenvlib.py --2.5.6
+
+#: Get grammar coverage for Python 2.6
+grammar-coverage-2.6:
+	SPARK_PARSER_COVERAGE=/tmp/spark-grammar-26.cover $(PYTHON) test_pythonlib.py --bytecode-2.6
+	SPARK_PARSER_COVERAGE=/tmp/spark-grammar-26.cover $(PYTHON) test_pyenvlib.py --2.6.9
+
+#: Get grammar coverage for Python 2.7
+grammar-coverage-2.7:
+	SPARK_PARSER_COVERAGE=/tmp/spark-grammar-27.cover $(PYTHON) test_pythonlib.py --bytecode-2.7
+	SPARK_PARSER_COVERAGE=/tmp/spark-grammar-27.cover $(PYTHON) test_pyenvlib.py --2.7.13
+
 #: Check deparsing Python 2.6
 check-bytecode-2.6:
-	$(PYTHON) test_pythonlib.py --bytecode-2.6
+	$(PYTHON) test_pythonlib.py --bytecode-2.6 --weak-verify
 
 #: Check deparsing Python 2.7
 check-bytecode-2.7:

test/simple_source/bug22/01_ops.py

@@ -0,0 +1,18 @@
+# Statements to beef up grammar coverage rules
+# Force "inplace" ops
+y = +10  # UNARY_POSITIVE
+y /= 1   # INPLACE_DIVIDE
+y %= 4   # INPLACE_MODULO
+y **= 1  # INPLACE POWER
+y >>= 2  # INPLACE_RSHIFT
+y <<= 2  # INPLACE_LSHIFT
+y //= 1  # INPLACE_TRUE_DIVIDE
+y &= 1   # INPLACE_AND
+y ^= 1   # INPLACE_XOR
+
+`y`      # UNARY_CONVERT  - No in Python 3.x
+
+# Beef up augassign and STORE_SLICE+3
+x = [1,2,3,4,5]
+x[0:1] = 1
+x[0:3] += 1, 2, 3

test/simple_source/bug25/02_try_else.py

@@ -1,7 +1,6 @@
 # Python 2.5 bug
 # Was turning into tryelse when there in fact is no else.
 def options(self, section):
-    """Return a list of option names for the given section name."""
     try:
         opts = self._sections[section].copy()
     except KeyError:
@@ -10,3 +9,19 @@ def options(self, section):
     if '__name__' in opts:
         del opts['__name__']
     return opts.keys()
+
+# From python2.5/distutils/command/register.py
+def post_to_server(self, urllib2):
+    try:
+        result = 5
+    except urllib2.HTTPError, e:
+        result = e.code, e.msg
+    except urllib2.URLError, e:
+        result = 500
+    else:
+        if self.show_response:
+            result = 10
+        result = 200
+    if self.show_response:
+        result = 8
+    return result

test/simple_source/bug25/03_if_for.py

@@ -0,0 +1,7 @@
+# From Python 2.6. distutils/sysconfig.py
+def get_config_vars(_config_vars, args):
+    if _config_vars:
+        if args == 1:
+            if args < 8:
+                for key in ('LDFLAGS', 'BASECFLAGS'):
+                    _config_vars[key] = 4

test/simple_source/bug31/04_def_annotate.py

@@ -8,3 +8,12 @@ def open(file, mode = "r", buffering = None,
          encoding = None, errors = None,
          newline = None, closefd = True) -> "IOBase":
     return text
+
+def foo(x: 'an argument that defaults to 5' = 5):
+    print(x)
+
+def div(a: dict(type=float, help='the dividend'),
+        b: dict(type=float, help='the divisor (must be different than 0)')
+    ) -> dict(type=float, help='the result of dividing a by b'):
+    """Divide a by b"""
+    return a / b

test/simple_source/bug33/02_while1.py

@@ -0,0 +1,14 @@
+# From Python 3.4 mailcap
+def readmailcapfile(caps):
+    while 1:
+        line = 'abc'
+        if line[0] == '#' or line == '':
+            continue
+        key, fields = (1,2)
+        if not (key and fields):
+            continue
+        if key in caps:
+            caps[key].append(fields)
+        else:
+            caps[key] = [fields]
+    return caps

test/simple_source/bug35/03_async_await.py

@@ -0,0 +1,26 @@
+# Python 3.5+ async and await
+async def await_test(asyncio):
+    reader, writer = await asyncio.open_connection(80)
+    await bar()
+
+async def afor_test():
+
+    async for i in [1,2,3]:
+        x = i
+
+
+async def afor_else_test():
+
+    async for i in [1,2,3]:
+        x = i
+    else:
+        z = 4
+
+
+async def awith_test():
+    async with i:
+        print(i)
+
+async def awith_as_test():
+    async with 1 as i:
+        print(i)

test/simple_source/bug35/03_double_star_unpack.py

@@ -0,0 +1,9 @@
+# Bug in Python 3.5 is getting the two star'd arguments right.
+def sum(a,b,c,d):
+    return a + b + c + d
+
+args=(1,2)
+sum(*args, *args)
+
+# FIXME: this is handled incorrectly
+# (*c,) = (3,4)

test/simple_source/bug35/03_while-if-break.py

@@ -0,0 +1,7 @@
+# Python 3.5 and 3.6 break inside a
+# while True and if / break
+def display_date(loop):
+    while True:
+        if loop.time():
+            break
+        x = 5

test/simple_source/exception/02_try_finally.py

@@ -0,0 +1,11 @@
+# From 2.6.9 cmd.py
+try:
+    if __file__:
+        x = 2
+    x = 3
+finally:
+    if x and __file__:
+        try:
+            x = 1
+        except:
+            pass

test/test_pyenvlib.py

@@ -30,10 +30,10 @@ from fnmatch import fnmatch
 TEST_VERSIONS=('2.3.7', '2.4.6', '2.5.6', '2.6.9',
                'pypy-2.4.0', 'pypy-2.6.1',
                'pypy-5.0.1', 'pypy-5.3.1',
-               '2.7.10', '2.7.11', '2.7.12',
+               '2.7.10', '2.7.11', '2.7.12', '2.7.13',
                '3.0.1', '3.1.5', '3.2.6',
                '3.3.5', '3.3.6',
-               '3.4.2', '3.5.1')
+               '3.4.2', '3.5.1', '3.6.0')
 
 target_base = '/tmp/py-dis/'
 lib_prefix = os.path.join(os.environ['HOME'], '.pyenv/versions')
@@ -106,28 +106,40 @@ def do_tests(src_dir, patterns, target_dir, start_with=None, do_verify=False):
 if __name__ == '__main__':
     import getopt, sys
 
-    do_verify = False
+    do_coverage = do_verify = False
     test_dirs = []
     start_with = None
 
     test_options_keys = list(test_options.keys())
     test_options_keys.sort()
     opts, args = getopt.getopt(sys.argv[1:], '',
-                               ['start-with=', 'verify', 'weak-verify', 'all', ] \
+                               ['start-with=', 'verify', 'weak-verify',
+                                'coverage', 'all', ] \
                                + test_options_keys )
+    vers = ''
     for opt, val in opts:
         if opt == '--verify':
             do_verify = True
         if opt == '--weak-verify':
             do_verify = 'weak'
+        if opt == '--coverage':
+            do_coverage = True
         elif opt == '--start-with':
             start_with = val
         elif opt[2:] in test_options_keys:
-            test_dirs.append(test_options[opt[2:]])
+            triple = test_options[opt[2:]]
+            vers = triple[-1]
+            test_dirs.append(triple)
         elif opt == '--all':
+            vers = 'all'
             for val in test_options_keys:
                 test_dirs.append(test_options[val])
 
+    if do_coverage:
+        os.environ['SPARK_PARSER_COVERAGE'] = (
+            '/tmp/spark-grammar-%s.cover' % vers
+            )
+
     for src_dir, pattern, target_dir in test_dirs:
         if os.path.exists(src_dir):
             target_dir = os.path.join(target_base, target_dir)

test/test_pythonlib.py

@@ -190,6 +190,7 @@ if __name__ == '__main__':
     test_options_keys.sort()
     opts, args = getopt.getopt(sys.argv[1:], '',
                                ['start-with=', 'verify', 'weak-verify', 'all', 'compile',
+                                'coverage',
                                 'no-rm'] \
                                + test_options_keys )
     if not opts: help()
@@ -198,7 +199,8 @@ if __name__ == '__main__':
         'do_compile': False,
         'do_verify': False,
         'start_with': None,
-        'rmtree' : True
+        'rmtree' : True,
+        'coverage' : False
         }
 
     for opt, val in opts:
@@ -217,11 +219,18 @@ if __name__ == '__main__':
         elif opt == '--all':
             for val in test_options_keys:
                 test_dirs.append(test_options[val])
+        elif opt == '--coverage':
+            test_opts['coverage'] = True
         else:
             help()
             pass
         pass
 
+    if test_opts['coverage']:
+        os.environ['SPARK_PARSER_COVERAGE'] = (
+            '/tmp/spark-grammar-python-lib%s.cover' % test_dirs[0][-1]
+            )
+
     last_compile_version = None
     for src_dir, pattern, target_dir, compiled_version in test_dirs:
         if os.path.isdir(src_dir):

uncompyle6/__init__.py

@@ -41,13 +41,18 @@ PYTHON_VERSION_STR  = "%s.%s" % (sys.version_info[0], sys.version_info[1])
 
 IS_PYPY = '__pypy__' in sys.builtin_module_names
 
-sys.setrecursionlimit(5000)
+if hasattr(sys, 'setrecursionlimit'):
+    # pyston doesn't have setrecursionlimit
+    sys.setrecursionlimit(5000)
 
 import uncompyle6.semantics.pysource
 import uncompyle6.semantics.fragments
 
 # Export some functions
-from uncompyle6.main import uncompyle_file
+from uncompyle6.main import decompile_file
+
+# For compaitility
+uncompyle_file = decompile_file
 
 # Conventience functions so you can say:
 # from uncompyle6 import deparse_code

uncompyle6/main.py

@@ -11,7 +11,7 @@ from uncompyle6.linenumbers import line_number_mapping
 
 from xdis.load import load_module
 
-def uncompyle(
+def decompile(
         bytecode_version, co, out=None, showasm=None, showast=False,
         timestamp=None, showgrammar=False, code_objects={},
         source_size=None, is_pypy=False, magic_int=None):
@@ -48,8 +48,10 @@ def uncompyle(
         # deparsing failed
         raise pysource.SourceWalkerError(str(e))
 
+# For compatiblity
+uncompyle = decompile
 
-def uncompyle_file(filename, outstream=None, showasm=None, showast=False,
+def decompile_file(filename, outstream=None, showasm=None, showast=False,
                    showgrammar=False):
     """
     decompile Python byte-code file (.pyc)
@@ -62,16 +64,20 @@ def uncompyle_file(filename, outstream=None, showasm=None, showast=False,
 
     if type(co) == list:
         for con in co:
-            uncompyle(version, con, outstream, showasm, showast,
+            decompile(version, con, outstream, showasm, showast,
                       timestamp, showgrammar, code_objects=code_objects,
                       is_pypy=is_pypy, magic_int=magic_int)
     else:
-        uncompyle(version, co, outstream, showasm, showast,
+        decompile(version, co, outstream, showasm, showast,
                   timestamp, showgrammar,
                   code_objects=code_objects, source_size=source_size,
                   is_pypy=is_pypy, magic_int=magic_int)
     co = None
 
+# For compatiblity
+uncompyle_file = decompile_file
+
+
 # FIXME: combine into an options parameter
 def main(in_base, out_base, files, codes, outfile=None,
          showasm=None, showast=False, do_verify=False,
@@ -101,12 +107,6 @@ def main(in_base, out_base, files, codes, outfile=None,
 
     tot_files = okay_files = failed_files = verify_failed_files = 0
 
-    # for code in codes:
-    #    version = sys.version[:3] # "2.5"
-    #    with open(code, "r") as f:
-    #        co = compile(f.read(), "", "exec")
-    #    uncompyle(sys.version[:3], co, sys.stdout, showasm=showasm, showast=showast)
-
     for filename in files:
         infile = os.path.join(in_base, filename)
         if not os.path.exists(infile):
@@ -126,8 +126,9 @@ def main(in_base, out_base, files, codes, outfile=None,
                     prefix = prefix[:-len('.py')]
                 junk, outfile = tempfile.mkstemp(suffix=".py",
                                              prefix=prefix)
-                # Unbuffer output
-                sys.stdout = os.fdopen(sys.stdout.fileno(), 'w', 0)
+                # Unbuffer output if possible
+                buffering = -1 if sys.stdout.isatty() else 0
+                sys.stdout = os.fdopen(sys.stdout.fileno(), 'w', buffering)
                 tee = subprocess.Popen(["tee", outfile], stdin=subprocess.PIPE)
                 os.dup2(tee.stdin.fileno(), sys.stdout.fileno())
                 os.dup2(tee.stdin.fileno(), sys.stderr.fileno())
@@ -141,7 +142,7 @@ def main(in_base, out_base, files, codes, outfile=None,
 
         # Try to uncompile the input file
         try:
-            uncompyle_file(infile, outstream, showasm, showast, showgrammar)
+            decompile_file(infile, outstream, showasm, showast, showgrammar)
             tot_files += 1
         except (ValueError, SyntaxError, ParserError, pysource.SourceWalkerError) as e:
             sys.stdout.write("\n")
@@ -230,11 +231,11 @@ def status_msg(do_verify, tot_files, okay_files, failed_files,
                verify_failed_files):
     if tot_files == 1:
         if failed_files:
-            return "decompile failed"
+            return "\n# decompile failed"
         elif verify_failed_files:
-            return "decompile verify failed"
+            return "\n# decompile verify failed"
         else:
-            return "Successfully decompiled file"
+            return "\n# Successfully decompiled file"
             pass
         pass
     mess = "decompiled %i files: %i okay, %i failed" % (tot_files, okay_files, failed_files)

uncompyle6/parser.py

@@ -1,4 +1,4 @@
-#  Copyright (c) 2015-2016 Rocky Bernstein
+#  Copyright (c) 2015-2017 Rocky Bernstein
 #  Copyright (c) 2005 by Dan Pascu <dan@windowmaker.org>
 #  Copyright (c) 2000-2002 by hartmut Goebel <h.goebel@crazy-compilers.com>
 #  Copyright (c) 1999 John Aycock
@@ -28,6 +28,16 @@ nop_func = lambda self, args: None
 
 class PythonParser(GenericASTBuilder):
 
+    def __init__(self, AST, start, debug):
+        super(PythonParser, self).__init__(AST, start, debug)
+        self.collect = frozenset(
+            ['stmts', 'except_stmts', '_stmts',
+             'exprlist', 'kvlist', 'kwargs', 'come_froms',
+              # Python < 3
+             'print_items',
+             # PyPy:
+             'kvlist_n'])
+
     def add_unique_rule(self, rule, opname, count, customize):
         """Add rule to grammar, but only if it hasn't been added previously
            opname and count are used in the customize() semantic the actions
@@ -115,11 +125,7 @@ class PythonParser(GenericASTBuilder):
         return token.type
 
     def nonterminal(self, nt, args):
-        collect = ('stmts', 'exprlist', 'kvlist', '_stmts', 'print_items', 'kwargs',
-                   # PYPY:
-                   'kvlist_n')
-
-        if nt in collect and len(args) > 1:
+        if nt in self.collect and len(args) > 1:
             #
             #  Collect iterated thingies together. That is rather than
             #  stmts -> stmts stmt -> stmts stmt -> ...

uncompyle6/parsers/parse2.py

@@ -126,6 +126,7 @@ class Python2Parser(PythonParser):
         assert_expr_and ::= assert_expr jmp_false expr
 
         ifstmt ::= testexpr _ifstmts_jump
+        ifstmt ::= testexpr return_if_stmts COME_FROM
 
         testexpr ::= testfalse
         testexpr ::= testtrue
@@ -144,6 +145,8 @@ class Python2Parser(PythonParser):
 
         ifelsestmtr ::= testexpr return_if_stmts return_stmts
 
+        ifelsestmtr ::= testexpr return_if_stmts COME_FROM return_stmts
+
         ifelsestmtl ::= testexpr c_stmts_opt JUMP_BACK else_suitel
 
 

uncompyle6/parsers/parse25.py

@@ -1,4 +1,4 @@
-#  Copyright (c) 2016 Rocky Bernstein
+#  Copyright (c) 2016-2017 Rocky Bernstein
 """
 spark grammar differences over Python2.6 for Python 2.5.
 """
@@ -20,13 +20,18 @@ class Python25Parser(Python26Parser):
         return_if_stmt ::= ret_expr  RETURN_END_IF JUMP_BACK
 
         # Python 2.6 uses ROT_TWO instead of the STORE_xxx
+        # withas is allowed as a "from future" in 2.5
         setupwithas ::= DUP_TOP LOAD_ATTR store LOAD_ATTR CALL_FUNCTION_0
                         setup_finally
 
         store ::= STORE_FAST
         store ::= STORE_NAME
 
-        # Python 2.6 omits ths LOAD_FAST DELETE_FAST below
+        tryelsestmt    ::= SETUP_EXCEPT suite_stmts_opt POP_BLOCK
+                           try_middle else_suite COME_FROM
+
+        # Python 2.6 omits the LOAD_FAST DELETE_FAST below
+        # withas is allowed as a "from future" in 2.5
         withasstmt ::= expr setupwithas designator suite_stmts_opt
                        POP_BLOCK LOAD_CONST COME_FROM
                        with_cleanup
@@ -46,18 +51,6 @@ class Python25Parser(Python26Parser):
                                                 tokens, first, last)
         if invalid:
             return invalid
-        lhs = rule[0]
-        if lhs in ('tryelsestmt', ):
-            # The end of the else part of try/else come_from has to come
-            # from an END_FINALLY statement
-            if tokens[last-1].type.startswith('COME_FROM'):
-                end_finally_offset = int(tokens[last-1].pattr)
-                current  = first
-                while current < last:
-                    offset = tokens[current].offset
-                    if offset == end_finally_offset:
-                        return tokens[current].type != 'END_FINALLY'
-                    current += 1
         return False
 
 

uncompyle6/parsers/parse26.py

@@ -1,4 +1,4 @@
-#  Copyright (c) 2016 Rocky Bernstein
+#  Copyright (c) 2017 Rocky Bernstein
 """
 spark grammar differences over Python2 for Python 2.6.
 """
@@ -22,25 +22,22 @@ class Python26Parser(Python2Parser):
                          JUMP_IF_FALSE POP_TOP POP_TOP designator POP_TOP
 
         try_middle   ::= JUMP_FORWARD COME_FROM except_stmts
-                         come_from_pop END_FINALLY COME_FROM
+                         come_from_pop END_FINALLY come_froms
+
+        try_middle   ::= JUMP_FORWARD COME_FROM except_stmts END_FINALLY
+                         come_froms
 
         try_middle   ::= jmp_abs COME_FROM except_stmts
                          POP_TOP END_FINALLY
 
         try_middle   ::= jmp_abs COME_FROM except_stmts
-                         POP_TOP END_FINALLY
-
-        trystmt      ::= SETUP_EXCEPT suite_stmts_opt POP_TOP
-                         try_middle
+                         END_FINALLY JUMP_FORWARD
 
         # Sometimes we don't put in COME_FROM to the next statement
         # like we do in 2.7. Perhaps we should?
         trystmt      ::= SETUP_EXCEPT suite_stmts_opt POP_BLOCK
                          try_middle
 
-        trystmt      ::= SETUP_EXCEPT suite_stmts_opt POP_BLOCK
-                         try_middle come_froms
-
         tryelsestmt    ::= SETUP_EXCEPT suite_stmts_opt POP_BLOCK
                            try_middle else_suite COME_FROM
 
@@ -113,16 +110,10 @@ class Python26Parser(Python2Parser):
 
         break_stmt ::= BREAK_LOOP JUMP_BACK
 
-        # Semantic actions want the else to be at position 3
-        ifelsestmt  ::= testexpr c_stmts_opt jf_cf_pop else_suite come_froms
-        ifelsestmt  ::= testexpr c_stmts_opt filler else_suitel come_froms POP_TOP
-
         # Semantic actions want else_suitel to be at index 3
         ifelsestmtl ::= testexpr c_stmts_opt jb_cf_pop else_suitel
         ifelsestmtc ::= testexpr c_stmts_opt ja_cf_pop else_suitec
 
-        iflaststmt  ::= testexpr c_stmts_opt JUMP_ABSOLUTE come_froms POP_TOP
-
         # Semantic actions want suite_stmts_opt to be at index 3
         withstmt ::= expr setupwith SETUP_FINALLY suite_stmts_opt
                      POP_BLOCK LOAD_CONST COME_FROM WITH_CLEANUP END_FINALLY
@@ -157,8 +148,33 @@ class Python26Parser(Python2Parser):
         iflaststmtl ::= testexpr c_stmts_opt JUMP_BACK come_from_pop
         iflaststmt  ::= testexpr c_stmts_opt JUMP_ABSOLUTE come_from_pop
 
+        lastc_stmt ::= iflaststmt COME_FROM
+
         while1stmt ::= SETUP_LOOP l_stmts_opt JUMP_BACK COME_FROM
 
+        ifstmt         ::= testexpr_then _ifstmts_jump
+
+        # Semantic actions want the else to be at position 3
+        ifelsestmt     ::= testexpr      c_stmts_opt jf_cf_pop else_suite come_froms
+        ifelsestmt     ::= testexpr_then c_stmts_opt jf_cf_pop else_suite come_froms
+        ifelsestmt     ::= testexpr      c_stmts_opt filler else_suitel come_froms POP_TOP
+        ifelsestmt     ::= testexpr_then c_stmts_opt filler else_suitel come_froms POP_TOP
+
+        # Semantic actions want else_suitel to be at index 3
+        ifelsestmtl    ::= testexpr_then c_stmts_opt jb_cf_pop else_suitel
+        ifelsestmtc    ::= testexpr_then c_stmts_opt ja_cf_pop else_suitec
+
+        iflaststmt     ::= testexpr_then c_stmts_opt JUMP_ABSOLUTE come_froms POP_TOP
+        iflaststmt     ::= testexpr      c_stmts_opt JUMP_ABSOLUTE come_froms POP_TOP
+
+        testexpr_then  ::= testtrue_then
+        testexpr_then  ::= testfalse_then
+        testtrue_then  ::= expr jmp_true_then
+        testfalse_then ::= expr jmp_false_then
+
+        jmp_false_then ::= JUMP_IF_FALSE THEN POP_TOP
+        jmp_true_then  ::= JUMP_IF_TRUE THEN POP_TOP
+
         # Common with 2.7
         while1stmt ::= SETUP_LOOP return_stmts bp_come_from
         while1stmt ::= SETUP_LOOP return_stmts COME_FROM
@@ -196,16 +212,16 @@ class Python26Parser(Python2Parser):
         genexpr_func ::= setup_loop_lf FOR_ITER designator comp_iter JUMP_BACK come_from_pop
                          jb_bp_come_from
         genexpr ::= LOAD_GENEXPR MAKE_FUNCTION_0 expr GET_ITER CALL_FUNCTION_1 COME_FROM
-
+        list_if ::= list_if ::= expr jmp_false_then list_iter
         '''
 
     def p_ret26(self, args):
         '''
-        ret_and  ::= expr jmp_false ret_expr_or_cond COME_FROM
-        ret_or   ::= expr jmp_true ret_expr_or_cond COME_FROM
-        ret_cond ::= expr jmp_false expr RETURN_END_IF POP_TOP ret_expr_or_cond
-        ret_cond ::= expr jmp_false expr ret_expr_or_cond
-        ret_cond_not ::= expr jmp_true expr RETURN_END_IF POP_TOP ret_expr_or_cond
+        ret_and      ::= expr jmp_false ret_expr_or_cond COME_FROM
+        ret_or       ::= expr jmp_true ret_expr_or_cond COME_FROM
+        ret_cond     ::= expr jmp_false_then expr RETURN_END_IF POP_TOP ret_expr_or_cond
+        ret_cond     ::= expr jmp_false_then expr ret_expr_or_cond
+        ret_cond_not ::= expr jmp_true_then  expr RETURN_END_IF POP_TOP ret_expr_or_cond
 
         return_if_stmt ::= ret_expr RETURN_END_IF POP_TOP
         return_stmt ::= ret_expr RETURN_VALUE POP_TOP
@@ -215,17 +231,37 @@ class Python26Parser(Python2Parser):
         '''
 
     def p_except26(self, args):
-        '''
+        """
         except_suite ::= c_stmts_opt jmp_abs POP_TOP
-        '''
+        """
 
     def p_misc26(self, args):
-        '''
+        """
         conditional  ::= expr jmp_false expr jf_cf_pop expr come_from_opt
         and  ::= expr JUMP_IF_FALSE POP_TOP expr JUMP_IF_FALSE POP_TOP
         cmp_list ::= expr cmp_list1 ROT_TWO COME_FROM POP_TOP _come_from
-        '''
 
+        conditional_lambda ::= expr jmp_false_then return_if_stmt return_stmt LAMBDA_MARKER
+        """
+
+    def add_custom_rules(self, tokens, customize):
+        super(Python26Parser, self).add_custom_rules(tokens, customize)
+        self.check_reduce['and'] = 'AST'
+
+    def reduce_is_invalid(self, rule, ast, tokens, first, last):
+        invalid = super(Python26Parser,
+                        self).reduce_is_invalid(rule, ast,
+                                                tokens, first, last)
+        if invalid:
+            return invalid
+        if rule == ('and', ('expr', 'jmp_false', 'expr', '\\e_come_from_opt')):
+            # Test that jmp_false jumps to the end of "and"
+            # or that it jumps to the same place as the end of "and"
+            jmp_false = ast[1][0]
+            jmp_target = jmp_false.offset + jmp_false.attr + 3
+            return not (jmp_target == tokens[last].offset or
+                        tokens[last].pattr == jmp_false.pattr)
+        return False
 class Python26ParserSingle(Python2Parser, PythonParserSingle):
     pass
 

uncompyle6/parsers/parse27.py

@@ -61,7 +61,7 @@ class Python27Parser(Python2Parser):
 
         ret_and  ::= expr JUMP_IF_FALSE_OR_POP ret_expr_or_cond COME_FROM
         ret_or   ::= expr JUMP_IF_TRUE_OR_POP ret_expr_or_cond COME_FROM
-        ret_cond ::= expr POP_JUMP_IF_FALSE expr RETURN_END_IF ret_expr_or_cond
+        ret_cond ::= expr POP_JUMP_IF_FALSE expr RETURN_END_IF COME_FROM ret_expr_or_cond
         ret_cond_not ::= expr POP_JUMP_IF_TRUE expr RETURN_END_IF ret_expr_or_cond
 
         or   ::= expr JUMP_IF_TRUE_OR_POP expr COME_FROM

uncompyle6/parsers/parse3.py

@@ -44,6 +44,10 @@ class Python3Parser(PythonParser):
 
         list_for ::= expr FOR_ITER designator list_iter jb_or_c
 
+        # This is seen in PyPy, but possibly it appears on other Python 3?
+        list_if     ::= expr jmp_false list_iter COME_FROM
+        list_if_not ::= expr jmp_true list_iter COME_FROM
+
         jb_or_c ::= JUMP_BACK
         jb_or_c ::= CONTINUE
 
@@ -52,6 +56,9 @@ class Python3Parser(PythonParser):
         setcomp_func ::= BUILD_SET_0 LOAD_FAST FOR_ITER designator comp_iter
                 JUMP_BACK RETURN_VALUE RETURN_LAST
 
+        setcomp_func ::= BUILD_SET_0 LOAD_FAST FOR_ITER designator comp_iter
+                COME_FROM JUMP_BACK RETURN_VALUE RETURN_LAST
+
         comp_body ::= dict_comp_body
         comp_body ::= set_comp_body
         dict_comp_body ::= expr expr MAP_ADD
@@ -113,9 +120,11 @@ class Python3Parser(PythonParser):
         classdefdeco1 ::= expr classdefdeco1 CALL_FUNCTION_1
         classdefdeco1 ::= expr classdefdeco2 CALL_FUNCTION_1
 
-        assert ::= assert_expr jmp_true LOAD_ASSERT RAISE_VARARGS_1
-        assert2 ::= assert_expr jmp_true LOAD_ASSERT expr CALL_FUNCTION_1 RAISE_VARARGS_1
-        assert2 ::= assert_expr jmp_true LOAD_ASSERT expr RAISE_VARARGS_2
+        assert ::= assert_expr jmp_true LOAD_ASSERT RAISE_VARARGS_1 COME_FROM
+        assert2 ::= assert_expr jmp_true LOAD_ASSERT expr CALL_FUNCTION_1
+                    RAISE_VARARGS_1 COME_FROM
+        assert2 ::= assert_expr jmp_true LOAD_ASSERT expr
+                    RAISE_VARARGS_2 COME_FROM
 
         assert_expr ::= expr
         assert_expr ::= assert_expr_or
@@ -132,6 +141,7 @@ class Python3Parser(PythonParser):
 
         _ifstmts_jump ::= return_if_stmts
         _ifstmts_jump ::= c_stmts_opt JUMP_FORWARD COME_FROM
+        _ifstmts_jump ::= c_stmts_opt COME_FROM
 
         iflaststmt ::= testexpr c_stmts_opt JUMP_ABSOLUTE
 
@@ -155,6 +165,7 @@ class Python3Parser(PythonParser):
         ifelsestmtr ::= testexpr return_if_stmts return_stmts
 
         ifelsestmtl ::= testexpr c_stmts_opt JUMP_BACK else_suitel
+        ifelsestmtl ::= testexpr c_stmts_opt COME_FROM JUMP_BACK else_suitel
 
         # FIXME: this feels like a hack. Is it just 1 or two
         # COME_FROMs?  the parsed tree for this and even with just the
@@ -329,6 +340,9 @@ class Python3Parser(PythonParser):
         forelselaststmtl  ::= SETUP_LOOP expr _for designator for_block POP_BLOCK else_suitel
                               COME_FROM_LOOP
 
+        whilestmt         ::= SETUP_LOOP testexpr l_stmts_opt COME_FROM JUMP_BACK POP_BLOCK
+                              COME_FROM_LOOP
+
         whilestmt         ::= SETUP_LOOP testexpr l_stmts_opt JUMP_BACK POP_BLOCK
                               COME_FROM_LOOP
 
@@ -356,6 +370,9 @@ class Python3Parser(PythonParser):
         while1stmt        ::= SETUP_LOOP l_stmts
         while1stmt        ::= SETUP_LOOP l_stmts COME_FROM_LOOP
 
+        while1stmt        ::= SETUP_LOOP l_stmts COME_FROM JUMP_BACK COME_FROM_LOOP
+
+
         # FIXME: investigate - can code really produce a NOP?
         whileTruestmt     ::= SETUP_LOOP l_stmts_opt JUMP_BACK NOP
                               COME_FROM_LOOP
@@ -453,6 +470,15 @@ class Python3Parser(PythonParser):
                 ('kwarg ' * args_kw) +
                 'expr ' * nak + token.type)
         self.add_unique_rule(rule, token.type, args_pos, customize)
+        if self.version >= 3.5:
+            rule = ('async_call_function ::= expr ' +
+                    ('pos_arg ' * args_pos) +
+                    ('kwarg ' * args_kw) +
+                    'expr ' * nak + token.type +
+                    ' GET_AWAITABLE LOAD_CONST YIELD_FROM')
+            self.add_unique_rule(rule, token.type, args_pos, customize)
+            self.add_unique_rule('expr ::= async_call_function', token.type, args_pos, customize)
+
         rule = ('classdefdeco2 ::= LOAD_BUILD_CLASS mkfunc %s%s_%d'
                 %  (('expr ' * (args_pos-1)), opname, args_pos))
         self.add_unique_rule(rule, token.type, args_pos, customize)
@@ -476,8 +502,15 @@ class Python3Parser(PythonParser):
 
             # build_class (see load_build_class)
 
-            build_list  ::= {expr}^n BUILD_LIST_n
-            build_list  ::= {expr}^n BUILD_TUPLE_n
+            # Even the below say _list, in the semantic rules we
+            # disambiguate tuples, and sets from lists
+
+            build_list ::= {expr}^n BUILD_LIST_n
+            build_list ::= {expr}^n BUILD_TUPLE_n
+            build_list ::= {expr}^n BUILD_SET_n
+            build_list ::= {expr}^n BUILD_LIST_UNPACK_n
+            build_list ::= {expr}^n BUILD_SET_UNPACK_n
+            build_list ::= {expr}^n BUILD_TUPLE_UNPACK_n
 
             load_closure  ::= {LOAD_CLOSURE}^n BUILD_TUPLE_n
             # call_function (see custom_classfunc_rule)
@@ -591,15 +624,21 @@ class Python3Parser(PythonParser):
                     self.add_unique_rule(rule, 'kvlist_n', 1, customize)
                     rule = "mapexpr ::=  BUILD_MAP_n kvlist_n"
                 elif self.version >= 3.5:
-                    if  opname != 'BUILD_MAP_WITH_CALL':
+                    if opname != 'BUILD_MAP_WITH_CALL':
                         rule = kvlist_n + ' ::= ' + 'expr ' * (token.attr*2)
                         self.add_unique_rule(rule, opname, token.attr, customize)
                         rule = "mapexpr ::=  %s %s" % (kvlist_n, opname)
+
                 else:
                     rule = kvlist_n + ' ::= ' + 'expr expr STORE_MAP ' * token.attr
                     self.add_unique_rule(rule, opname, token.attr, customize)
                     rule = "mapexpr ::=  %s %s" % (opname, kvlist_n)
                 self.add_unique_rule(rule, opname, token.attr, customize)
+            elif opname_base == 'BUILD_CONST_KEY_MAP':
+                # This is in 3.6+
+                kvlist_n = 'expr ' * (token.attr)
+                rule = "mapexpr ::= %sLOAD_CONST %s" % (kvlist_n, opname)
+                self.add_unique_rule(rule, opname, token.attr, customize)
             elif opname_base in ('UNPACK_EX',):
                 before_count, after_count = token.attr
                 rule = 'unpack ::= ' + opname + ' designator' * (before_count + after_count + 1)
@@ -636,6 +675,27 @@ class Python3Parser(PythonParser):
                     rule = ('mkfunc ::= kwargs %sexpr %s' %
                             ('pos_arg ' * args_pos, opname))
                 self.add_unique_rule(rule, opname, token.attr, customize)
+                if opname.startswith('MAKE_FUNCTION_A'):
+                    # rule = ('mkfunc2 ::= %s%sEXTENDED_ARG %s' %
+                    #         ('pos_arg ' * (args_pos), 'kwargs ' * (annotate_args-1), opname))
+                    self.add_unique_rule(rule, opname, token.attr, customize)
+                    if self.version >= 3.3:
+                        rule = ('mkfunc_annotate ::= %s%sannotate_tuple LOAD_CONST LOAD_CONST EXTENDED_ARG %s' %
+                                (('pos_arg ' * (args_pos)),
+                                 ('call_function ' * (annotate_args-1)), opname))
+                        self.add_unique_rule(rule, opname, token.attr, customize)
+                        rule = ('mkfunc_annotate ::= %s%sannotate_tuple LOAD_CONST LOAD_CONST EXTENDED_ARG %s' %
+                                (('pos_arg ' * (args_pos)),
+                                 ('annotate_arg ' * (annotate_args-1)), opname))
+                    else:
+                        rule = ('mkfunc_annotate ::= %s%sannotate_tuple LOAD_CONST EXTENDED_ARG %s' %
+                                (('pos_arg ' * (args_pos)),
+                                 ('annotate_arg ' * (annotate_args-1)), opname))
+                        self.add_unique_rule(rule, opname, token.attr, customize)
+                        rule = ('mkfunc_annotate ::= %s%sannotate_tuple LOAD_CONST EXTENDED_ARG %s' %
+                                (('pos_arg ' * (args_pos)),
+                                 ('call_function ' * (annotate_args-1)), opname))
+                    self.add_unique_rule(rule, opname, token.attr, customize)
             elif opname_base == 'CALL_METHOD':
                 # PyPy only - DRY with parse2
 
@@ -696,6 +756,8 @@ class Python3Parser(PythonParser):
         self.check_reduce['augassign1'] = 'AST'
         self.check_reduce['augassign2'] = 'AST'
         self.check_reduce['while1stmt'] = 'noAST'
+        self.check_reduce['annotate_tuple'] = 'noAST'
+        self.check_reduce['kwarg'] = 'noAST'
         # FIXME: remove parser errors caused by the below
         # self.check_reduce['while1elsestmt'] = 'noAST'
         return
@@ -704,6 +766,10 @@ class Python3Parser(PythonParser):
         lhs = rule[0]
         if lhs in ('augassign1', 'augassign2') and ast[0][0] == 'and':
             return True
+        elif lhs == 'annotate_tuple':
+            return not isinstance(tokens[first].attr, tuple)
+        elif lhs == 'kwarg':
+            return not isinstance(tokens[first].attr, str)
         elif lhs == 'while1elsestmt':
             # if SETUP_LOOP target spans the else part, then this is
             # not while1else. Also do for whileTrue?

uncompyle6/parsers/parse30.py

@@ -5,9 +5,9 @@ spark grammar differences over Python 3.1 for Python 3.0.
 from __future__ import print_function
 
 from uncompyle6.parser import PythonParserSingle
-from uncompyle6.parsers.parse3 import Python3Parser
+from uncompyle6.parsers.parse31 import Python31Parser
 
-class Python30Parser(Python3Parser):
+class Python30Parser(Python31Parser):
 
     def p_30(self, args):
         """
@@ -15,6 +15,11 @@ class Python30Parser(Python3Parser):
         stmt         ::= store_locals
         store_locals ::= LOAD_FAST STORE_LOCALS
 
+        # FIXME: combine with parse3.2
+        whileTruestmt     ::= SETUP_LOOP l_stmts_opt JUMP_BACK
+                              COME_FROM_LOOP
+        whileTruestmt     ::= SETUP_LOOP return_stmts
+                              COME_FROM_LOOP
 
         # In many ways Python 3.0 code generation is more like Python 2.6 than
         # it is 2.7 or 3.1. So we have a number of 2.6ish (and before) rules below
@@ -39,5 +44,10 @@ class Python30Parser(Python3Parser):
         setup_finally ::= STORE_FAST SETUP_FINALLY LOAD_FAST DELETE_FAST
         """
 
+    def add_custom_rules(self, tokens, customize):
+        super(Python30Parser, self).add_custom_rules(tokens, customize)
+        return
+    pass
+
 class Python30ParserSingle(Python30Parser, PythonParserSingle):
     pass

uncompyle6/parsers/parse31.py

@@ -36,16 +36,8 @@ class Python31Parser(Python32Parser):
 
     def add_custom_rules(self, tokens, customize):
         super(Python31Parser, self).add_custom_rules(tokens, customize)
-        for i, token in enumerate(tokens):
-            opname = token.type
-            if opname.startswith('MAKE_FUNCTION_A'):
-                args_pos, args_kw, annotate_args  = token.attr
-                # Check that there are 2 annotated params?
-                # rule = ('mkfunc2 ::= %s%sEXTENDED_ARG %s' %
-                #         ('pos_arg ' * (args_pos), 'kwargs ' * (annotate_args-1), opname))
-                rule = ('mkfunc_annotate ::= %s%sannotate_tuple LOAD_CONST EXTENDED_ARG %s' %
-                        (('pos_arg ' * (args_pos)),
-                         ('annotate_arg ' * (annotate_args-1)), opname))
-                self.add_unique_rule(rule, opname, token.attr, customize)
+        return
+    pass
+
 class Python31ParserSingle(Python31Parser, PythonParserSingle):
     pass

uncompyle6/parsers/parse32.py

@@ -46,6 +46,9 @@ class Python32Parser(Python3Parser):
                         (('pos_arg ' * (args_pos)),
                          ('annotate_arg ' * (annotate_args-1)), opname))
                 self.add_unique_rule(rule, opname, token.attr, customize)
+                pass
+            return
+        pass
 
 
 class Python32ParserSingle(Python32Parser, PythonParserSingle):

uncompyle6/parsers/parse35.py

@@ -16,36 +16,103 @@ class Python35Parser(Python34Parser):
 
     def p_35on(self, args):
         """
+        # The number of canned instructions in new statements is mind boggling.
+        # I'm sure by the time Python 4 comes around these will be turned
+        # into special opcodes
+
+        while1stmt  ::= SETUP_LOOP l_stmts COME_FROM JUMP_BACK
+                        POP_BLOCK COME_FROM_LOOP
+
+        # Python 3.5+ Await statement
+        stmt ::= await_stmt
+        await_stmt ::= call_function GET_AWAITABLE LOAD_CONST YIELD_FROM POP_TOP
+
         # Python 3.5+ has WITH_CLEANUP_START/FINISH
 
-        withstmt ::= expr SETUP_WITH exprlist suite_stmts_opt
-                    POP_BLOCK LOAD_CONST COME_FROM_WITH
-                    WITH_CLEANUP_START WITH_CLEANUP_FINISH END_FINALLY
+        withstmt    ::= expr
+                        SETUP_WITH exprlist suite_stmts_opt
+                        POP_BLOCK LOAD_CONST COME_FROM_WITH
+                        WITH_CLEANUP_START WITH_CLEANUP_FINISH END_FINALLY
 
-        withstmt ::= expr SETUP_WITH POP_TOP suite_stmts_opt
-                     POP_BLOCK LOAD_CONST COME_FROM_WITH
-                     WITH_CLEANUP_START WITH_CLEANUP_FINISH END_FINALLY
+        withstmt   ::= expr
+                       SETUP_WITH POP_TOP suite_stmts_opt
+                       POP_BLOCK LOAD_CONST COME_FROM_WITH
+                       WITH_CLEANUP_START WITH_CLEANUP_FINISH END_FINALLY
+
+        withasstmt ::= expr
+                       SETUP_WITH designator suite_stmts_opt
+                       POP_BLOCK LOAD_CONST COME_FROM_WITH
+                       WITH_CLEANUP_START WITH_CLEANUP_FINISH END_FINALLY
+
+
+        # Python 3.5+ async additions
+
+        stmt            ::= async_with_stmt
+        async_with_stmt ::= expr
+                            BEFORE_ASYNC_WITH GET_AWAITABLE LOAD_CONST YIELD_FROM
+                            SETUP_ASYNC_WITH POP_TOP suite_stmts_opt
+                            POP_BLOCK LOAD_CONST
+                            WITH_CLEANUP_START
+                            GET_AWAITABLE LOAD_CONST YIELD_FROM
+                            WITH_CLEANUP_FINISH END_FINALLY
+
+        stmt               ::= async_with_as_stmt
+        async_with_as_stmt ::= expr
+                               BEFORE_ASYNC_WITH GET_AWAITABLE LOAD_CONST YIELD_FROM
+                               SETUP_ASYNC_WITH designator suite_stmts_opt
+                               POP_BLOCK LOAD_CONST
+                               WITH_CLEANUP_START
+                               GET_AWAITABLE LOAD_CONST YIELD_FROM
+                               WITH_CLEANUP_FINISH END_FINALLY
+
+
+        stmt               ::= async_for_stmt
+        async_for_stmt     ::= SETUP_LOOP expr
+                               GET_AITER
+                               LOAD_CONST YIELD_FROM SETUP_EXCEPT GET_ANEXT LOAD_CONST
+                               YIELD_FROM
+                               designator
+                               POP_BLOCK JUMP_FORWARD COME_FROM_EXCEPT DUP_TOP
+                               LOAD_GLOBAL COMPARE_OP POP_JUMP_IF_FALSE
+                               POP_TOP POP_TOP POP_TOP POP_EXCEPT POP_BLOCK
+                               JUMP_ABSOLUTE END_FINALLY COME_FROM
+                               for_block POP_BLOCK JUMP_ABSOLUTE
+                               opt_come_from_loop
+
+        stmt               ::= async_forelse_stmt
+        async_forelse_stmt ::= SETUP_LOOP expr
+                               GET_AITER
+                               LOAD_CONST YIELD_FROM SETUP_EXCEPT GET_ANEXT LOAD_CONST
+                               YIELD_FROM
+                               designator
+                               POP_BLOCK JUMP_FORWARD COME_FROM_EXCEPT DUP_TOP
+                               LOAD_GLOBAL COMPARE_OP POP_JUMP_IF_FALSE
+                               POP_TOP POP_TOP POP_TOP POP_EXCEPT POP_BLOCK
+                               JUMP_ABSOLUTE END_FINALLY COME_FROM
+                               for_block POP_BLOCK JUMP_ABSOLUTE
+                               else_suite COME_FROM_LOOP
 
-        withasstmt ::= expr SETUP_WITH designator suite_stmts_opt
-                POP_BLOCK LOAD_CONST COME_FROM_WITH
-                WITH_CLEANUP_START WITH_CLEANUP_FINISH END_FINALLY
 
         inplace_op ::= INPLACE_MATRIX_MULTIPLY
         binary_op  ::= BINARY_MATRIX_MULTIPLY
 
         # Python 3.5+ does jump optimization
         # In <.3.5 the below is a JUMP_FORWARD to a JUMP_ABSOLUTE.
-        # in return_stmt, we will need the semantic actions in pysource.py
-        # to work out whether to dedent or not based on the presence of
-        # RETURN_END_IF vs RETURN_VALUE
+
+        return_if_stmt ::= ret_expr RETURN_END_IF POP_BLOCK
 
         ifelsestmtc ::= testexpr c_stmts_opt JUMP_FORWARD else_suitec
+        ifelsestmtc ::= testexpr c_stmts_opt jf_else else_suitec
+
         # ifstmt ::= testexpr c_stmts_opt
 
+        iflaststmt ::= testexpr c_stmts_opt JUMP_FORWARD
+
         # Python 3.3+ also has yield from. 3.5 does it
         # differently than 3.3, 3.4
 
         yield_from ::= expr GET_YIELD_FROM_ITER LOAD_CONST YIELD_FROM
+
         """
 
     def add_custom_rules(self, tokens, customize):

uncompyle6/parsers/parse36.py

@@ -21,6 +21,9 @@ class Python36Parser(Python35Parser):
 
         func_args36   ::= expr BUILD_TUPLE_0
         call_function ::= func_args36 unmapexpr CALL_FUNCTION_EX
+
+        withstmt ::= expr SETUP_WITH POP_TOP suite_stmts_opt POP_BLOCK LOAD_CONST
+                     WITH_CLEANUP_START WITH_CLEANUP_FINISH END_FINALLY
         """
 
     def add_custom_rules(self, tokens, customize):

uncompyle6/scanner.py

@@ -226,7 +226,7 @@ class Scanner(object):
         for given opcode <op>.
         """
         if op < self.opc.HAVE_ARGUMENT:
-            return 1
+            return 2 if self.version >= 3.6 else 1
         else:
             return 2 if self.version >= 3.6 else 3
 

uncompyle6/scanners/scanner2.py

@@ -159,7 +159,6 @@ class Scanner2(scan.Scanner):
                 # we sort them). That way, specific COME_FROM tags will match up
                 # properly. For example, a "loop" with an "if" nested in it should have the
                 # "loop" tag last so the grammar rule matches that properly.
-                # last_offset = -1
                 for jump_offset  in sorted(jump_targets[offset], reverse=True):
                     # if jump_offset == last_offset:
                     #     continue
@@ -265,13 +264,14 @@ class Scanner2(scan.Scanner):
                 # rule for that.
                 target = self.get_target(offset)
                 if target <= offset:
+                    op_name = 'JUMP_BACK'
                     if (offset in self.stmts
                         and self.code[offset+3] not in (self.opc.END_FINALLY,
-                                                        self.opc.POP_BLOCK)
-                        and offset not in self.not_continue):
-                        op_name = 'CONTINUE'
-                    else:
-                        op_name = 'JUMP_BACK'
+                                                        self.opc.POP_BLOCK)):
+                        if ((offset in self.linestartoffsets and
+                            self.code[self.prev[offset]] == self.opc.JUMP_ABSOLUTE)
+                            or offset not in self.not_continue):
+                            op_name = 'CONTINUE'
 
             elif op == self.opc.LOAD_GLOBAL:
                 if offset in self.load_asserts:
@@ -427,7 +427,7 @@ class Scanner2(scan.Scanner):
                 j = self.prev[s]
                 while code[j] in self.designator_ops:
                     j = self.prev[j]
-                if self.version >= 2.1 and code[j] == self.opc.FOR_ITER:
+                if self.version > 2.1 and code[j] == self.opc.FOR_ITER:
                     stmts.remove(s)
                     continue
             last_stmt = s
@@ -478,26 +478,27 @@ class Scanner2(scan.Scanner):
             elif op in self.setup_ops:
                 count_SETUP_ += 1
 
-    def detect_structure(self, offset, op):
+    def detect_control_flow(self, offset, op):
         """
         Detect type of block structures and their boundaries to fix optimized jumps
         in python2.3+
         """
 
-        # TODO: check the struct boundaries more precisely -Dan
-
         code = self.code
 
         # Detect parent structure
         parent = self.structs[0]
         start  = parent['start']
         end    = parent['end']
+
+        # Pick inner-most parent for our offset
         for struct in self.structs:
-            _start = struct['start']
-            _end   = struct['end']
-            if (_start <= offset < _end) and (_start >= start and _end <= end):
-                start  = _start
-                end    = _end
+            current_start = struct['start']
+            current_end   = struct['end']
+            if ((current_start <= offset < current_end)
+                and (current_start >= start and current_end <= end)):
+                start  = current_start
+                end    = current_end
                 parent = struct
 
         if op == self.opc.SETUP_LOOP:
@@ -588,7 +589,7 @@ class Scanner2(scan.Scanner):
 
                 target = self.get_target(jump_back, self.opc.JUMP_ABSOLUTE)
 
-                if (self.version >= 2.0 and
+                if (self.version > 2.1 and
                     code[target] in (self.opc.FOR_ITER, self.opc.GET_ITER)):
                     loop_type = 'for'
                 else:
@@ -616,7 +617,7 @@ class Scanner2(scan.Scanner):
                                        'start': jump_back+3,
                                        'end':   end})
         elif op == self.opc.SETUP_EXCEPT:
-            start  = offset+3
+            start  = offset + self.op_size(op)
             target = self.get_target(offset, op)
             end    = self.restrict_to_parent(target, parent)
             if target != end:
@@ -629,9 +630,23 @@ class Scanner2(scan.Scanner):
             # Now isolate the except and else blocks
             end_else = start_else = self.get_target(self.prev[end])
 
+
+            end_finally_offset = end
+            setup_except_nest = 0
+            while end_finally_offset < len(self.code):
+                if self.code[end_finally_offset] == self.opc.END_FINALLY:
+                    if setup_except_nest == 0:
+                        break
+                    else:
+                        setup_except_nest -= 1
+                elif self.code[end_finally_offset] == self.opc.SETUP_EXCEPT:
+                    setup_except_nest += 1
+                end_finally_offset += self.op_size(code[end_finally_offset])
+                pass
+
             # Add the except blocks
             i = end
-            while i < len(self.code) and self.code[i] != self.opc.END_FINALLY:
+            while i < len(self.code) and i < end_finally_offset:
                 jmp = self.next_except_jump(i)
                 if jmp is None: # check
                     i = self.next_stmt[i]
@@ -676,6 +691,8 @@ class Scanner2(scan.Scanner):
                 self.fixed_jumps[offset] = rtarget
                 return
 
+            jump_if_offset = offset
+
             start = offset+3
             pre = self.prev
 
@@ -698,6 +715,10 @@ class Scanner2(scan.Scanner):
                                        'end':   pre[target]})
                 return
 
+            # The op offset just before the target jump offset is important
+            # in making a determination of what we have. Save that.
+            pre_rtarget = pre[rtarget]
+
             # Is it an "and" inside an "if" or "while" block
             if op == self.opc.PJIF:
 
@@ -708,22 +729,22 @@ class Scanner2(scan.Scanner):
 
                 # If we still have any offsets in set, start working on it
                 if match:
-                    if code[pre[rtarget]] in self.jump_forward \
-                            and pre[rtarget] not in self.stmts \
-                            and self.restrict_to_parent(self.get_target(pre[rtarget]), parent) == rtarget:
-                        if code[pre[pre[rtarget]]] == self.opc.JUMP_ABSOLUTE \
+                    if code[pre_rtarget] in self.jump_forward \
+                            and pre_rtarget not in self.stmts \
+                            and self.restrict_to_parent(self.get_target(pre_rtarget), parent) == rtarget:
+                        if code[pre[pre_rtarget]] == self.opc.JUMP_ABSOLUTE \
                                 and self.remove_mid_line_ifs([offset]) \
-                                and target == self.get_target(pre[pre[rtarget]]) \
-                                and (pre[pre[rtarget]] not in self.stmts or self.get_target(pre[pre[rtarget]]) > pre[pre[rtarget]])\
-                                and 1 == len(self.remove_mid_line_ifs(self.rem_or(start, pre[pre[rtarget]], self.pop_jump_if, target))):
+                                and target == self.get_target(pre[pre_rtarget]) \
+                                and (pre[pre_rtarget] not in self.stmts or self.get_target(pre[pre_rtarget]) > pre[pre_rtarget])\
+                                and 1 == len(self.remove_mid_line_ifs(self.rem_or(start, pre[pre_rtarget], self.pop_jump_if, target))):
                             pass
-                        elif code[pre[pre[rtarget]]] == self.opc.RETURN_VALUE \
+                        elif code[pre[pre_rtarget]] == self.opc.RETURN_VALUE \
                                 and self.remove_mid_line_ifs([offset]) \
                                 and 1 == (len(set(self.remove_mid_line_ifs(self.rem_or(start,
-                                                                                       pre[pre[rtarget]],
+                                                                                       pre[pre_rtarget],
                                                                                        self.pop_jump_if, target)))
-                                              | set(self.remove_mid_line_ifs(self.rem_or(start, pre[pre[rtarget]],
-                                                            (self.opc.PJIF, self.opc.PJIT, self.opc.JUMP_ABSOLUTE), pre[rtarget], True))))):
+                                              | set(self.remove_mid_line_ifs(self.rem_or(start, pre[pre_rtarget],
+                                                            (self.opc.PJIF, self.opc.PJIT, self.opc.JUMP_ABSOLUTE), pre_rtarget, True))))):
                             pass
                         else:
                             fix = None
@@ -756,7 +777,7 @@ class Scanner2(scan.Scanner):
                 else:
                     assert_offset = offset + 3
                 if (assert_offset) in self.load_asserts:
-                    if code[pre[rtarget]] == self.opc.RAISE_VARARGS:
+                    if code[pre_rtarget] == self.opc.RAISE_VARARGS:
                         return
                     self.load_asserts.remove(assert_offset)
 
@@ -765,7 +786,7 @@ class Scanner2(scan.Scanner):
                     pass
                 elif code[next] in self.jump_forward and target == self.get_target(next):
                     if code[pre[next]] == self.opc.PJIF:
-                        if code[next] == self.opc.JUMP_FORWARD or target != rtarget or code[pre[pre[rtarget]]] not in (self.opc.JUMP_ABSOLUTE, self.opc.RETURN_VALUE):
+                        if code[next] == self.opc.JUMP_FORWARD or target != rtarget or code[pre[pre_rtarget]] not in (self.opc.JUMP_ABSOLUTE, self.opc.RETURN_VALUE):
                             self.fixed_jumps[offset] = pre[next]
                             return
                 elif code[next] == self.opc.JUMP_ABSOLUTE and code[target] in self.jump_forward:
@@ -782,17 +803,17 @@ class Scanner2(scan.Scanner):
                 return
 
             if self.version == 2.7:
-                if code[pre[rtarget]] == self.opc.JUMP_ABSOLUTE and pre[rtarget] in self.stmts \
-                        and pre[rtarget] != offset and pre[pre[rtarget]] != offset:
+                if code[pre_rtarget] == self.opc.JUMP_ABSOLUTE and pre_rtarget in self.stmts \
+                        and pre_rtarget != offset and pre[pre_rtarget] != offset:
                     if code[rtarget] == self.opc.JUMP_ABSOLUTE and code[rtarget+3] == self.opc.POP_BLOCK:
-                        if code[pre[pre[rtarget]]] != self.opc.JUMP_ABSOLUTE:
+                        if code[pre[pre_rtarget]] != self.opc.JUMP_ABSOLUTE:
                             pass
-                        elif self.get_target(pre[pre[rtarget]]) != target:
+                        elif self.get_target(pre[pre_rtarget]) != target:
                             pass
                         else:
-                            rtarget = pre[rtarget]
+                            rtarget = pre_rtarget
                     else:
-                        rtarget = pre[rtarget]
+                        rtarget = pre_rtarget
 
             # Does the "jump if" jump beyond a jump op?
             # That is, we have something like:
@@ -808,7 +829,6 @@ class Scanner2(scan.Scanner):
             # There are other contexts we may need to consider
             # like whether the target is "END_FINALLY"
             # or if the condition jump is to a forward location
-            pre_rtarget = pre[rtarget]
             code_pre_rtarget = code[pre_rtarget]
 
             if code_pre_rtarget in self.jump_forward:
@@ -828,20 +848,86 @@ class Scanner2(scan.Scanner):
                             jump_target = self.get_target(next_offset, next_op)
                             if jump_target in self.setup_loops:
                                 self.structs.append({'type':  'while-loop',
-                                       'start': start - 3,
+                                       'start': jump_if_offset,
                                        'end':   jump_target})
-                                self.fixed_jumps[start-3] = jump_target
+                                self.fixed_jumps[jump_if_offset] = jump_target
                                 return
 
                 end = self.restrict_to_parent(if_end, parent)
 
-                self.structs.append({'type':  'if-then',
-                                       'start': start-3,
-                                       'end':   pre_rtarget})
+                if_then_maybe = None
+
+                if 2.2 <= self.version <= 2.6:
+                    # Take the JUMP_IF target. In an "if/then", it will be
+                    # a POP_TOP instruction and the instruction before it
+                    # will be a JUMP_FORWARD to just after the POP_TOP.
+                    # For example:
+                    # Good:
+                    # 3  JUMP_IF_FALSE        33  'to 39'
+                    # ..
+                    # 36  JUMP_FORWARD          1  'to 40'
+                    # 39  POP_TOP
+                    # 40 ...
+                    # example:
+
+                    # BAD (is an "and"):
+                    # 28  JUMP_IF_FALSE         4  'to 35'
+                    # ...
+                    # 32  JUMP_ABSOLUTE        40  'to 40' # should be 36 or there should
+                    #                                      # be a COME_FROM at the pop top
+                    #                                      # before 40 to 35
+                    # 35  POP_TOP
+                    # 36 ...
+                    # 39  POP_TOP
+                    # 39_0  COME_FROM 3
+                    # 40 ...
+
+                    if self.opc.opname[code[jump_if_offset]].startswith('JUMP_IF'):
+                        jump_if_target = code[jump_if_offset+1]
+                        if self.opc.opname[code[jump_if_target + jump_if_offset + 3]] == 'POP_TOP':
+                            jump_inst = jump_if_target + jump_if_offset
+                            jump_offset = code[jump_inst+1]
+                            jump_op = self.opc.opname[code[jump_inst]]
+                            if (jump_op == 'JUMP_FORWARD' and jump_offset == 1):
+                                self.structs.append({'type':  'if-then',
+                                                     'start': start-3,
+                                                     'end':   pre_rtarget})
+
+                                self.thens[start] = end
+                            elif jump_op == 'JUMP_ABSOLUTE':
+                                if_then_maybe = {'type':  'if-then',
+                                                 'start': start-3,
+                                                 'end':   pre_rtarget}
+
+                elif self.version == 2.7:
+                    self.structs.append({'type':  'if-then',
+                                         'start': start-3,
+                                         'end':   pre_rtarget})
 
                 self.not_continue.add(pre_rtarget)
 
                 if rtarget < end:
+                    # We have an "else" block  of some kind.
+                    # Is it associated with "if_then_maybe" seen above?
+                    # These will be linked in this funny way:
+
+                    # 198  JUMP_IF_FALSE        18  'to 219'
+                    # 201  POP_TOP
+                    # ...
+                    # 216  JUMP_ABSOLUTE       256  'to 256'
+                    # 219  POP_TOP
+                    # ...
+                    # 252  JUMP_FORWARD          1  'to 256'
+                    # 255  POP_TOP
+                    # 256
+                    if if_then_maybe and jump_op == 'JUMP_ABSOLUTE':
+                        jump_target = self.get_target(jump_inst, code[jump_inst])
+                        if self.opc.opname[code[end]] == 'JUMP_FORWARD':
+                            end_target = self.get_target(end, code[end])
+                            if jump_target == end_target:
+                                self.structs.append(if_then_maybe)
+                                self.thens[start] = end
+
                     self.structs.append({'type':  'else',
                                        'start': rtarget,
                                        'end':   end})
@@ -850,7 +936,9 @@ class Scanner2(scan.Scanner):
                     self.structs.append({'type':  'if-then',
                                            'start': start,
                                            'end':   rtarget})
+                    self.thens[start] = rtarget
                     if self.version == 2.7 or code[pre_rtarget+1] != self.opc.JUMP_FORWARD:
+                        self.fixed_jumps[offset] = rtarget
                         self.return_end_ifs.add(pre_rtarget)
 
         elif op in self.pop_jump_if_or_pop:
@@ -883,11 +971,12 @@ class Scanner2(scan.Scanner):
         self.ignore_if = set()
         self.build_statement_indices()
 
-        # Containers filled by detect_structure()
+        # Containers filled by detect_control_flow()
         self.not_continue = set()
         self.return_end_ifs = set()
         self.setup_loop_targets = {}  # target given setup_loop offset
         self.setup_loops = {}  # setup_loop offset given target
+        self.thens = {} # JUMP_IF's that separate the 'then' part of an 'if'
 
         targets = {}
         for offset in self.op_range(0, n):
@@ -895,7 +984,7 @@ class Scanner2(scan.Scanner):
 
             # Determine structures and fix jumps in Python versions
             # since 2.3
-            self.detect_structure(offset, op)
+            self.detect_control_flow(offset, op)
 
             if op_has_argument(op, self.opc):
                 label = self.fixed_jumps.get(offset)

uncompyle6/scanners/scanner26.py

@@ -94,35 +94,32 @@ class Scanner26(scan.Scanner2):
             for instr in bytecode.get_instructions(co):
                 print(instr._disassemble())
 
-        # from xdis.bytecode import Bytecode
-        # bytecode = Bytecode(co, self.opc)
-        # for instr in bytecode.get_instructions(co):
-        #     print(instr._disassemble())
-
         # Container for tokens
         tokens = []
 
         customize = {}
+        if self.is_pypy:
+            customize['PyPy'] = 1
+
         Token = self.Token # shortcut
 
-        n = self.setup_code(co)
+        codelen = self.setup_code(co)
 
-        self.build_lines_data(co, n)
-        self.build_prev_op(n)
+        self.build_lines_data(co, codelen)
+        self.build_prev_op(codelen)
 
         free, names, varnames = self.unmangle_code_names(co, classname)
         self.names = names
 
-        codelen = len(self.code)
-
         # Scan for assertions. Later we will
         # turn 'LOAD_GLOBAL' to 'LOAD_ASSERT'.
         # 'LOAD_ASSERT' is used in assert statements.
         self.load_asserts = set()
-        for i in self.op_range(0, n):
-            # We need to detect the difference between
-            # "raise AssertionError" and
-            # "assert"
+        for i in self.op_range(0, codelen):
+            # We need to detect the difference between:
+            #   raise AssertionError
+            #  and
+            #   assert ...
             if (self.code[i] == self.opc.JUMP_IF_TRUE and
                 i + 4 < codelen and
                 self.code[i+3] == self.opc.POP_TOP and
@@ -161,12 +158,20 @@ class Scanner26(scan.Scanner2):
                 # we sort them). That way, specific COME_FROM tags will match up
                 # properly. For example, a "loop" with an "if" nested in it should have the
                 # "loop" tag last so the grammar rule matches that properly.
+                last_jump_offset = -1
                 for jump_offset  in sorted(jump_targets[offset], reverse=True):
-                    tokens.append(Token(
-                        'COME_FROM', None, repr(jump_offset),
-                        offset="%s_%d" % (offset, jump_idx),
-                        has_arg = True))
-                    jump_idx += 1
+                    if jump_offset != last_jump_offset:
+                        tokens.append(Token(
+                            'COME_FROM', None, repr(jump_offset),
+                            offset="%s_%d" % (offset, jump_idx),
+                            has_arg = True))
+                        jump_idx += 1
+                        last_jump_offset = jump_offset
+            elif offset in self.thens:
+                tokens.append(Token(
+                    'THEN', None, self.thens[offset],
+                    offset="%s_0" % offset,
+                    has_arg = True))
 
             has_arg = (op >= self.opc.HAVE_ARGUMENT)
             if has_arg:
@@ -240,16 +245,21 @@ class Scanner26(scan.Scanner2):
                 # boundaries The continue-type jumps help us get
                 # "continue" statements with would otherwise be turned
                 # into a "pass" statement because JUMPs are sometimes
-                # ignored in rules as just boundary overhead.
+                # ignored in rules as just boundary overhead.  In
+                # comprehensions we might sometimes classify JUMP_BACK
+                # as CONTINUE, but that's okay since we add a grammar
+                # rule for that.
                 target = self.get_target(offset)
                 if target <= offset:
+                    op_name = 'JUMP_BACK'
                     if (offset in self.stmts
                         and self.code[offset+3] not in (self.opc.END_FINALLY,
-                                                          self.opc.POP_BLOCK)
-                        and offset not in self.not_continue):
-                        op_name = 'CONTINUE'
+                                                          self.opc.POP_BLOCK)):
+                        if ((offset in self.linestartoffsets and
+                            tokens[-1].type == 'JUMP_BACK')
+                            or offset not in self.not_continue):
+                            op_name = 'CONTINUE'
                     else:
-                        op_name = 'JUMP_BACK'
                         # FIXME: this is a hack to catch stuff like:
                         #   if x: continue
                         # the "continue" is not on a new line.

uncompyle6/scanners/scanner3.py

@@ -126,7 +126,12 @@ class Scanner3(Scanner):
 
         if is_pypy:
             varargs_ops.add(self.opc.CALL_METHOD)
+        if self.version >= 3.6:
+            varargs_ops.add(self.opc.BUILD_CONST_KEY_MAP)
+
         self.varargs_ops = frozenset(varargs_ops)
+        # FIXME: remove the above in favor of:
+        # self.varargs_ops = frozenset(self.opc.hasvargs)
 
     def opName(self, offset):
         return self.opc.opname[self.code[offset]]
@@ -148,7 +153,7 @@ class Scanner3(Scanner):
         """
 
         show_asm = self.show_asm if not show_asm else show_asm
-        # show_asm = 'after'
+        # show_asm = 'both'
         if show_asm in ('both', 'before'):
             bytecode = Bytecode(co, self.opc)
             for instr in bytecode.get_instructions(co):
@@ -199,6 +204,7 @@ class Scanner3(Scanner):
         # Get jump targets
         # Format: {target offset: [jump offsets]}
         jump_targets = self.find_jump_targets(show_asm)
+        last_op_was_break = False
 
         for inst in bytecode:
 
@@ -321,11 +327,15 @@ class Scanner3(Scanner):
                 if target <= inst.offset:
                     next_opname = self.opname[self.code[inst.offset+3]]
                     if (inst.offset in self.stmts and
-                        next_opname not in ('END_FINALLY', 'POP_BLOCK',
+                        (self.version != 3.0 or (hasattr(inst, 'linestart'))) and
+                        (next_opname not in ('END_FINALLY', 'POP_BLOCK',
                                             # Python 3.0 only uses POP_TOP
-                                            'POP_TOP')
-                        and inst.offset not in self.not_continue):
-                        opname = 'CONTINUE'
+                                            'POP_TOP'))):
+                        if (self.version >= 3.4 or
+                            (inst.offset not in self.not_continue) or
+                            (tokens[-1].type == 'RETURN_VALUE')):
+                            opname = 'CONTINUE'
+                            pass
                     else:
                         opname = 'JUMP_BACK'
                         # FIXME: this is a hack to catch stuff like:
@@ -334,15 +344,21 @@ class Scanner3(Scanner):
                         # There are other situations where we don't catch
                         # CONTINUE as well.
                         if tokens[-1].type == 'JUMP_BACK' and tokens[-1].attr <= argval:
-                            # intern is used because we are changing the *previous* token
-                            tokens[-1].type = intern('CONTINUE')
-
+                            if tokens[-2].type == 'BREAK_LOOP':
+                                del tokens[-1]
+                            else:
+                                # intern is used because we are changing the *previous* token
+                                tokens[-1].type = intern('CONTINUE')
+                    if last_op_was_break and opname == 'CONTINUE':
+                        last_op_was_break = False
+                        continue
             elif op == self.opc.RETURN_VALUE:
                 if inst.offset in self.return_end_ifs:
                     opname = 'RETURN_END_IF'
             elif inst.offset in self.load_asserts:
                 opname = 'LOAD_ASSERT'
 
+            last_op_was_break = opname == 'BREAK_LOOP'
             tokens.append(
                 Token(
                     type_ = opname,
@@ -434,7 +450,7 @@ class Scanner3(Scanner):
         self.build_statement_indices()
         self.else_start = {}
 
-        # Containers filled by detect_structure()
+        # Containers filled by detect_control_flow()
         self.not_continue = set()
         self.return_end_ifs = set()
         self.setup_loop_targets = {}  # target given setup_loop offset
@@ -446,16 +462,20 @@ class Scanner3(Scanner):
 
             # Determine structures and fix jumps in Python versions
             # since 2.3
-            self.detect_structure(offset, targets)
+            self.detect_control_flow(offset, targets)
 
             has_arg = (op >= op3.HAVE_ARGUMENT)
             if has_arg:
                 label = self.fixed_jumps.get(offset)
-                oparg = code[offset+1] + code[offset+2] * 256
+                if self.version >= 3.6:
+                    oparg = code[offset+1]
+                else:
+                    oparg = code[offset+1] + code[offset+2] * 256
+                next_offset = offset + self.op_size(op)
 
                 if label is None:
                     if op in op3.hasjrel and op != self.opc.FOR_ITER:
-                        label = offset + 3 + oparg
+                        label = next_offset + oparg
                     elif op in op3.hasjabs:
                         if op in self.jump_if_pop:
                             if oparg > offset:
@@ -485,7 +505,7 @@ class Scanner3(Scanner):
         prelim = self.all_instr(start, end, self.statement_opcodes)
 
         # Initialize final container with statements with
-        # preliminnary data
+        # preliminary data
         stmts = self.stmts = set(prelim)
 
         # Same for opcode sequences
@@ -564,16 +584,16 @@ class Scanner3(Scanner):
         op = self.code[offset]
         if self.version  >= 3.6:
             target = self.code[offset+1]
-            if op in op3.hasjrel:
+            if op in self.opc.hasjrel:
                 target += offset + 2
         else:
             target = self.code[offset+1] + self.code[offset+2] * 256
-            if op in op3.hasjrel:
+            if op in self.opc.hasjrel:
                 target += offset + 3
 
         return target
 
-    def detect_structure(self, offset, targets):
+    def detect_control_flow(self, offset, targets):
         """
         Detect structures and their boundaries to fix optimized jumps
         in python2.3+
@@ -600,13 +620,15 @@ class Scanner3(Scanner):
                 parent = struct
 
         if op == self.opc.SETUP_LOOP:
-
             # We categorize loop types: 'for', 'while', 'while 1' with
             # possibly suffixes '-loop' and '-else'
             # Try to find the jump_back instruction of the loop.
             # It could be a return instruction.
 
-            start = offset+3
+            if self.version <= 3.5:
+                start = offset+3
+            else:
+                start = offset+2
             target = self.get_target(offset)
             end    = self.restrict_to_parent(target, parent)
             self.setup_loop_targets[offset] = target
@@ -616,22 +638,35 @@ class Scanner3(Scanner):
                 self.fixed_jumps[offset] = end
             (line_no, next_line_byte) = self.lines[offset]
             jump_back = self.last_instr(start, end, self.opc.JUMP_ABSOLUTE,
-                                          next_line_byte, False)
+                                            next_line_byte, False)
 
-            if jump_back and jump_back != self.prev_op[end] and self.is_jump_forward(jump_back+3):
-                if (code[self.prev_op[end]] == self.opc.RETURN_VALUE
-                    or (code[self.prev_op[end]] == self.opc.POP_BLOCK
-                         and code[self.prev_op[self.prev_op[end]]] == self.opc.RETURN_VALUE)):
+            if jump_back:
+                jump_forward_offset = jump_back+3
+            else:
+                jump_forward_offset = None
+
+            return_val_offset1 = self.prev[self.prev[end]]
+
+            if (jump_back and jump_back != self.prev_op[end]
+                and self.is_jump_forward(jump_forward_offset)):
+                if (code[self.prev_op[end]] == self.opc.RETURN_VALUE or
+                    (code[self.prev_op[end]] == self.opc.POP_BLOCK
+                     and code[return_val_offset1] == self.opc.RETURN_VALUE)):
                     jump_back = None
-            if not jump_back: # loop suite ends in return. wtf right?
-                jump_back = self.last_instr(start, end, self.opc.RETURN_VALUE) + 1
+            if not jump_back:
+                jump_back = self.last_instr(start, end, self.opc.RETURN_VALUE)
                 if not jump_back:
                     return
+
+                jump_back += 2
+                if_offset = None
                 if code[self.prev_op[next_line_byte]] not in POP_JUMP_TF:
-                    loop_type = 'for'
-                else:
+                    if_offset = self.prev[next_line_byte]
+                if if_offset:
                     loop_type = 'while'
-                    self.ignore_if.add(self.prev_op[next_line_byte])
+                    self.ignore_if.add(if_offset)
+                else:
+                    loop_type = 'for'
                 target = next_line_byte
                 end = jump_back + 3
             else:
@@ -645,6 +680,7 @@ class Scanner3(Scanner):
                 elif target < offset:
                     self.fixed_jumps[offset] = jump_back+4
                     end = jump_back+4
+
                 target = self.get_target(jump_back)
 
                 if code[target] in (self.opc.FOR_ITER, self.opc.GET_ITER):
@@ -652,6 +688,7 @@ class Scanner3(Scanner):
                 else:
                     loop_type = 'while'
                     test = self.prev_op[next_line_byte]
+
                     if test == offset:
                         loop_type = 'while 1'
                     elif self.code[test] in op3.hasjabs+op3.hasjrel:
@@ -692,38 +729,40 @@ class Scanner3(Scanner):
                                      'end': prev_op[target]})
                 return
 
-            # Is it an "and" inside an "if" block
+            # The op offset just before the target jump offset is important
+            # in making a determination of what we have. Save that.
+            pre_rtarget = prev_op[rtarget]
+
+            # Is it an "and" inside an "if" or "while" block
             if op == self.opc.POP_JUMP_IF_FALSE:
+
                 # Search for another POP_JUMP_IF_FALSE targetting the same op,
                 # in current statement, starting from current offset, and filter
                 # everything inside inner 'or' jumps and midline ifs
                 match = self.rem_or(start, self.next_stmt[offset],
                                     self.opc.POP_JUMP_IF_FALSE, target)
-                # We can't remove mid-line ifs because line structures have changed
-                # from restructBytecode().
-                #  match = self.remove_mid_line_ifs(match)
 
                 # If we still have any offsets in set, start working on it
                 if match:
-                    is_jump_forward = self.is_jump_forward(prev_op[rtarget])
-                    if (is_jump_forward and prev_op[rtarget] not in self.stmts and
-                        self.restrict_to_parent(self.get_target(prev_op[rtarget]), parent) == rtarget):
-                        if (code[prev_op[prev_op[rtarget]]] == self.opc.JUMP_ABSOLUTE
+                    is_jump_forward = self.is_jump_forward(pre_rtarget)
+                    if (is_jump_forward and pre_rtarget not in self.stmts and
+                        self.restrict_to_parent(self.get_target(pre_rtarget), parent) == rtarget):
+                        if (code[prev_op[pre_rtarget]] == self.opc.JUMP_ABSOLUTE
                             and self.remove_mid_line_ifs([offset]) and
-                            target == self.get_target(prev_op[prev_op[rtarget]]) and
-                            (prev_op[prev_op[rtarget]] not in self.stmts or
-                             self.get_target(prev_op[prev_op[rtarget]]) > prev_op[prev_op[rtarget]]) and
-                            1 == len(self.remove_mid_line_ifs(self.rem_or(start, prev_op[prev_op[rtarget]], POP_JUMP_TF, target)))):
+                            target == self.get_target(prev_op[pre_rtarget]) and
+                            (prev_op[pre_rtarget] not in self.stmts or
+                             self.get_target(prev_op[pre_rtarget]) > prev_op[pre_rtarget]) and
+                            1 == len(self.remove_mid_line_ifs(self.rem_or(start, prev_op[pre_rtarget], POP_JUMP_TF, target)))):
                             pass
-                        elif (code[prev_op[prev_op[rtarget]]] == self.opc.RETURN_VALUE
+                        elif (code[prev_op[pre_rtarget]] == self.opc.RETURN_VALUE
                               and self.remove_mid_line_ifs([offset]) and
-                              1 == (len(set(self.remove_mid_line_ifs(self.rem_or(start, prev_op[prev_op[rtarget]],
+                              1 == (len(set(self.remove_mid_line_ifs(self.rem_or(start, prev_op[pre_rtarget],
                                                                                  POP_JUMP_TF, target))) |
-                                    set(self.remove_mid_line_ifs(self.rem_or(start, prev_op[prev_op[rtarget]],
+                                    set(self.remove_mid_line_ifs(self.rem_or(start, prev_op[pre_rtarget],
                                                                              (self.opc.POP_JUMP_IF_FALSE,
                                                                               self.opc.POP_JUMP_IF_TRUE,
                                                                               self.opc.JUMP_ABSOLUTE),
-                                                                             prev_op[rtarget], True)))))):
+                                                                             pre_rtarget, True)))))):
                             pass
                         else:
                             fix = None
@@ -751,7 +790,7 @@ class Scanner3(Scanner):
                     if code[prev_op[next]] == self.opc.POP_JUMP_IF_FALSE:
                         if (code[next] == self.opc.JUMP_FORWARD
                             or target != rtarget
-                            or code[prev_op[prev_op[rtarget]]] not in
+                            or code[prev_op[pre_rtarget]] not in
                             (self.opc.JUMP_ABSOLUTE, self.opc.RETURN_VALUE)):
                             self.fixed_jumps[offset] = prev_op[next]
                             return
@@ -764,14 +803,14 @@ class Scanner3(Scanner):
             if offset in self.ignore_if:
                 return
 
-            if (code[prev_op[rtarget]] == self.opc.JUMP_ABSOLUTE and
-                prev_op[rtarget] in self.stmts and
-                prev_op[rtarget] != offset and
-                prev_op[prev_op[rtarget]] != offset and
+            if (code[pre_rtarget] == self.opc.JUMP_ABSOLUTE and
+                pre_rtarget in self.stmts and
+                pre_rtarget != offset and
+                prev_op[pre_rtarget] != offset and
                 not (code[rtarget] == self.opc.JUMP_ABSOLUTE and
                      code[rtarget+3] == self.opc.POP_BLOCK and
-                     code[prev_op[prev_op[rtarget]]] != self.opc.JUMP_ABSOLUTE)):
-                rtarget = prev_op[rtarget]
+                     code[prev_op[pre_rtarget]] != self.opc.JUMP_ABSOLUTE)):
+                rtarget = pre_rtarget
 
             # Does the "jump if" jump beyond a jump op?
             # That is, we have something like:
@@ -787,12 +826,11 @@ class Scanner3(Scanner):
             # There are other contexts we may need to consider
             # like whether the target is "END_FINALLY"
             # or if the condition jump is to a forward location
-            if self.is_jump_forward(prev_op[rtarget]):
-                rrtarget = prev_op[rtarget]
-                if_end = self.get_target(rrtarget)
+            if self.is_jump_forward(pre_rtarget):
+                if_end = self.get_target(pre_rtarget)
 
                 # If the jump target is back, we are looping
-                if (if_end < rrtarget and
+                if (if_end < pre_rtarget and
                     (code[prev_op[if_end]] == self.opc.SETUP_LOOP)):
                     if (if_end > start):
                         return
@@ -801,19 +839,26 @@ class Scanner3(Scanner):
 
                 self.structs.append({'type': 'if-then',
                                      'start': start,
-                                     'end': prev_op[rtarget]})
-                self.not_continue.add(prev_op[rtarget])
+                                     'end': pre_rtarget})
+                self.not_continue.add(pre_rtarget)
 
                 if rtarget < end and (
                         code[rtarget] not in (self.opc.END_FINALLY,
                                               self.opc.JUMP_ABSOLUTE) and
-                        code[prev_op[rrtarget]] not in (self.opc.POP_EXCEPT,
+                        code[prev_op[pre_rtarget]] not in (self.opc.POP_EXCEPT,
                                                         self.opc.END_FINALLY)):
                     self.structs.append({'type': 'else',
                                          'start': rtarget,
                                          'end': end})
                     self.else_start[rtarget] = end
-            elif code[prev_op[rtarget]] == self.opc.RETURN_VALUE:
+            elif self.is_jump_back(pre_rtarget):
+                if_end = rtarget
+                self.structs.append({'type': 'if-then',
+                                     'start': start,
+                                     'end': pre_rtarget})
+                self.not_continue.add(pre_rtarget)
+            elif code[pre_rtarget] in (self.opc.RETURN_VALUE,
+                                       self.opc.BREAK_LOOP):
                 self.structs.append({'type': 'if-then',
                                      'start': start,
                                      'end': rtarget})
@@ -843,8 +888,24 @@ class Scanner3(Scanner):
                                 return
                             pass
                     pass
-                self.return_end_ifs.add(prev_op[rtarget])
+                if code[pre_rtarget] == self.opc.RETURN_VALUE:
+                    self.return_end_ifs.add(pre_rtarget)
+                else:
+                    self.fixed_jumps[offset] = rtarget
+                    self.not_continue.add(pre_rtarget)
+            else:
+                # For now, we'll only tag forward jump.
+                if rtarget > offset:
+                    self.fixed_jumps[offset] = rtarget
 
+        elif op == self.opc.SETUP_EXCEPT:
+            target = self.get_target(offset)
+            end    = self.restrict_to_parent(target, parent)
+            self.fixed_jumps[offset] = end
+        elif op == self.opc.SETUP_FINALLY:
+            target = self.get_target(offset)
+            end    = self.restrict_to_parent(target, parent)
+            self.fixed_jumps[offset] = end
         elif op in self.jump_if_pop:
             target = self.get_target(offset)
             if target > offset:
@@ -882,6 +943,16 @@ class Scanner3(Scanner):
                 pass
         return
 
+    def is_jump_back(self, offset):
+        """
+        Return True if the code at offset is some sort of jump back.
+        That is, it is ether "JUMP_FORWARD" or an absolute jump that
+        goes forward.
+        """
+        if self.code[offset] != self.opc.JUMP_ABSOLUTE:
+            return False
+        return offset > self.get_target(offset)
+
     def next_except_jump(self, start):
         """
         Return the next jump that was generated by an except SomeException:

uncompyle6/scanners/scanner30.py

@@ -1,4 +1,4 @@
-#  Copyright (c) 2016 by Rocky Bernstein
+#  Copyright (c) 2016, 2017 by Rocky Bernstein
 """
 Python 3.0 bytecode scanner/deparser
 
@@ -12,6 +12,8 @@ from __future__ import print_function
 from xdis.opcodes import opcode_30 as opc
 JUMP_OPs = map(lambda op: opc.opname[op], opc.hasjrel + opc.hasjabs)
 
+JUMP_TF = frozenset([opc.JUMP_IF_FALSE, opc.JUMP_IF_TRUE])
+
 from uncompyle6.scanners.scanner3 import Scanner3
 class Scanner30(Scanner3):
 
@@ -20,6 +22,373 @@ class Scanner30(Scanner3):
         return
     pass
 
+    def detect_control_flow(self, offset, targets):
+        """
+        Detect structures and their boundaries to fix optimized jumps
+        Python 3.0 is more like Python 2.6 than it is Python 3.x.
+        So we have a special routine here.
+        """
+
+        code = self.code
+        op = code[offset]
+
+        # Detect parent structure
+        parent = self.structs[0]
+        start = parent['start']
+        end = parent['end']
+
+        # Pick inner-most parent for our offset
+        for struct in self.structs:
+            current_start = struct['start']
+            current_end   = struct['end']
+            if ((current_start <= offset < current_end)
+                and (current_start >= start and current_end <= end)):
+                start = current_start
+                end = current_end
+                parent = struct
+
+        if op == self.opc.SETUP_LOOP:
+            # We categorize loop types: 'for', 'while', 'while 1' with
+            # possibly suffixes '-loop' and '-else'
+            # Try to find the jump_back instruction of the loop.
+            # It could be a return instruction.
+
+            start = offset+3
+            target = self.get_target(offset)
+            end    = self.restrict_to_parent(target, parent)
+            self.setup_loop_targets[offset] = target
+            self.setup_loops[target] = offset
+
+            if target != end:
+                self.fixed_jumps[offset] = end
+
+            (line_no, next_line_byte) = self.lines[offset]
+            jump_back = self.last_instr(start, end, self.opc.JUMP_ABSOLUTE,
+                                            next_line_byte, False)
+
+            if jump_back:
+                jump_forward_offset = jump_back+3
+            else:
+                jump_forward_offset = None
+
+            return_val_offset1 = self.prev[self.prev[end]]
+
+            if (jump_back and jump_back != self.prev_op[end]
+                and self.is_jump_forward(jump_forward_offset)):
+                if (code[self.prev_op[end]] == self.opc.RETURN_VALUE or
+                    (code[self.prev_op[end]] == self.opc.POP_BLOCK
+                     and code[return_val_offset1] == self.opc.RETURN_VALUE)):
+                    jump_back = None
+            if not jump_back:
+                # loop suite ends in return
+                jump_back = self.last_instr(start, end, self.opc.RETURN_VALUE)
+                if not jump_back:
+                    return
+
+                jump_back += 2
+                if_offset = None
+                if code[self.prev_op[next_line_byte]] not in JUMP_TF:
+                    if_offset = self.prev[next_line_byte]
+                if if_offset:
+                    loop_type = 'while'
+                    self.ignore_if.add(if_offset)
+                else:
+                    loop_type = 'for'
+                target = next_line_byte
+                end = jump_back + 3
+            else:
+                if self.get_target(jump_back) >= next_line_byte:
+                    jump_back = self.last_instr(start, end, self.opc.JUMP_ABSOLUTE, start, False)
+                if end > jump_back+4 and self.is_jump_forward(end):
+                    if self.is_jump_forward(jump_back+4):
+                        if self.get_target(jump_back+4) == self.get_target(end):
+                            self.fixed_jumps[offset] = jump_back+4
+                            end = jump_back+4
+                elif target < offset:
+                    self.fixed_jumps[offset] = jump_back+4
+                    end = jump_back+4
+
+                target = self.get_target(jump_back)
+
+                if code[target] in (self.opc.FOR_ITER, self.opc.GET_ITER):
+                    loop_type = 'for'
+                else:
+                    loop_type = 'while'
+                    test = self.prev_op[next_line_byte]
+
+                    if test == offset:
+                        loop_type = 'while 1'
+                    elif self.code[test] in opc.hasjabs+opc.hasjrel:
+                        self.ignore_if.add(test)
+                        test_target = self.get_target(test)
+                        if test_target > (jump_back+3):
+                            jump_back = test_target
+                self.not_continue.add(jump_back)
+            self.loops.append(target)
+            self.structs.append({'type': loop_type + '-loop',
+                                   'start': target,
+                                   'end':   jump_back})
+            if jump_back+3 != end:
+                self.structs.append({'type': loop_type + '-else',
+                                       'start': jump_back+3,
+                                       'end':   end})
+        elif op in JUMP_TF:
+            start = offset + self.op_size(op)
+            target = self.get_target(offset)
+            rtarget = self.restrict_to_parent(target, parent)
+            prev_op = self.prev_op
+
+            # Do not let jump to go out of parent struct bounds
+            if target != rtarget and parent['type'] == 'and/or':
+                self.fixed_jumps[offset] = rtarget
+                return
+
+            # Does this jump to right after another conditional jump that is
+            # not myself?  If so, it's part of a larger conditional.
+            # rocky: if we have a conditional jump to the next instruction, then
+            # possibly I am "skipping over" a "pass" or null statement.
+
+            if ((code[prev_op[target]] in self.pop_jump_if_pop) and
+                (target > offset) and prev_op[target] != offset):
+                self.fixed_jumps[offset] = prev_op[target]
+                self.structs.append({'type': 'and/or',
+                                     'start': start,
+                                     'end': prev_op[target]})
+                return
+
+            # The op offset just before the target jump offset is important
+            # in making a determination of what we have. Save that.
+            pre_rtarget = prev_op[rtarget]
+
+            # Is it an "and" inside an "if" or "while" block
+            if op == opc.JUMP_IF_FALSE:
+
+                # Search for another JUMP_IF_FALSE targetting the same op,
+                # in current statement, starting from current offset, and filter
+                # everything inside inner 'or' jumps and midline ifs
+                match = self.rem_or(start, self.next_stmt[offset],
+                                    opc.JUMP_IF_FALSE, target)
+
+                # If we still have any offsets in set, start working on it
+                if match:
+                    is_jump_forward = self.is_jump_forward(pre_rtarget)
+                    if (is_jump_forward and pre_rtarget not in self.stmts and
+                        self.restrict_to_parent(self.get_target(pre_rtarget), parent) == rtarget):
+                        if (code[prev_op[pre_rtarget]] == self.opc.JUMP_ABSOLUTE
+                            and self.remove_mid_line_ifs([offset]) and
+                            target == self.get_target(prev_op[pre_rtarget]) and
+                            (prev_op[pre_rtarget] not in self.stmts or
+                             self.get_target(prev_op[pre_rtarget]) > prev_op[pre_rtarget]) and
+                            1 == len(self.remove_mid_line_ifs(self.rem_or(start, prev_op[pre_rtarget], JUMP_TF, target)))):
+                            pass
+                        elif (code[prev_op[pre_rtarget]] == self.opc.RETURN_VALUE
+                              and self.remove_mid_line_ifs([offset]) and
+                              1 == (len(set(self.remove_mid_line_ifs(self.rem_or(start, prev_op[pre_rtarget],
+                                                                                 JUMP_TF, target))) |
+                                    set(self.remove_mid_line_ifs(self.rem_or(start, prev_op[pre_rtarget],
+                                                                             (opc.JUMP_IF_FALSE,
+                                                                              opc.JUMP_IF_TRUE,
+                                                                              opc.JUMP_ABSOLUTE),
+                                                                             pre_rtarget, True)))))):
+                            pass
+                        else:
+                            fix = None
+                            jump_ifs = self.all_instr(start, self.next_stmt[offset],
+                                                      opc.JUMP_IF_FALSE)
+                            last_jump_good = True
+                            for j in jump_ifs:
+                                if target == self.get_target(j):
+                                    if self.lines[j].next == j + 3 and last_jump_good:
+                                        fix = j
+                                        break
+                                else:
+                                    last_jump_good = False
+                            self.fixed_jumps[offset] = fix or match[-1]
+                            return
+                    else:
+                        self.fixed_jumps[offset] = match[-1]
+                        return
+            # op == JUMP_IF_TRUE
+            else:
+                next = self.next_stmt[offset]
+                if prev_op[next] == offset:
+                    pass
+                elif self.is_jump_forward(next) and target == self.get_target(next):
+                    if code[prev_op[next]] == opc.JUMP_IF_FALSE:
+                        if (code[next] == self.opc.JUMP_FORWARD
+                            or target != rtarget
+                            or code[prev_op[pre_rtarget]] not in
+                            (self.opc.JUMP_ABSOLUTE, self.opc.RETURN_VALUE)):
+                            self.fixed_jumps[offset] = prev_op[next]
+                            return
+                elif (code[next] == self.opc.JUMP_ABSOLUTE and self.is_jump_forward(target) and
+                      self.get_target(target) == self.get_target(next)):
+                    self.fixed_jumps[offset] = prev_op[next]
+                    return
+
+            # Don't add a struct for a while test, it's already taken care of
+            if offset in self.ignore_if:
+                return
+
+            if (code[pre_rtarget] == self.opc.JUMP_ABSOLUTE and
+                pre_rtarget in self.stmts and
+                pre_rtarget != offset and
+                prev_op[pre_rtarget] != offset and
+                not (code[rtarget] == self.opc.JUMP_ABSOLUTE and
+                     code[rtarget+3] == self.opc.POP_BLOCK and
+                     code[prev_op[pre_rtarget]] != self.opc.JUMP_ABSOLUTE)):
+                rtarget = pre_rtarget
+
+            # Does the "jump if" jump beyond a jump op?
+            # That is, we have something like:
+            #  JUMP_IF_FALSE HERE
+            #  ...
+            # JUMP_FORWARD
+            # HERE:
+            #
+            # If so, this can be block inside an "if" statement
+            # or a conditional assignment like:
+            #   x = 1 if x else 2
+            #
+            # There are other contexts we may need to consider
+            # like whether the target is "END_FINALLY"
+            # or if the condition jump is to a forward location
+            if self.is_jump_forward(pre_rtarget):
+                if_end = self.get_target(pre_rtarget)
+
+                # If the jump target is back, we are looping
+                if (if_end < pre_rtarget and
+                    (code[prev_op[if_end]] == self.opc.SETUP_LOOP)):
+                    if (if_end > start):
+                        return
+
+                end = self.restrict_to_parent(if_end, parent)
+
+                self.structs.append({'type': 'if-then',
+                                     'start': start,
+                                     'end': pre_rtarget})
+                self.not_continue.add(pre_rtarget)
+
+                # if rtarget < end and (
+                #         code[rtarget] not in (self.opc.END_FINALLY,
+                #                               self.opc.JUMP_ABSOLUTE) and
+                #         code[prev_op[pre_rtarget]] not in (self.opc.POP_EXCEPT,
+                #                                         self.opc.END_FINALLY)):
+                #     self.structs.append({'type': 'else',
+                #                          'start': rtarget,
+                #                          'end': end})
+                #     self.else_start[rtarget] = end
+            elif self.is_jump_back(pre_rtarget):
+                if_end = rtarget
+                self.structs.append({'type': 'if-then',
+                                     'start': start,
+                                     'end': pre_rtarget})
+                self.not_continue.add(pre_rtarget)
+            elif code[pre_rtarget] in (self.opc.RETURN_VALUE,
+                                       self.opc.BREAK_LOOP):
+                self.structs.append({'type': 'if-then',
+                                     'start': start,
+                                     'end': rtarget})
+                # It is important to distingish if this return is inside some sort
+                # except block return
+                jump_prev = prev_op[offset]
+                if self.is_pypy and code[jump_prev] == self.opc.COMPARE_OP:
+                    if self.opc.cmp_op[code[jump_prev+1]] == 'exception match':
+                        return
+                if self.version >= 3.5:
+                    # Python 3.5 may remove as dead code a JUMP
+                    # instruction after a RETURN_VALUE. So we check
+                    # based on seeing SETUP_EXCEPT various places.
+                    if code[rtarget] == self.opc.SETUP_EXCEPT:
+                        return
+                    # Check that next instruction after pops and jump is
+                    # not from SETUP_EXCEPT
+                    next_op = rtarget
+                    if code[next_op] == self.opc.POP_BLOCK:
+                        next_op += self.op_size(self.code[next_op])
+                    if code[next_op] == self.opc.JUMP_ABSOLUTE:
+                        next_op += self.op_size(self.code[next_op])
+                    if next_op in targets:
+                        for try_op in targets[next_op]:
+                            come_from_op = code[try_op]
+                            if come_from_op == self.opc.SETUP_EXCEPT:
+                                return
+                            pass
+                    pass
+                if code[pre_rtarget] == self.opc.RETURN_VALUE:
+                    self.return_end_ifs.add(pre_rtarget)
+                else:
+                    self.fixed_jumps[offset] = rtarget
+                    self.not_continue.add(pre_rtarget)
+
+
+        elif op == self.opc.SETUP_EXCEPT:
+            target = self.get_target(offset)
+            end    = self.restrict_to_parent(target, parent)
+            self.fixed_jumps[offset] = end
+        elif op == self.opc.SETUP_FINALLY:
+            target = self.get_target(offset)
+            end    = self.restrict_to_parent(target, parent)
+            self.fixed_jumps[offset] = end
+        elif op in self.jump_if_pop:
+            target = self.get_target(offset)
+            if target > offset:
+                unop_target = self.last_instr(offset, target, self.opc.JUMP_FORWARD, target)
+                if unop_target and code[unop_target+3] != self.opc.ROT_TWO:
+                    self.fixed_jumps[offset] = unop_target
+                else:
+                    self.fixed_jumps[offset] = self.restrict_to_parent(target, parent)
+                    pass
+                pass
+        elif self.version >= 3.5:
+            # 3.5+ has Jump optimization which too often causes RETURN_VALUE to get
+            # misclassified as RETURN_END_IF. Handle that here.
+            # In RETURN_VALUE, JUMP_ABSOLUTE, RETURN_VALUE is never RETURN_END_IF
+            if op == self.opc.RETURN_VALUE:
+                if (offset+1 < len(code) and code[offset+1] == self.opc.JUMP_ABSOLUTE and
+                    offset in self.return_end_ifs):
+                    self.return_end_ifs.remove(offset)
+                    pass
+                pass
+            elif op == self.opc.JUMP_FORWARD:
+                # If we have:
+                #   JUMP_FORWARD x, [non-jump, insns], RETURN_VALUE, x:
+                # then RETURN_VALUE is not RETURN_END_IF
+                rtarget = self.get_target(offset)
+                rtarget_prev = self.prev[rtarget]
+                if (code[rtarget_prev] == self.opc.RETURN_VALUE and
+                    rtarget_prev in self.return_end_ifs):
+                    i = rtarget_prev
+                    while i != offset:
+                        if code[i] in [opc.JUMP_FORWARD, opc.JUMP_ABSOLUTE]:
+                            return
+                        i = self.prev[i]
+                    self.return_end_ifs.remove(rtarget_prev)
+                pass
+        return
+
+    def rem_or(self, start, end, instr, target=None, include_beyond_target=False):
+        """
+        Find offsets of all requested <instr> between <start> and <end>,
+        optionally <target>ing specified offset, and return list found
+        <instr> offsets which are not within any POP_JUMP_IF_TRUE jumps.
+        """
+        assert(start>=0 and end<=len(self.code) and start <= end)
+
+        # Find all offsets of requested instructions
+        instr_offsets = self.all_instr(start, end, instr, target, include_beyond_target)
+        # Get all JUMP_IF_TRUE (or) offsets
+        pjit_offsets = self.all_instr(start, end, opc.JUMP_IF_TRUE)
+        filtered = []
+        for pjit_offset in pjit_offsets:
+            pjit_tgt = self.get_target(pjit_offset) - 3
+            for instr_offset in instr_offsets:
+                if instr_offset <= pjit_offset or instr_offset >= pjit_tgt:
+                    filtered.append(instr_offset)
+            instr_offsets = filtered
+            filtered = []
+        return instr_offsets
+
 if __name__ == "__main__":
     from uncompyle6 import PYTHON_VERSION
     if PYTHON_VERSION == 3.0:

uncompyle6/semantics/consts.py

@@ -0,0 +1,347 @@
+#  Copyright (c) 2017 by Rocky Bernstein
+"""Constants used in pysource.py"""
+
+import re, sys
+from uncompyle6.parsers.astnode import AST
+from uncompyle6 import PYTHON3
+from uncompyle6.scanners.tok import Token, NoneToken
+
+if PYTHON3:
+    minint = -sys.maxsize-1
+    maxint = sys.maxsize
+else:
+    minint = -sys.maxint-1
+    maxint = sys.maxint
+
+
+LINE_LENGTH = 80
+
+# Some ASTs used for comparing code fragments (like 'return None' at
+# the end of functions).
+
+RETURN_LOCALS = AST('return_stmt',
+                    [ AST('ret_expr', [AST('expr', [ Token('LOAD_LOCALS') ])]),
+                      Token('RETURN_VALUE')])
+
+NONE = AST('expr', [ NoneToken ] )
+
+RETURN_NONE = AST('stmt',
+                  [ AST('return_stmt',
+                        [ NONE, Token('RETURN_VALUE')]) ])
+
+PASS = AST('stmts',
+           [ AST('sstmt',
+                 [ AST('stmt',
+                       [ AST('passstmt', [])])])])
+
+ASSIGN_DOC_STRING = lambda doc_string: \
+  AST('stmt',
+      [ AST('assign',
+            [ AST('expr', [ Token('LOAD_CONST', pattr=doc_string) ]),
+              AST('designator', [ Token('STORE_NAME', pattr='__doc__')])
+            ])])
+
+NAME_MODULE = AST('stmt',
+                [ AST('assign',
+                    [ AST('expr',
+                          [Token('LOAD_NAME', pattr='__name__', offset=0, has_arg=True)]),
+                      AST('designator',
+                          [ Token('STORE_NAME', pattr='__module__', offset=3, has_arg=True)])
+                      ])])
+
+# God intended \t, but Python has decided to use 4 spaces.
+# If you want real tabs, use Go.
+# TAB = '\t'
+TAB = ' ' * 4
+INDENT_PER_LEVEL = ' ' # additional intent per pretty-print level
+
+TABLE_R = {
+    'STORE_ATTR':	( '%c.%[1]{pattr}', 0),
+#   'STORE_SUBSCR':	( '%c[%c]', 0, 1 ),
+    'DELETE_ATTR':	( '%|del %c.%[-1]{pattr}\n', 0 ),
+#   'EXEC_STMT':	( '%|exec %c in %[1]C\n', 0, (0,maxint,', ') ),
+}
+
+TABLE_R0 = {
+#    'BUILD_LIST':	( '[%C]',      (0,-1,', ') ),
+#    'BUILD_TUPLE':	( '(%C)',      (0,-1,', ') ),
+#    'CALL_FUNCTION':	( '%c(%P)', 0, (1,-1,', ') ),
+}
+TABLE_DIRECT = {
+    'BINARY_ADD':	( '+' ,),
+    'BINARY_SUBTRACT':	( '-' ,),
+    'BINARY_MULTIPLY':	( '*' ,),
+    'BINARY_DIVIDE':	( '/' ,),
+    'BINARY_MATRIX_MULTIPLY':	( '@' ,),
+    'BINARY_TRUE_DIVIDE':	( '/' ,),   # Not in <= 2.1
+    'BINARY_FLOOR_DIVIDE':	( '//' ,),
+    'BINARY_MODULO':	( '%%',),
+    'BINARY_POWER':	( '**',),
+    'BINARY_LSHIFT':	( '<<',),
+    'BINARY_RSHIFT':	( '>>',),
+    'BINARY_AND':	( '&' ,),
+    'BINARY_OR':	( '|' ,),
+    'BINARY_XOR':	( '^' ,),
+    'INPLACE_ADD':	( '+=' ,),
+    'INPLACE_SUBTRACT':	( '-=' ,),
+    'INPLACE_MULTIPLY':	( '*=' ,),
+    'INPLACE_MATRIX_MULTIPLY':	( '@=' ,),
+    'INPLACE_DIVIDE':	( '/=' ,),
+    'INPLACE_TRUE_DIVIDE':	( '/=' ,),  # Not in <= 2.1; 2.6 generates INPLACE_DIVIDE only?
+    'INPLACE_FLOOR_DIVIDE':	( '//=' ,),
+    'INPLACE_MODULO':	( '%%=',),
+    'INPLACE_POWER':	( '**=',),
+    'INPLACE_LSHIFT':	( '<<=',),
+    'INPLACE_RSHIFT':	( '>>=',),
+    'INPLACE_AND':	( '&=' ,),
+    'INPLACE_OR':	( '|=' ,),
+    'INPLACE_XOR':	( '^=' ,),
+    'binary_expr':	( '%c %c %c', 0, -1, 1 ),
+
+    'UNARY_POSITIVE':	( '+',),
+    'UNARY_NEGATIVE':	( '-',),
+    'UNARY_INVERT':	( '~%c'),
+    'unary_expr':   ( '%c%c', 1, 0),
+
+    'unary_not':	( 'not %c', 0 ),
+    'unary_convert':	( '`%c`', 0 ),
+    'get_iter':	( 'iter(%c)', 0 ),
+    'slice0':		( '%c[:]', 0 ),
+    'slice1':		( '%c[%p:]', 0, (1, 100) ),
+    'slice2':		( '%c[:%p]', 0, (1, 100) ),
+    'slice3':		( '%c[%p:%p]', 0, (1, 100), (2, 100) ),
+
+    'IMPORT_FROM':	( '%{pattr}', ),
+    'load_attr':	( '%c.%[1]{pattr}', 0),
+    'LOAD_FAST':	( '%{pattr}', ),
+    'LOAD_NAME':	( '%{pattr}', ),
+    'LOAD_CLASSNAME':	( '%{pattr}', ),
+    'LOAD_GLOBAL':	( '%{pattr}', ),
+    'LOAD_DEREF':	( '%{pattr}', ),
+    'LOAD_LOCALS':	( 'locals()', ),
+    'LOAD_ASSERT':  ( '%{pattr}', ),
+#   'LOAD_CONST':	( '%{pattr}', ),	# handled by n_LOAD_CONST
+    'DELETE_FAST':	( '%|del %{pattr}\n', ),
+    'DELETE_NAME':	( '%|del %{pattr}\n', ),
+    'DELETE_GLOBAL':	( '%|del %{pattr}\n', ),
+    'delete_subscr':	( '%|del %c[%c]\n', 0, 1,),
+    'binary_subscr':	( '%c[%p]', 0, (1, 100)),
+    'binary_subscr2':	( '%c[%p]', 0, (1, 100)),
+    'store_subscr':	( '%c[%c]', 0, 1),
+    'STORE_FAST':	( '%{pattr}', ),
+    'STORE_NAME':	( '%{pattr}', ),
+    'STORE_GLOBAL':	( '%{pattr}', ),
+    'STORE_DEREF':	( '%{pattr}', ),
+    'unpack':		( '%C%,', (1, maxint, ', ') ),
+
+    # This nonterminal we create on the fly in semantic routines
+    'unpack_w_parens':	( '(%C%,)', (1, maxint, ', ') ),
+
+    'unpack_list':	( '[%C]', (1, maxint, ', ') ),
+    'build_tuple2':	( '%P', (0, -1, ', ', 100) ),
+
+    # 'list_compr':	( '[ %c ]', -2),	# handled by n_list_compr
+    'list_iter':	( '%c', 0),
+    'list_for':		( ' for %c in %c%c', 2, 0, 3 ),
+    'list_if':		( ' if %c%c', 0, 2 ),
+    'list_if_not':		( ' if not %p%c', (0, 22), 2 ),
+    'lc_body':		( '', ),	# ignore when recusing
+
+    'comp_iter':	( '%c', 0),
+    'comp_if':		( ' if %c%c', 0, 2 ),
+    'comp_ifnot':	( ' if not %p%c', (0, 22), 2 ),
+    'comp_body':	( '', ),	# ignore when recusing
+    'set_comp_body':    ( '%c', 0 ),
+    'gen_comp_body':    ( '%c', 0 ),
+    'dict_comp_body':   ( '%c:%c', 1, 0 ),
+
+    'assign':		( '%|%c = %p\n', -1, (0, 200) ),
+
+    # The 2nd parameter should have a = suffix.
+    # There is a rule with a 4th parameter "designator"
+    # which we don't use here.
+    'augassign1':	( '%|%c %c %c\n', 0, 2, 1),
+
+    'augassign2':	( '%|%c.%[2]{pattr} %c %c\n', 0, -3, -4),
+    'designList':	( '%c = %c', 0, -1 ),
+    'and':          	( '%c and %c', 0, 2 ),
+    'ret_and':        	( '%c and %c', 0, 2 ),
+    'and2':          	( '%c', 3 ),
+    'or':           	( '%c or %c', 0, 2 ),
+    'ret_or':           	( '%c or %c', 0, 2 ),
+    'conditional':  ( '%p if %p else %p', (2, 27), (0, 27), (4, 27)),
+    'ret_cond':     ( '%p if %p else %p', (2, 27), (0, 27), (-1, 27)),
+    'conditionalnot':  ( '%p if not %p else %p', (2, 27), (0, 22), (4, 27)),
+    'ret_cond_not':  ( '%p if not %p else %p', (2, 27), (0, 22), (-1, 27)),
+    'conditional_lambda':  ( '(%c if %c else %c)', 2, 0, 3),
+    'return_lambda':    ('%c', 0),
+    'compare':		( '%p %[-1]{pattr} %p', (0, 19), (1, 19) ),
+    'cmp_list':		( '%p %p', (0, 20), (1, 19)),
+    'cmp_list1':	( '%[3]{pattr} %p %p', (0, 19), (-2, 19)),
+    'cmp_list2':	( '%[1]{pattr} %p', (0, 19)),
+#   'classdef': 	(), # handled by n_classdef()
+    'funcdef':  	( '\n\n%|def %c\n', -2), # -2 to handle closures
+    'funcdefdeco':  	( '\n\n%c', 0),
+    'mkfuncdeco':  	( '%|@%c\n%c', 0, 1),
+    'mkfuncdeco0':  	( '%|def %c\n', 0),
+    'classdefdeco':  	( '\n\n%c', 0),
+    'classdefdeco1':  	( '%|@%c\n%c', 0, 1),
+    'kwarg':    	( '%[0]{pattr}=%c', 1),
+    'kwargs':    	( '%D', (0, maxint, ', ') ),
+
+    'assert_expr_or': ( '%c or %c', 0, 2 ),
+    'assert_expr_and':    ( '%c and %c', 0, 2 ),
+    'print_items_stmt': ( '%|print %c%c,\n', 0, 2),  # Python 2 only
+    'print_items_nl_stmt': ( '%|print %c%c\n', 0, 2),
+    'print_item':  ( ', %c', 0),
+    'print_nl':	( '%|print\n', ),
+    'print_to':		( '%|print >> %c, %c,\n', 0, 1 ),
+    'print_to_nl':	( '%|print >> %c, %c\n', 0, 1 ),
+    'print_nl_to':	( '%|print >> %c\n', 0 ),
+    'print_to_items':	( '%C', (0, 2, ', ') ),
+
+    'call_stmt':	( '%|%p\n', (0, 200)),
+    'break_stmt':	( '%|break\n', ),
+    'continue_stmt':	( '%|continue\n', ),
+
+    'raise_stmt0':	( '%|raise\n', ),
+    'raise_stmt1':	( '%|raise %c\n', 0),
+    'raise_stmt3':	( '%|raise %c, %c, %c\n', 0, 1, 2),
+#    'yield':	( 'yield %c', 0),
+#    'return_stmt':	( '%|return %c\n', 0),
+
+    'ifstmt':		( '%|if %c:\n%+%c%-', 0, 1 ),
+    'iflaststmt':		( '%|if %c:\n%+%c%-', 0, 1 ),
+    'iflaststmtl':		( '%|if %c:\n%+%c%-', 0, 1 ),
+    'testtrue':         ( 'not %p', (0, 22) ),
+
+    'ifelsestmt':	( '%|if %c:\n%+%c%-%|else:\n%+%c%-', 0, 1, 3 ),
+    'ifelsestmtc':	( '%|if %c:\n%+%c%-%|else:\n%+%c%-', 0, 1, 3 ),
+    'ifelsestmtl':	( '%|if %c:\n%+%c%-%|else:\n%+%c%-', 0, 1, 3 ),
+    'ifelifstmt':	( '%|if %c:\n%+%c%-%c', 0, 1, 3 ),
+    'elifelifstmt':	( '%|elif %c:\n%+%c%-%c', 0, 1, 3 ),
+    'elifstmt':		( '%|elif %c:\n%+%c%-', 0, 1 ),
+    'elifelsestmt':	( '%|elif %c:\n%+%c%-%|else:\n%+%c%-', 0, 1, 3 ),
+    'ifelsestmtr':	( '%|if %c:\n%+%c%-%|else:\n%+%c%-', 0, 1, 2 ),
+    'ifelsestmtr2':	( '%|if %c:\n%+%c%-%|else:\n%+%c%-\n\n', 0, 1, 3 ), # has COME_FROM
+    'elifelsestmtr':	( '%|elif %c:\n%+%c%-%|else:\n%+%c%-\n\n', 0, 1, 2 ),
+    'elifelsestmtr2':	( '%|elif %c:\n%+%c%-%|else:\n%+%c%-\n\n', 0, 1, 3 ), # has COME_FROM
+
+    'whileTruestmt':	( '%|while True:\n%+%c%-\n\n', 1 ),
+    'whilestmt':	( '%|while %c:\n%+%c%-\n\n', 1, 2 ),
+    'while1stmt':	( '%|while 1:\n%+%c%-\n\n', 1 ),
+    'while1elsestmt':  ( '%|while 1:\n%+%c%-%|else:\n%+%c%-\n\n', 1, -2 ),
+    'whileelsestmt':	( '%|while %c:\n%+%c%-%|else:\n%+%c%-\n\n', 1, 2, -2 ),
+    'whileelselaststmt':	( '%|while %c:\n%+%c%-%|else:\n%+%c%-', 1, 2, -2 ),
+    'forstmt':		( '%|for %c in %c:\n%+%c%-\n\n', 3, 1, 4 ),
+    'forelsestmt':	(
+        '%|for %c in %c:\n%+%c%-%|else:\n%+%c%-\n\n', 3, 1, 4, -2),
+    'forelselaststmt':	(
+        '%|for %c in %c:\n%+%c%-%|else:\n%+%c%-', 3, 1, 4, -2),
+    'forelselaststmtl':	(
+        '%|for %c in %c:\n%+%c%-%|else:\n%+%c%-\n\n', 3, 1, 4, -2),
+    'trystmt':		( '%|try:\n%+%c%-%c\n\n', 1, 3 ),
+    'tryelsestmt':	( '%|try:\n%+%c%-%c%|else:\n%+%c%-\n\n', 1, 3, 4 ),
+    'tryelsestmtc':	( '%|try:\n%+%c%-%c%|else:\n%+%c%-', 1, 3, 4 ),
+    'tryelsestmtl':	( '%|try:\n%+%c%-%c%|else:\n%+%c%-', 1, 3, 4 ),
+    'tf_trystmt':	( '%c%-%c%+', 1, 3 ),
+    'tf_tryelsestmt':	( '%c%-%c%|else:\n%+%c', 1, 3, 4 ),
+    'tryfinallystmt':	( '%|try:\n%+%c%-%|finally:\n%+%c%-\n\n', 1, 5 ),
+    'except':           ( '%|except:\n%+%c%-', 3 ),
+    'except_cond1':	( '%|except %c:\n', 1 ),
+    'except_suite':     ( '%+%c%-%C', 0, (1, maxint, '') ),
+    'except_suite_finalize':     ( '%+%c%-%C', 1, (3, maxint, '') ),
+    'passstmt':		( '%|pass\n', ),
+    'STORE_FAST':	( '%{pattr}', ),
+    'kv':		( '%c: %c', 3, 1 ),
+    'kv2':		( '%c: %c', 1, 2 ),
+    'mapexpr':		( '{%[1]C}', (0, maxint, ', ') ),
+    'importstmt': ( '%|import %c\n', 2),
+    'importfrom': ( '%|from %[2]{pattr} import %c\n', 3 ),
+    'importstar': ( '%|from %[2]{pattr} import *\n', ),
+}
+
+
+MAP_DIRECT = (TABLE_DIRECT, )
+MAP_R0 = (TABLE_R0, -1, 0)
+MAP_R = (TABLE_R, -1)
+
+MAP = {
+    'stmt':		MAP_R,
+    'call_function':	MAP_R,
+    'del_stmt':		MAP_R,
+    'designator':	MAP_R,
+    'exprlist':		MAP_R0,
+}
+
+PRECEDENCE = {
+    'build_list':           0,
+    'mapexpr':              0,
+    'unary_convert':        0,
+    'dictcomp':             0,
+    'setcomp':              0,
+    'list_compr':           0,
+    'genexpr':              0,
+
+    'load_attr':            2,
+    'binary_subscr':        2,
+    'binary_subscr2':       2,
+    'slice0':               2,
+    'slice1':               2,
+    'slice2':               2,
+    'slice3':               2,
+    'buildslice2':          2,
+    'buildslice3':          2,
+    'call_function':        2,
+
+    'BINARY_POWER':         4,
+
+    'unary_expr':           6,
+
+    'BINARY_MULTIPLY':      8,
+    'BINARY_DIVIDE':        8,
+    'BINARY_TRUE_DIVIDE':   8,
+    'BINARY_FLOOR_DIVIDE':  8,
+    'BINARY_MODULO':        8,
+
+    'BINARY_ADD':           10,
+    'BINARY_SUBTRACT':      10,
+
+    'BINARY_LSHIFT':        12,
+    'BINARY_RSHIFT':        12,
+
+    'BINARY_AND':           14,
+
+    'BINARY_XOR':           16,
+
+    'BINARY_OR':            18,
+
+    'cmp':                  20,
+
+    'unary_not':            22,
+
+    'and':                  24,
+    'ret_and':              24,
+
+    'or':                   26,
+    'ret_or':               26,
+
+    'conditional':          28,
+    'conditionalnot':       28,
+    'ret_cond':             28,
+    'ret_cond_not':         28,
+
+    '_mklambda':            30,
+    'yield':                101,
+    'yield_from':           101
+}
+
+ASSIGN_TUPLE_PARAM = lambda param_name: \
+             AST('expr', [ Token('LOAD_FAST', pattr=param_name) ])
+
+escape = re.compile(r'''
+            (?P<prefix> [^%]* )
+            % ( \[ (?P<child> -? \d+ ) \] )?
+                ((?P<type> [^{] ) |
+                 ( [{] (?P<expr> [^}]* ) [}] ))
+        ''', re.VERBOSE)

uncompyle6/semantics/fragments.py

@@ -69,17 +69,22 @@ from uncompyle6.show import (
     maybe_show_ast_param_default,
 )
 
-from uncompyle6.semantics.pysource import AST, INDENT_PER_LEVEL, NONE, PRECEDENCE, \
-     ParserError, TABLE_DIRECT, escape, find_globals, minint, MAP
+from uncompyle6.parsers.astnode import AST
+
+from uncompyle6.semantics.pysource import (
+    ParserError, find_globals, StringIO)
+
+from uncompyle6.semantics.consts import (
+    INDENT_PER_LEVEL, NONE, PRECEDENCE,
+    TABLE_DIRECT, escape, minint, MAP
+    )
 
 from uncompyle6.semantics.make_function import find_all_globals, find_none
 
 if PYTHON3:
     from itertools import zip_longest
-    from io import StringIO
 else:
     from itertools import izip_longest as zip_longest
-    from StringIO import StringIO
 
 
 from spark_parser import DEFAULT_DEBUG as PARSER_DEFAULT_DEBUG

uncompyle6/semantics/make_function.py

@@ -71,7 +71,11 @@ def make_function3_annotate(self, node, isLambda, nested=1,
     # MAKE_FUNCTION_... or MAKE_CLOSURE_...
     assert node[-1].type.startswith('MAKE_')
 
-    annotate_tuple = node[annotate_last]
+    annotate_tuple = None
+    for annotate_last in range(len(node)-1, -1, -1):
+        if node[annotate_last] == 'annotate_tuple':
+            annotate_tuple = node[annotate_last]
+            break
     annotate_args = {}
 
     if (annotate_tuple == 'annotate_tuple'
@@ -148,13 +152,19 @@ def make_function3_annotate(self, node, isLambda, nested=1,
     suffix = ''
     for param in paramnames[:i]:
         self.write(suffix, param)
-        if param in annotate_args:
-            value, string = annotate_args[param]
-            if string:
-                self.write(': "%s"' % value)
-            else:
-                self.write(': %s' % value)
         suffix = ', '
+        if param in annotate_tuple[0].attr:
+            p = annotate_tuple[0].attr.index(param)
+            self.write(': ')
+            self.preorder(node[p])
+            if (line_number != self.line_number):
+                suffix = ",\n" + indent
+                line_number = self.line_number
+            # value, string = annotate_args[param]
+            # if string:
+            #     self.write(': "%s"' % value)
+            # else:
+            #     self.write(': %s' % value)
 
     suffix = ', ' if i > 0 else ''
     for n in node:
@@ -163,7 +173,10 @@ def make_function3_annotate(self, node, isLambda, nested=1,
             param = paramnames[i]
             self.write(param)
             if param in annotate_args:
-                self.write(':"%s' % annotate_args[param])
+                aa = annotate_args[param]
+                if isinstance(aa, tuple):
+                    aa = aa[0]
+                self.write(': "%s"' % aa)
             self.write('=')
             i += 1
             self.preorder(n)
@@ -189,6 +202,9 @@ def make_function3_annotate(self, node, isLambda, nested=1,
         i = 0
         for n in node[0]:
             if n == 'kwarg':
+                if (line_number != self.line_number):
+                    self.write("\n" + indent)
+                    line_number = self.line_number
                 self.write('%s=' % n[0].pattr)
                 self.preorder(n[1])
                 if i < last:
@@ -207,19 +223,24 @@ def make_function3_annotate(self, node, isLambda, nested=1,
         self.write(": ")
     else:
         self.write(')')
-        if 'return' in annotate_args:
-            value, string = annotate_args['return']
-            if string:
-                self.write(' -> "%s"' % value)
-            else:
-                self.write(' -> %s' % value)
+        if 'return' in annotate_tuple[0].attr:
+            if (line_number != self.line_number):
+                self.write("\n" + indent)
+                line_number = self.line_number
+            self.write(' -> ')
+            # value, string = annotate_args['return']
+            # if string:
+            #     self.write(' -> "%s"' % value)
+            # else:
+            #     self.write(' -> %s' % value)
+            self.preorder(node[annotate_last-1])
 
         self.println(":")
 
     if (len(code.co_consts) > 0 and
         code.co_consts[0] is not None and not isLambda): # ugly
         # docstring exists, dump it
-        print_docstring(self, indent, code.co_consts[0])
+        print_docstring(self, self.indent, code.co_consts[0])
 
     code._tokens = None # save memory
     assert ast == 'stmts'

uncompyle6/semantics/pysource.py

@@ -1,4 +1,4 @@
-#  Copyright (c) 2015, 2016 by Rocky Bernstein
+#  Copyright (c) 2015-2017 by Rocky Bernstein
 #  Copyright (c) 2005 by Dan Pascu <dan@windowmaker.org>
 #  Copyright (c) 2000-2002 by hartmut Goebel <h.goebel@crazy-compilers.com>
 #  Copyright (c) 1999 John Aycock
@@ -69,21 +69,30 @@ methods implement most of the below.
 """
 from __future__ import print_function
 
-import sys, re
+import sys
 
 from uncompyle6 import PYTHON3
 from xdis.code import iscode
+from xdis.util import COMPILER_FLAG_BIT
+
 from uncompyle6.parser import get_python_parser
 from uncompyle6.parsers.astnode import AST
 from spark_parser import GenericASTTraversal, DEFAULT_DEBUG as PARSER_DEFAULT_DEBUG
 from uncompyle6.scanner import Code, get_scanner
-from uncompyle6.scanners.tok import Token, NoneToken
 import uncompyle6.parser as python_parser
 from uncompyle6.semantics.make_function import (
     make_function2, make_function3, make_function3_annotate, find_globals)
 from uncompyle6.semantics.parser_error import ParserError
 from uncompyle6.semantics.check_ast import checker
 from uncompyle6.semantics.helper import print_docstring
+from uncompyle6.scanners.tok import Token
+
+from uncompyle6.semantics.consts import (
+    LINE_LENGTH, RETURN_LOCALS, NONE, RETURN_NONE, PASS,
+    ASSIGN_DOC_STRING, NAME_MODULE, TAB,
+    INDENT_PER_LEVEL, TABLE_R, TABLE_DIRECT, MAP_DIRECT,
+    MAP, PRECEDENCE, ASSIGN_TUPLE_PARAM, escape, maxint, minint)
+
 
 from uncompyle6.show import (
     maybe_show_ast,
@@ -91,348 +100,8 @@ from uncompyle6.show import (
 
 if PYTHON3:
     from io import StringIO
-    minint = -sys.maxsize-1
-    maxint = sys.maxsize
 else:
     from StringIO import StringIO
-    minint = -sys.maxint-1
-    maxint = sys.maxint
-
-LINE_LENGTH = 80
-
-# Some ASTs used for comparing code fragments (like 'return None' at
-# the end of functions).
-
-RETURN_LOCALS = AST('return_stmt',
-                    [ AST('ret_expr', [AST('expr', [ Token('LOAD_LOCALS') ])]),
-                      Token('RETURN_VALUE')])
-
-NONE = AST('expr', [ NoneToken ] )
-
-RETURN_NONE = AST('stmt',
-                  [ AST('return_stmt',
-                        [ NONE, Token('RETURN_VALUE')]) ])
-
-PASS = AST('stmts',
-           [ AST('sstmt',
-                 [ AST('stmt',
-                       [ AST('passstmt', [])])])])
-
-ASSIGN_DOC_STRING = lambda doc_string: \
-  AST('stmt',
-      [ AST('assign',
-            [ AST('expr', [ Token('LOAD_CONST', pattr=doc_string) ]),
-              AST('designator', [ Token('STORE_NAME', pattr='__doc__')])
-            ])])
-
-BUILD_TUPLE_0 = AST('expr',
-                    [ AST('build_list',
-                          [ Token('BUILD_TUPLE_0') ])])
-
-NAME_MODULE = AST('stmt',
-                [ AST('assign',
-                    [ AST('expr',
-                          [Token('LOAD_NAME', pattr='__name__', offset=0, has_arg=True)]),
-                      AST('designator',
-                          [ Token('STORE_NAME', pattr='__module__', offset=3, has_arg=True)])
-                      ])])
-
-# God intended \t, but Python has decided to use 4 spaces.
-# If you want real tabs, use Go.
-# TAB = '\t'
-TAB = ' ' * 4
-INDENT_PER_LEVEL = ' ' # additional intent per pretty-print level
-
-TABLE_R = {
-    'STORE_ATTR':	( '%c.%[1]{pattr}', 0),
-#   'STORE_SUBSCR':	( '%c[%c]', 0, 1 ),
-    'DELETE_ATTR':	( '%|del %c.%[-1]{pattr}\n', 0 ),
-#   'EXEC_STMT':	( '%|exec %c in %[1]C\n', 0, (0,maxint,', ') ),
-}
-
-TABLE_R0 = {
-#    'BUILD_LIST':	( '[%C]',      (0,-1,', ') ),
-#    'BUILD_TUPLE':	( '(%C)',      (0,-1,', ') ),
-#    'CALL_FUNCTION':	( '%c(%P)', 0, (1,-1,', ') ),
-}
-TABLE_DIRECT = {
-    'BINARY_ADD':	( '+' ,),
-    'BINARY_SUBTRACT':	( '-' ,),
-    'BINARY_MULTIPLY':	( '*' ,),
-    'BINARY_DIVIDE':	( '/' ,),
-    'BINARY_MATRIX_MULTIPLY':	( '@' ,),
-    'BINARY_TRUE_DIVIDE':	( '/' ,),
-    'BINARY_FLOOR_DIVIDE':	( '//' ,),
-    'BINARY_MODULO':	( '%%',),
-    'BINARY_POWER':	( '**',),
-    'BINARY_LSHIFT':	( '<<',),
-    'BINARY_RSHIFT':	( '>>',),
-    'BINARY_AND':	( '&' ,),
-    'BINARY_OR':	( '|' ,),
-    'BINARY_XOR':	( '^' ,),
-    'INPLACE_ADD':	( '+=' ,),
-    'INPLACE_SUBTRACT':	( '-=' ,),
-    'INPLACE_MULTIPLY':	( '*=' ,),
-    'INPLACE_MATRIX_MULTIPLY':	( '@=' ,),
-    'INPLACE_DIVIDE':	( '/=' ,),
-    'INPLACE_TRUE_DIVIDE':	( '/=' ,),
-    'INPLACE_FLOOR_DIVIDE':	( '//=' ,),
-    'INPLACE_MODULO':	( '%%=',),
-    'INPLACE_POWER':	( '**=',),
-    'INPLACE_LSHIFT':	( '<<=',),
-    'INPLACE_RSHIFT':	( '>>=',),
-    'INPLACE_AND':	( '&=' ,),
-    'INPLACE_OR':	( '|=' ,),
-    'INPLACE_XOR':	( '^=' ,),
-    'binary_expr':	( '%c %c %c', 0, -1, 1 ),
-
-    'UNARY_POSITIVE':	( '+',),
-    'UNARY_NEGATIVE':	( '-',),
-    'UNARY_INVERT':	( '~%c'),
-    'unary_expr':   ( '%c%c', 1, 0),
-
-    'unary_not':	( 'not %c', 0 ),
-    'unary_convert':	( '`%c`', 0 ),
-    'get_iter':	( 'iter(%c)', 0 ),
-    'slice0':		( '%c[:]', 0 ),
-    'slice1':		( '%c[%p:]', 0, (1, 100) ),
-    'slice2':		( '%c[:%p]', 0, (1, 100) ),
-    'slice3':		( '%c[%p:%p]', 0, (1, 100), (2, 100) ),
-
-    'IMPORT_FROM':	( '%{pattr}', ),
-    'load_attr':	( '%c.%[1]{pattr}', 0),
-    'LOAD_FAST':	( '%{pattr}', ),
-    'LOAD_NAME':	( '%{pattr}', ),
-    'LOAD_CLASSNAME':	( '%{pattr}', ),
-    'LOAD_GLOBAL':	( '%{pattr}', ),
-    'LOAD_DEREF':	( '%{pattr}', ),
-    'LOAD_LOCALS':	( 'locals()', ),
-    'LOAD_ASSERT':  ( '%{pattr}', ),
-#   'LOAD_CONST':	( '%{pattr}', ),	# handled by n_LOAD_CONST
-    'DELETE_FAST':	( '%|del %{pattr}\n', ),
-    'DELETE_NAME':	( '%|del %{pattr}\n', ),
-    'DELETE_GLOBAL':	( '%|del %{pattr}\n', ),
-    'delete_subscr':	( '%|del %c[%c]\n', 0, 1,),
-    'binary_subscr':	( '%c[%p]', 0, (1, 100)),
-    'binary_subscr2':	( '%c[%p]', 0, (1, 100)),
-    'store_subscr':	( '%c[%c]', 0, 1),
-    'STORE_FAST':	( '%{pattr}', ),
-    'STORE_NAME':	( '%{pattr}', ),
-    'STORE_GLOBAL':	( '%{pattr}', ),
-    'STORE_DEREF':	( '%{pattr}', ),
-    'unpack':		( '%C%,', (1, maxint, ', ') ),
-
-    # This nonterminal we create on the fly in semantic routines
-    'unpack_w_parens':	( '(%C%,)', (1, maxint, ', ') ),
-
-    'unpack_list':	( '[%C]', (1, maxint, ', ') ),
-    'build_tuple2':	( '%P', (0, -1, ', ', 100) ),
-
-    # 'list_compr':	( '[ %c ]', -2),	# handled by n_list_compr
-    'list_iter':	( '%c', 0),
-    'list_for':		( ' for %c in %c%c', 2, 0, 3 ),
-    'list_if':		( ' if %c%c', 0, 2 ),
-    'list_if_not':		( ' if not %p%c', (0, 22), 2 ),
-    'lc_body':		( '', ),	# ignore when recusing
-
-    'comp_iter':	( '%c', 0),
-    'comp_if':		( ' if %c%c', 0, 2 ),
-    'comp_ifnot':	( ' if not %p%c', (0, 22), 2 ),
-    'comp_body':	( '', ),	# ignore when recusing
-    'set_comp_body':    ( '%c', 0 ),
-    'gen_comp_body':    ( '%c', 0 ),
-    'dict_comp_body':   ( '%c:%c', 1, 0 ),
-
-    'assign':		( '%|%c = %p\n', -1, (0, 200) ),
-
-    # The 2nd parameter should have a = suffix.
-    # There is a rule with a 4th parameter "designator"
-    # which we don't use here.
-    'augassign1':	( '%|%c %c %c\n', 0, 2, 1),
-
-    'augassign2':	( '%|%c.%[2]{pattr} %c %c\n', 0, -3, -4),
-    'designList':	( '%c = %c', 0, -1 ),
-    'and':          	( '%c and %c', 0, 2 ),
-    'ret_and':        	( '%c and %c', 0, 2 ),
-    'and2':          	( '%c', 3 ),
-    'or':           	( '%c or %c', 0, 2 ),
-    'ret_or':           	( '%c or %c', 0, 2 ),
-    'conditional':  ( '%p if %p else %p', (2, 27), (0, 27), (4, 27)),
-    'ret_cond':     ( '%p if %p else %p', (2, 27), (0, 27), (-1, 27)),
-    'conditionalnot':  ( '%p if not %p else %p', (2, 27), (0, 22), (4, 27)),
-    'ret_cond_not':  ( '%p if not %p else %p', (2, 27), (0, 22), (-1, 27)),
-    'conditional_lambda':  ( '(%c if %c else %c)', 2, 0, 3),
-    'return_lambda':    ('%c', 0),
-    'compare':		( '%p %[-1]{pattr} %p', (0, 19), (1, 19) ),
-    'cmp_list':		( '%p %p', (0, 20), (1, 19)),
-    'cmp_list1':	( '%[3]{pattr} %p %p', (0, 19), (-2, 19)),
-    'cmp_list2':	( '%[1]{pattr} %p', (0, 19)),
-#   'classdef': 	(), # handled by n_classdef()
-    'funcdef':  	( '\n\n%|def %c\n', -2), # -2 to handle closures
-    'funcdefdeco':  	( '\n\n%c', 0),
-    'mkfuncdeco':  	( '%|@%c\n%c', 0, 1),
-    'mkfuncdeco0':  	( '%|def %c\n', 0),
-    'classdefdeco':  	( '\n\n%c', 0),
-    'classdefdeco1':  	( '%|@%c\n%c', 0, 1),
-    'kwarg':    	( '%[0]{pattr}=%c', 1),
-    'kwargs':    	( '%D', (0, maxint, ', ') ),
-
-    'assert_expr_or': ( '%c or %c', 0, 2 ),
-    'assert_expr_and':    ( '%c and %c', 0, 2 ),
-    'print_items_stmt': ( '%|print %c%c,\n', 0, 2),  # Python 2 only
-    'print_items_nl_stmt': ( '%|print %c%c\n', 0, 2),
-    'print_item':  ( ', %c', 0),
-    'print_nl':	( '%|print\n', ),
-    'print_to':		( '%|print >> %c, %c,\n', 0, 1 ),
-    'print_to_nl':	( '%|print >> %c, %c\n', 0, 1 ),
-    'print_nl_to':	( '%|print >> %c\n', 0 ),
-    'print_to_items':	( '%C', (0, 2, ', ') ),
-
-    'call_stmt':	( '%|%p\n', (0, 200)),
-    'break_stmt':	( '%|break\n', ),
-    'continue_stmt':	( '%|continue\n', ),
-
-    'raise_stmt0':	( '%|raise\n', ),
-    'raise_stmt1':	( '%|raise %c\n', 0),
-    'raise_stmt3':	( '%|raise %c, %c, %c\n', 0, 1, 2),
-#    'yield':	( 'yield %c', 0),
-#    'return_stmt':	( '%|return %c\n', 0),
-
-    'ifstmt':		( '%|if %c:\n%+%c%-', 0, 1 ),
-    'iflaststmt':		( '%|if %c:\n%+%c%-', 0, 1 ),
-    'iflaststmtl':		( '%|if %c:\n%+%c%-', 0, 1 ),
-    'testtrue':     ( 'not %p', (0, 22) ),
-
-    'ifelsestmt':	( '%|if %c:\n%+%c%-%|else:\n%+%c%-', 0, 1, 3 ),
-    'ifelsestmtc':	( '%|if %c:\n%+%c%-%|else:\n%+%c%-', 0, 1, 3 ),
-    'ifelsestmtl':	( '%|if %c:\n%+%c%-%|else:\n%+%c%-', 0, 1, 3 ),
-    'ifelifstmt':	( '%|if %c:\n%+%c%-%c', 0, 1, 3 ),
-    'elifelifstmt':	( '%|elif %c:\n%+%c%-%c', 0, 1, 3 ),
-    'elifstmt':		( '%|elif %c:\n%+%c%-', 0, 1 ),
-    'elifelsestmt':	( '%|elif %c:\n%+%c%-%|else:\n%+%c%-', 0, 1, 3 ),
-    'ifelsestmtr':	( '%|if %c:\n%+%c%-%|else:\n%+%c%-', 0, 1, 2 ),
-    'elifelsestmtr':	( '%|elif %c:\n%+%c%-%|else:\n%+%c%-\n\n', 0, 1, 2 ),
-
-    'whileTruestmt':	( '%|while True:\n%+%c%-\n\n', 1 ),
-    'whilestmt':	( '%|while %c:\n%+%c%-\n\n', 1, 2 ),
-    'while1stmt':	( '%|while 1:\n%+%c%-\n\n', 1 ),
-    'while1elsestmt':  ( '%|while 1:\n%+%c%-%|else:\n%+%c%-\n\n', 1, -2 ),
-    'whileelsestmt':	( '%|while %c:\n%+%c%-%|else:\n%+%c%-\n\n', 1, 2, -2 ),
-    'whileelselaststmt':	( '%|while %c:\n%+%c%-%|else:\n%+%c%-', 1, 2, -2 ),
-    'forstmt':		( '%|for %c in %c:\n%+%c%-\n\n', 3, 1, 4 ),
-    'forelsestmt':	(
-        '%|for %c in %c:\n%+%c%-%|else:\n%+%c%-\n\n', 3, 1, 4, -2),
-    'forelselaststmt':	(
-        '%|for %c in %c:\n%+%c%-%|else:\n%+%c%-', 3, 1, 4, -2),
-    'forelselaststmtl':	(
-        '%|for %c in %c:\n%+%c%-%|else:\n%+%c%-\n\n', 3, 1, 4, -2),
-    'trystmt':		( '%|try:\n%+%c%-%c\n\n', 1, 3 ),
-    'tryelsestmt':	( '%|try:\n%+%c%-%c%|else:\n%+%c%-\n\n', 1, 3, 4 ),
-    'tryelsestmtc':	( '%|try:\n%+%c%-%c%|else:\n%+%c%-', 1, 3, 4 ),
-    'tryelsestmtl':	( '%|try:\n%+%c%-%c%|else:\n%+%c%-', 1, 3, 4 ),
-    'tf_trystmt':	( '%c%-%c%+', 1, 3 ),
-    'tf_tryelsestmt':	( '%c%-%c%|else:\n%+%c', 1, 3, 4 ),
-    'tryfinallystmt':	( '%|try:\n%+%c%-%|finally:\n%+%c%-\n\n', 1, 5 ),
-    'except':           ( '%|except:\n%+%c%-', 3 ),
-    'except_cond1':	( '%|except %c:\n', 1 ),
-    'except_suite':     ( '%+%c%-%C', 0, (1, maxint, '') ),
-    'except_suite_finalize':     ( '%+%c%-%C', 1, (3, maxint, '') ),
-    'withstmt':     ( '%|with %c:\n%+%c%-', 0, 3),
-    'withasstmt':   ( '%|with %c as %c:\n%+%c%-', 0, 2, 3),
-    'passstmt':		( '%|pass\n', ),
-    'STORE_FAST':	( '%{pattr}', ),
-    'kv':		( '%c: %c', 3, 1 ),
-    'kv2':		( '%c: %c', 1, 2 ),
-    'mapexpr':		( '{%[1]C}', (0, maxint, ', ') ),
-    'importstmt': ( '%|import %c\n', 2),
-    'importfrom': ( '%|from %[2]{pattr} import %c\n', 3 ),
-    'importstar': ( '%|from %[2]{pattr} import *\n', ),
-}
-
-
-MAP_DIRECT = (TABLE_DIRECT, )
-MAP_R0 = (TABLE_R0, -1, 0)
-MAP_R = (TABLE_R, -1)
-
-MAP = {
-    'stmt':		MAP_R,
-    'call_function':	MAP_R,
-    'del_stmt':		MAP_R,
-    'designator':	MAP_R,
-    'exprlist':		MAP_R0,
-}
-
-PRECEDENCE = {
-    'build_list':           0,
-    'mapexpr':              0,
-    'unary_convert':        0,
-    'dictcomp':             0,
-    'setcomp':              0,
-    'list_compr':           0,
-    'genexpr':              0,
-
-    'load_attr':            2,
-    'binary_subscr':        2,
-    'binary_subscr2':       2,
-    'slice0':               2,
-    'slice1':               2,
-    'slice2':               2,
-    'slice3':               2,
-    'buildslice2':          2,
-    'buildslice3':          2,
-    'call_function':        2,
-
-    'BINARY_POWER':         4,
-
-    'unary_expr':           6,
-
-    'BINARY_MULTIPLY':      8,
-    'BINARY_DIVIDE':        8,
-    'BINARY_TRUE_DIVIDE':   8,
-    'BINARY_FLOOR_DIVIDE':  8,
-    'BINARY_MODULO':        8,
-
-    'BINARY_ADD':           10,
-    'BINARY_SUBTRACT':      10,
-
-    'BINARY_LSHIFT':        12,
-    'BINARY_RSHIFT':        12,
-
-    'BINARY_AND':           14,
-
-    'BINARY_XOR':           16,
-
-    'BINARY_OR':            18,
-
-    'cmp':                  20,
-
-    'unary_not':            22,
-
-    'and':                  24,
-    'ret_and':              24,
-
-    'or':                   26,
-    'ret_or':               26,
-
-    'conditional':          28,
-    'conditionalnot':       28,
-    'ret_cond':             28,
-    'ret_cond_not':         28,
-
-    '_mklambda':            30,
-    'yield':                101,
-    'yield_from':           101
-}
-
-ASSIGN_TUPLE_PARAM = lambda param_name: \
-             AST('expr', [ Token('LOAD_FAST', pattr=param_name) ])
-
-escape = re.compile(r'''
-            (?P<prefix> [^%]* )
-            % ( \[ (?P<child> -? \d+ ) \] )?
-                ((?P<type> [^{] ) |
-                 ( [{] (?P<expr> [^}]* ) [}] ))
-        ''', re.VERBOSE)
 
 def is_docstring(node):
     try:
@@ -572,6 +241,9 @@ class SourceWalker(GenericASTTraversal, object):
                 TABLE_DIRECT.update({
                     'importmultiple': ( '%|import %c%c\n', 2, 3 ),
                     'import_cont'   : ( ', %c', 2 ),
+                    # With/as is allowed as "from future" thing in 2.5
+                    'withstmt':     ( '%|with %c:\n%+%c%-', 0, 3),
+                    'withasstmt':   ( '%|with %c as %c:\n%+%c%-', 0, 2, 3),
                 })
 
         ########################################
@@ -589,7 +261,9 @@ class SourceWalker(GenericASTTraversal, object):
             })
         else:
             TABLE_DIRECT.update({
-                'except_cond3':	( '%|except %c, %c:\n', 1, 6 ),
+                'except_cond3':	 ( '%|except %c, %c:\n', 1, 6 ),
+                'testtrue_then': ( 'not %p', (0, 22) ),
+
             })
         if 2.4 <= version <= 2.6:
             TABLE_DIRECT.update({
@@ -622,7 +296,9 @@ class SourceWalker(GenericASTTraversal, object):
                     code = node[-3]
 
                 self.indentMore()
-                annotate_last = -4 if self.version == 3.1 else -5
+                for annotate_last in range(len(node)-1, -1, -1):
+                    if node[annotate_last] == 'annotate_tuple':
+                        break
 
                 # FIXME: handle and pass full annotate args
                 make_function3_annotate(self, node, isLambda=False,
@@ -643,7 +319,42 @@ class SourceWalker(GenericASTTraversal, object):
                 TABLE_DIRECT.update({
                     'LOAD_CLASSDEREF':	( '%{pattr}', ),
                     })
+                ########################
+                # Python 3.5+ Additions
+                #######################
                 if version >= 3.5:
+                    TABLE_DIRECT.update({
+                        'await_stmt':	       ( '%|await %c', 0),
+                        'async_for_stmt':      (
+                            '%|async for %c in %c:\n%+%c%-\n\n', 9, 1, 25 ),
+                        'async_forelse_stmt':  (
+                            '%|async for %c in %c:\n%+%c%-%|else:\n%+%c%-\n\n', 9, 1, 25, 28 ),
+                        'async_with_stmt':     (
+                            '%|async with %c:\n%+%c%-', 0, 7),
+                        'async_with_as_stmt':  (
+                            '%|async with %c as %c:\n%+%c%-', 0, 6, 7),
+
+                    })
+                    def n_async_call_function(node):
+                        self.f.write('async ')
+                        node.type == 'call_function'
+                        p = self.prec
+                        self.prec = 80
+                        self.engine(('%c(%P)', 0, (1, -4, ', ', 100)), node)
+                        self.prec = p
+                        self.prune()
+                    self.n_async_call_function = n_async_call_function
+
+                    def n_funcdef(node):
+                        code_node = node[0][1]
+                        if (code_node == 'LOAD_CONST' and iscode(code_node.attr)
+                            and code_node.attr.co_flags & COMPILER_FLAG_BIT['COROUTINE']):
+                            self.engine(('\n\n%|async def %c\n', -2), node)
+                        else:
+                            self.engine(('\n\n%|def %c\n', -2), node)
+                        self.prune()
+                    self.n_funcdef = n_funcdef
+
                     def n_unmapexpr(node):
                         last_n = node[0][-1]
                         for n in node[0]:
@@ -1015,11 +726,18 @@ class SourceWalker(GenericASTTraversal, object):
     n_ifelsestmtc = n_ifelsestmtl = n_ifelsestmt
 
     def n_ifelsestmtr(self, node):
-        if len(node[2]) != 2:
+        if node[2] == 'COME_FROM':
+            return_stmts_node = node[3]
+            node.type = 'ifelsestmtr2'
+        else:
+            return_stmts_node = node[2]
+        if len(return_stmts_node) != 2:
             self.default(node)
 
-        if not (node[2][0][0][0] == 'ifstmt' and node[2][0][0][0][1][0] == 'return_if_stmts') \
-                and not (node[2][0][-1][0] == 'ifstmt' and node[2][0][-1][0][1][0] == 'return_if_stmts'):
+        if (not (return_stmts_node[0][0][0] == 'ifstmt'
+                 and return_stmts_node[0][0][0][1][0] == 'return_if_stmts')
+            and not (return_stmts_node[0][-1][0] == 'ifstmt'
+                     and return_stmts_node[0][-1][0][1][0] == 'return_if_stmts')):
             self.default(node)
             return
 
@@ -1031,13 +749,14 @@ class SourceWalker(GenericASTTraversal, object):
         self.indentLess()
 
         if_ret_at_end = False
-        if len(node[2][0]) >= 3:
-            if node[2][0][-1][0] == 'ifstmt' and node[2][0][-1][0][1][0] == 'return_if_stmts':
+        if len(return_stmts_node[0]) >= 3:
+            if (return_stmts_node[0][-1][0] == 'ifstmt'
+                and return_stmts_node[0][-1][0][1][0] == 'return_if_stmts'):
                 if_ret_at_end = True
 
         past_else = False
         prev_stmt_is_if_ret = True
-        for n in node[2][0]:
+        for n in return_stmts_node[0]:
             if (n[0] == 'ifstmt' and n[0][1][0] == 'return_if_stmts'):
                 if prev_stmt_is_if_ret:
                     n[0].type = 'elifstmt'
@@ -1052,15 +771,22 @@ class SourceWalker(GenericASTTraversal, object):
         if not past_else or if_ret_at_end:
             self.println(self.indent, 'else:')
             self.indentMore()
-        self.preorder(node[2][1])
+        self.preorder(return_stmts_node[1])
         self.indentLess()
         self.prune()
+    n_ifelsestmtr2 = n_ifelsestmtr
 
     def n_elifelsestmtr(self, node):
-        if len(node[2]) != 2:
+        if node[2] == 'COME_FROM':
+            return_stmts_node = node[3]
+            node.type = 'elifelsestmtr2'
+        else:
+            return_stmts_node = node[2]
+
+        if len(return_stmts_node) != 2:
             self.default(node)
 
-        for n in node[2][0]:
+        for n in return_stmts_node[0]:
             if not (n[0] == 'ifstmt' and n[0][1][0] == 'return_if_stmts'):
                 self.default(node)
                 return
@@ -1072,12 +798,12 @@ class SourceWalker(GenericASTTraversal, object):
         self.preorder(node[1])
         self.indentLess()
 
-        for n in node[2][0]:
+        for n in return_stmts_node[0]:
             n[0].type = 'elifstmt'
             self.preorder(n)
         self.println(self.indent, 'else:')
         self.indentMore()
-        self.preorder(node[2][1])
+        self.preorder(return_stmts_node[1])
         self.indentLess()
         self.prune()
 
@@ -1718,6 +1444,17 @@ class SourceWalker(GenericASTTraversal, object):
                     i += 3
                     pass
                 pass
+            elif node[-1].type.startswith('BUILD_CONST_KEY_MAP'):
+                # Python 3.6+ style const map
+                keys = node[-2].pattr
+                values = node[:-2]
+                # FIXME: Line numbers?
+                for key, value in zip(keys, values):
+                    self.write(repr(key))
+                    self.write(':')
+                    self.write(self.traverse(value[0]))
+                    self.write(',')
+                pass
             pass
         else:
             # Python 2 style kvlist
@@ -1785,23 +1522,29 @@ class SourceWalker(GenericASTTraversal, object):
         self.prec = 100
         lastnode = node.pop()
         lastnodetype = lastnode.type
+
+        # If this build list is inside a CALL_FUNCTION_VAR,
+        # then the first * has already been printed.
+        # Until I have a better way to check for CALL_FUNCTION_VAR,
+        # will assume that if the text ends in *.
+        last_was_star = self.f.getvalue().endswith('*')
+
         have_star = False
-        if lastnodetype.startswith('BUILD_LIST'):
-            # 3.5+ has BUILD_LIST_UNPACK
-            if lastnodetype == 'BUILD_LIST_UNPACK':
-                # FIXME: need to handle range of BUILD_LIST_UNPACK
-                have_star = True
-                endchar = ''
-            else:
-                self.write('['); endchar = ']'
-        elif lastnodetype.startswith('BUILD_TUPLE'):
-            self.write('('); endchar = ')'
-        elif lastnodetype.startswith('BUILD_SET'):
-            self.write('{'); endchar = '}'
-        elif lastnodetype.startswith('ROT_TWO'):
-            self.write('('); endchar = ')'
+        if lastnodetype.endswith('UNPACK'):
+            # FIXME: need to handle range of BUILD_LIST_UNPACK
+            have_star = True
+            endchar = ''
         else:
-            raise 'Internal Error: n_build_list expects list or tuple'
+            if lastnodetype.startswith('BUILD_LIST'):
+                self.write('['); endchar = ']'
+            elif lastnodetype.startswith('BUILD_TUPLE'):
+                self.write('('); endchar = ')'
+            elif lastnodetype.startswith('BUILD_SET'):
+                self.write('{'); endchar = '}'
+            elif lastnodetype.startswith('ROT_TWO'):
+                self.write('('); endchar = ')'
+            else:
+                raise 'Internal Error: n_build_list expects list or tuple'
 
         flat_elems = []
         for elem in node:
@@ -1828,8 +1571,13 @@ class SourceWalker(GenericASTTraversal, object):
                 sep += '\n' + self.indent + INDENT_PER_LEVEL[:-1]
             else:
                 if sep != '': sep += ' '
-            if have_star:
-                sep += '*'
+            if not last_was_star:
+                if have_star:
+                    sep += '*'
+                    pass
+                pass
+            else:
+                last_was_star = False
             self.write(sep, value)
             sep = ','
         if lastnode.attr == 1 and lastnodetype.startswith('BUILD_TUPLE'):
@@ -1891,8 +1639,7 @@ class SourceWalker(GenericASTTraversal, object):
         beginning of this module for the how we interpret format specifications such as
         %c, %C, and so on.
         """
-
-        # self.println("----> ", startnode.type)
+        # self.println("----> ", startnode.type, ', ', entry[0])
         fmt = entry[0]
         arg = 1
         i = 0

uncompyle6/verify.py

@@ -201,13 +201,12 @@ def cmp_code_objects(version, is_pypy, code_obj1, code_obj2,
                 else:
                     import uncompyle6.scanners.scanner27 as scan
                     scanner = scan.Scanner27()
+            elif version == 3.0:
+                import uncompyle6.scanners.scanner30 as scan
+                scanner = scan.Scanner30()
             elif version == 3.1:
-                if is_pypy:
-                    import uncompyle6.scanners.pypy31 as scan
-                    scanner = scan.ScannerPyPy31()
-                else:
-                    import uncompyle6.scanners.scanner32 as scan
-                    scanner = scan.Scanner32()
+                import uncompyle6.scanners.scanner32 as scan
+                scanner = scan.Scanner32()
             elif version == 3.2:
                 if is_pypy:
                     import uncompyle6.scanners.pypy32 as scan
@@ -319,8 +318,14 @@ def cmp_code_objects(version, is_pypy, code_obj1, code_obj2,
                     elif tokens1[i1].type == 'LOAD_NAME' and tokens2[i2].type == 'LOAD_CONST' \
                          and tokens1[i1].pattr == 'None' and tokens2[i2].pattr is None:
                         pass
-                    elif tokens1[i1].type == 'RETURN_VALUE' and \
-                         tokens2[i2].type == 'RETURN_END_IF':
+                    elif tokens1[i1].type == 'LOAD_GLOBAL' and tokens2[i2].type == 'LOAD_NAME' \
+                         and tokens1[i1].pattr == tokens2[i2].pattr:
+                        pass
+                    elif (tokens1[i1].type == 'RETURN_VALUE' and
+                          tokens2[i2].type == 'RETURN_END_IF'):
+                        pass
+                    elif (tokens1[i1].type == 'BUILD_TUPLE_0' and
+                          tokens2[i2].pattr == ()):
                         pass
                     else:
                         raise CmpErrorCode(name, tokens1[i1].offset, tokens1[i1],
@@ -407,7 +412,8 @@ def compare_code_with_srcfile(pyc_filename, src_filename, weak_verify=False):
     try:
         code_obj2 = load_file(src_filename)
     except SyntaxError as e:
-        return str(e)
+        # src_filename can be the first of a group sometimes
+        return str(e).replace(src_filename, pyc_filename)
     cmp_code_objects(version, is_pypy, code_obj1, code_obj2, ignore_code=weak_verify)
     return None
 

uncompyle6/version.py

@@ -1,3 +1,3 @@
 # This file is suitable for sourcing inside bash as
 # well as importing into Python
-VERSION='2.9.8'
+VERSION='2.9.10'