Merge branch 'master' of github.com:rocky/python-uncompyle6

Revert "fix bug : not generate all files when use "-ro""

.travis.yml

@@ -8,6 +8,7 @@ python:
   - '2.6'
   - '3.3'
   - '3.4'
+  - '3.2'
 
 install:
 - pip install -r requirements.txt

ChangeLog

@@ -1,6 +1,539 @@
+2017-01-11  rocky <rb@dustyfeet.com>
+
+	* uncompyle6/version.py: Get ready for release 2.10.9
+
+2017-01-11  R. Bernstein <rocky@users.noreply.github.com>
+
+	* : Merge pull request #79 from rocky/revert-78-patch-1 Revert "fix bug : not generate all files when use "-ro""
+
+2017-01-11  R. Bernstein <rocky@users.noreply.github.com>
+
+	* : Merge pull request #78 from jlugjb/patch-1 fix bug : not generate all files when use "-ro"
+
+2017-01-10  rocky <rb@dustyfeet.com>
+
+	* test/simple_source/bug35/03_double_star_unpack.py,
+	uncompyle6/parsers/parse3.py, uncompyle6/semantics/pysource.py: 
+	Improve BUILD_xxx_UNPACK slightly
+
+2017-01-09  rocky <rb@dustyfeet.com>
+
+	* test/simple_source/bug35/03_async_await.py,
+	uncompyle6/parsers/parse3.py, uncompyle6/semantics/pysource.py: Add
+	async_call_function for 3.5+
+
+2017-01-09  rocky <rb@dustyfeet.com>
+
+	* : Reinstate test
+
+2017-01-08  rocky <rb@dustyfeet.com>
+
+	* : Works now
+
+2017-01-08  rocky <rb@dustyfeet.com>
+
+	* uncompyle6/parsers/parse30.py, uncompyle6/scanners/scanner3.py: 
+	Python 3.0 decompile bugs
+
+2017-01-08  rocky <rb@dustyfeet.com>
+
+	* uncompyle6/scanners/scanner2.py, uncompyle6/scanners/scanner3.py,
+	uncompyle6/scanners/scanner30.py: Towards better 3.0 decompilation Sync scanner2 and scanner3 better
+
+2017-01-08  rocky <rb@dustyfeet.com>
+
+	* test/simple_source/bug35/03_while-if-break.py,
+	uncompyle6/parsers/parse35.py, uncompyle6/scanner.py,
+	uncompyle6/scanners/scanner3.py: Fix 3.5, 3.6 while true if/break
+	bug
+
+2017-01-08  rocky <rb@dustyfeet.com>
+
+	* uncompyle6/__init__.py, uncompyle6/main.py,
+	uncompyle6/semantics/consts.py, uncompyle6/semantics/fragments.py,
+	uncompyle6/semantics/pysource.py: Misc cleanups Favor "decompile" over "uncompyle" since "decompile" is in common
+	use Reduce size of pysource.py by splitting out constants
+
+2017-01-08  rocky <rb@dustyfeet.com>
+
+	* test/simple_source/bug35/03_async_await.py,
+	uncompyle6/parsers/parse35.py, uncompyle6/scanners/scanner3.py,
+	uncompyle6/semantics/pysource.py: Add 3.5+ async with/for ..  scanner3.py: 3.6 bytecode vs wordcode fix
+
+2017-01-07  rocky <rb@dustyfeet.com>
+
+	* test/simple_source/bug35/03_async_await.py,
+	uncompyle6/parsers/parse35.py, uncompyle6/semantics/pysource.py: 
+	Start to add 3.5+ await and async
+
+2017-01-07  rocky <rb@dustyfeet.com>
+
+	* test/simple_source/bug31/04_def_annotate.py,
+	uncompyle6/semantics/make_function.py: More Python 3 annotation bugs
+
+2017-01-07  rocky <rb@dustyfeet.com>
+
+	* uncompyle6/parsers/parse3.py, uncompyle6/parsers/parse31.py,
+	uncompyle6/parsers/parse32.py,
+	uncompyle6/semantics/make_function.py,
+	uncompyle6/semantics/pysource.py: Fix some errors in deparsing
+	Python 3 annotations
+
+2017-01-07  rocky <rb@dustyfeet.com>
+
+	* uncompyle6/semantics/make_function.py: Small Pyhton 3.x annotate
+	bug
+
+2017-01-03  rocky <rb@dustyfeet.com>
+
+	* README.rst: Note what's up with Python 3 decompile quality
+
+2017-01-03  rocky <rb@dustyfeet.com>
+
+	* uncompyle6/scanners/scanner3.py: 3.5 continue check is needed on
+	3.6
+
+2017-01-03  rocky <rb@dustyfeet.com>
+
+	* test/test_pyenvlib.py, uncompyle6/parsers/parse36.py,
+	uncompyle6/scanners/scanner3.py: Towards better 3.6 support
+
+2017-01-02  rocky <rb@dustyfeet.com>
+
+	* uncompyle6/parsers/parse35.py, uncompyle6/scanners/scanner3.py: 
+	Python 3.5 continue detection bug
+
+2017-01-01  rocky <rb@dustyfeet.com>
+
+	* uncompyle6/scanners/scanner3.py: add come_from for setup_finally
+	and setup_except
+
+2017-01-01  rocky <rb@dustyfeet.com>
+
+	* uncompyle6/parsers/parse35.py, uncompyle6/scanners/scanner3.py: 
+	Towards fixing Python 3.5 return bugs
+
+2017-01-01  rocky <rb@dustyfeet.com>
+
+	* README.rst: Note how to verify correctness ...  with --verify, --weak-verify and cross checking with pycdc
+
+2016-12-31  rocky <rb@dustyfeet.com>
+
+	* ChangeLog, NEWS, uncompyle6/version.py: Get ready for release
+	2.9.9
+
+2016-12-31  rocky <rb@dustyfeet.com>
+
+	* uncompyle6/parsers/parse26.py: 2.x list_if may have a THEN in it
+
+2016-12-31  rocky <rb@dustyfeet.com>
+
+	* uncompyle6/scanners/scanner3.py: Towards fixing a Python 3.3
+	return/continue bug
+
+2016-12-30  rocky <rb@dustyfeet.com>
+
+	* uncompyle6/main.py: On --verify if we can't unbuffer output, don't
+
+2016-12-29  rocky <rb@dustyfeet.com>
+
+	* uncompyle6/scanners/scanner2.py, uncompyle6/scanners/scanner3.py: 
+	dectect_structure() -> detect_control_flow()
+
+2016-12-29  rocky <rb@dustyfeet.com>
+
+	* uncompyle6/scanners/scanner2.py, uncompyle6/scanners/scanner3.py: 
+	DRY code and emitted Python 3 source * Python 3: break; continue -> break * Use variable in detect_structure for pre[rtarget] * Make Python 2 and Python 3 detect_structure more alie
+
+2016-12-29  rocky <rb@dustyfeet.com>
+
+	* uncompyle6/scanners/scanner3.py: More if/then detection in Python
+	3.x
+
+2016-12-29  R. Bernstein <rocky@users.noreply.github.com>
+
+	* : Merge pull request #73 from rocky/then-crap Add THEN token to improve Python 2.2-2.6 control flow detection
+
+2016-12-28  rocky <rb@dustyfeet.com>
+
+	* uncompyle6/parsers/parse3.py, uncompyle6/scanners/tok.py: Misc
+	bugs
+
+2016-12-28  rocky <rb@dustyfeet.com>
+
+	* : commit 723fa5dfed5bb198c66741c594e2c277ded88970 Author: rocky
+	<rb@dustyfeet.com> Date:   Wed Dec 28 18:57:09 2016 -0500
+
+2016-12-28  rocky <rb@dustyfeet.com>
+
+	* uncompyle6/parsers/parse3.py, uncompyle6/parsers/parse32.py,
+	uncompyle6/parsers/parse33.py: Towards fixing a 3.2 while true: ...
+	break bug
+
+2016-12-28  rocky <rb@dustyfeet.com>
+
+	* test/Makefile, uncompyle6/main.py, uncompyle6/parsers/parse26.py,
+	uncompyle6/verify.py: Bugs in Python 2.6- "and" and "lambda"
+	handling ..  and clean up verify output
+
+2016-12-27  rocky <rb@dustyfeet.com>
+
+	* uncompyle6/parsers/parse26.py, uncompyle6/scanners/scanner2.py,
+	uncompyle6/scanners/scanner26.py, uncompyle6/semantics/pysource.py: 
+	WIP : Add THEN to disambigute from "and"
+
+2016-12-27  rocky <rb@dustyfeet.com>
+
+	* uncompyle6/scanners/scanner2.py: Make 2.6 and 2.7 ingest more
+	alike
+
+2016-12-26  rocky <rb@dustyfeet.com>
+
+	* : Update 2.7 bytecode file for last fix
+
+2016-12-26  R. Bernstein <rocky@users.noreply.github.com>
+
+	* : Merge pull request #71 from jiangpengcheng/tupple_bug tupples which contain only 1 element need a comma
+
+2016-12-26  jiangpch <jiangpch@gohighsec.com>
+
+	* uncompyle6/semantics/pysource.py: tupples which contain only 1
+	element need a comma
+
+2016-12-26  rocky <rb@dustyfeet.com>
+
+	* uncompyle6/parsers/parse25.py: fix bug in using python2 AST rules
+	in python 2.5
+
+2016-12-26  rocky <rb@dustyfeet.com>
+
+	* : commit f1a947f106b231fb1480ba301b15e3ceaf78c94f Author: rocky
+	<rb@dustyfeet.com> Date:   Mon Dec 26 00:43:02 2016 -0500
+
+2016-12-25  rocky <rb@dustyfeet.com>
+
+	* uncompyle6/scanners/scanner23.py,
+	uncompyle6/scanners/scanner24.py, uncompyle6/semantics/pysource.py,
+	uncompyle6/verify.py: Scanner call fixes. NAME_MODULE removal for
+	<=2.4
+
+2016-12-24  rocky <rb@dustyfeet.com>
+
+	* uncompyle6/parsers/astnode.py, uncompyle6/parsers/parse2.py,
+	uncompyle6/parsers/parse26.py, uncompyle6/parsers/parse3.py,
+	uncompyle6/parsers/parse36.py, uncompyle6/scanners/scanner15.py,
+	uncompyle6/scanners/scanner2.py, uncompyle6/scanners/scanner21.py,
+	uncompyle6/scanners/scanner22.py,
+	uncompyle6/semantics/fragments.py, uncompyle6/semantics/pysource.py: 
+	Lint
+
+2016-12-24  rocky <rb@dustyfeet.com>
+
+	* uncompyle6/semantics/pysource.py: Remove stray debug hook
+
+2016-12-20  rocky <rb@dustyfeet.com>
+
+	* uncompyle6/semantics/pysource.py: Bang on 3.6
+	build_map_unpack_with_call Probably will fix better in the future.
+
+2016-12-18  rocky <rb@dustyfeet.com>
+
+	* uncompyle6/bin/pydisassemble.py, uncompyle6/bin/uncompile.py,
+	uncompyle6/parsers/parse2.py, uncompyle6/parsers/parse25.py,
+	uncompyle6/parsers/parse27.py, uncompyle6/parsers/parse3.py,
+	uncompyle6/scanners/scanner2.py, uncompyle6/scanners/scanner3.py: 
+	Python flake8 crap Was testing realgud's C-x!8 (goto flake8 warning/error)
+
+2016-12-18  rocky <rb@dustyfeet.com>
+
+	* pytest/.gitignore, test/simple_source/bug25/02_try_else.py,
+	uncompyle6/parsers/parse25.py, uncompyle6/parsers/parse26.py: Python
+	2.5 mistaken try/else
+
+2016-12-17  rocky <rb@dustyfeet.com>
+
+	* uncompyle6/scanners/scanner2.py, uncompyle6/scanners/scanner25.py: 
+	show-asm on python2.5 is optional make scanner2 look a little more like scanner3
+
+2016-12-16  rocky <rb@dustyfeet.com>
+
+	* NEWS: Release 2.9.8 news
+
+2016-12-16  rocky <rb@dustyfeet.com>
+
+	* __pkginfo__.py, uncompyle6/version.py: Get ready for release 2.9.8
+
+2016-12-16  rocky <rb@dustyfeet.com>
+
+	* test/simple_source/bug35/02_build_map_unpack_with_call.py,
+	uncompyle6/parsers/parse3.py, uncompyle6/parsers/parse35.py,
+	uncompyle6/parsers/parse36.py, uncompyle6/scanners/scanner3.py,
+	uncompyle6/semantics/pysource.py: Start to handle 3.5
+	build_map_unpack_with_call 3.6 also started but needs even more work
+
+2016-12-15  rocky <rb@dustyfeet.com>
+
+	* uncompyle6/scanner.py, uncompyle6/scanners/scanner3.py: Some
+	Python 3.6 bytecode->wordcode fixes
+
+2016-12-13  rocky <rb@dustyfeet.com>
+
+	* uncompyle6/semantics/fragments.py: Was passing wrong type
+
+2016-12-11  rocky <rb@dustyfeet.com>
+
+	* uncompyle6/parser.py: option -g: show start-end range when
+	possible
+
+2016-12-11  rocky <rb@dustyfeet.com>
+
+	* uncompyle6/semantics/make_function.py, uncompyle6/verify.py: two
+	misc changes - track print_docstring move to help (used in python 3.1) - verify: allow RETURN_VALUE to match RETURN_END_IF
+
+2016-12-10  rocky <rb@dustyfeet.com>
+
+	* .travis.yml, test/Makefile: 3.2 needs --weak-verify
+
+2016-12-10  rocky <rb@dustyfeet.com>
+
+	* .travis.yml: Try testing on 3.2
+
+2016-12-10  rocky <rb@dustyfeet.com>
+
+	* __pkginfo__.py, uncompyle6/bin/uncompile.py: Can run in Python 3.1
+	and Python 3.2
+
+2016-12-10  rocky <rb@dustyfeet.com>
+
+	* test/Makefile, uncompyle6/parsers/parse3.py,
+	uncompyle6/scanners/scanner3.py: Another python 3 ELSE fixes and ...  Makefile:   - test python 3.0 bytecode   - turn full --verify back on Python 3.x
+
+2016-12-10  rocky <rb@dustyfeet.com>
+
+	* uncompyle6/scanners/scanner3.py: Another faulty Python3 ELSE tag
+	remove
+
+2016-12-09  rocky <rb@dustyfeet.com>
+
+	* pytest/test_grammar.py: Grammar check: ELSE on RHS is ok.
+
+2016-12-09  rocky <rb@dustyfeet.com>
+
+	* uncompyle6/scanners/tok.py, uncompyle6/verify.py: Back of some of
+	the verification changes
+
+2016-12-09  rocky <rb@dustyfeet.com>
+
+	* : commit a5d2237435ee51e681c73db9e7ea379d56456205 Author: rocky
+	<rb@dustyfeet.com> Date:   Fri Dec 9 21:10:10 2016 -0500
+
+2016-12-04  rocky <rb@dustyfeet.com>
+
+	* ChangeLog, NEWS, uncompyle6/main.py, uncompyle6/parser.py,
+	uncompyle6/parsers/parse26.py, uncompyle6/parsers/parse3.py,
+	uncompyle6/parsers/parse34.py, uncompyle6/parsers/parse35.py,
+	uncompyle6/scanner.py, uncompyle6/scanners/scanner2.py,
+	uncompyle6/scanners/scanner23.py, uncompyle6/scanners/scanner24.py,
+	uncompyle6/scanners/scanner3.py, uncompyle6/scanners/tok.py,
+	uncompyle6/semantics/make_function.py,
+	uncompyle6/semantics/pysource.py, uncompyle6/verify.py,
+	uncompyle6/version.py: Get ready for release 2.9.7 Some of the many lint things. Linting is kind of stupid though.
+
+2016-11-28  rocky <rb@dustyfeet.com>
+
+	* uncompyle6/parsers/parse3.py, uncompyle6/parsers/parse36.py: 
+	Shorten Python3 grammars with + and *
+
+2016-11-28  rocky <rb@dustyfeet.com>
+
+	* __pkginfo__.py, uncompyle6/parser.py,
+	uncompyle6/parsers/parse2.py: Try new spark 2.5.1 grammar syntax
+	shortcuts This package I now declare stable
+
+2016-11-28  R. Bernstein <rocky@users.noreply.github.com>
+
+	* README.rst: Update README.rst
+
+2016-11-27  rocky <rb@dustyfeet.com>
+
+	* README.rst: Limitations of decompiling control structures.
+
+2016-11-27  R. Bernstein <rocky@users.noreply.github.com>
+
+	* : Merge pull request #69 from rocky/ast-reduce-checks AST reduce checks
+
+2016-11-26  rocky <rb@dustyfeet.com>
+
+	* test/simple_source/bug26/03_elif_vs_continue.py,
+	uncompyle6/main.py, uncompyle6/parser.py,
+	uncompyle6/parsers/parse2.py, uncompyle6/scanners/scanner2.py,
+	uncompyle6/scanners/scanner26.py: Misc changes scanner26.py: make scanner2.py and scanner26.py more alike
+	scanner2.py: check that return stmt is last in list. (May change)
+	main.py: show filename on verify error test/*: add more
+
+2016-11-25  rocky <rb@dustyfeet.com>
+
+	* __pkginfo__.py, test/Makefile, uncompyle6/parser.py,
+	uncompyle6/parsers/parse2.py, uncompyle6/parsers/parse3.py: Start
+	grammar reduction checks
+
+2016-11-24  rocky <rb@dustyfeet.com>
+
+	* uncompyle6/parsers/parse27.py, uncompyle6/scanners/scanner2.py,
+	uncompyle6/semantics/helper.py, uncompyle6/semantics/pysource.py: 
+	2.7 grammar bug workaround. Fix docstring bug
+
+2016-11-24  rocky <rb@dustyfeet.com>
+
+	* uncompyle6/semantics/pysource.py: Better line number tracking Indent Python 2 list comprehensions, albeit badly.  DRY code a
+	little via indent_if_source_nl
+
+2016-11-24  rocky <rb@dustyfeet.com>
+
+	* uncompyle6/parsers/parse3.py, uncompyle6/scanners/scanner2.py: 
+	<2.7 "if" detection and dup Python 3 grammar rule
+
+2016-11-23  rocky <rb@dustyfeet.com>
+
+	* __pkginfo__.py, pytest/test_grammar.py, uncompyle6/parser.py,
+	uncompyle6/parsers/parse26.py: Python 2.6 grammary bug and..  __pkginfo.py__: Bump spark_parser version for parse_flags 'dups'
+
+2016-11-23  rocky <rb@dustyfeet.com>
+
+	* __pkginfo__.py: Note that we now work on 2.4 and 2.5
+
+2016-11-23  rocky <rb@dustyfeet.com>
+
+	* : commit 6aa1531972de83ecab15b4c96b89c873ea5a7458 Author: rocky
+	<rb@dustyfeet.com> Date:   Wed Nov 23 00:48:38 2016 -0500
+
+2016-11-22  rocky <rb@dustyfeet.com>
+
+	* uncompyle6/parsers/parse3.py, uncompyle6/parsers/parse32.py,
+	uncompyle6/parsers/parse33.py, uncompyle6/parsers/parse34.py,
+	uncompyle6/parsers/parse35.py: DRY Python3 grammar
+
+2016-11-22  rocky <rb@dustyfeet.com>
+
+	* uncompyle6/parsers/parse2.py, uncompyle6/parsers/parse27.py,
+	uncompyle6/scanners/scanner2.py: More detailed COME_FROMs For now we only add COME_FROM_FINALLY and COME_FROM_WITH and even
+	here only on 2.7
+
+2016-11-22  rocky <rb@dustyfeet.com>
+
+	* circle.yml, pytest/test_grammar.py, tox.ini,
+	uncompyle6/parser.py, uncompyle6/parsers/parse2.py,
+	uncompyle6/parsers/parse27.py: Remove redundant 2.7 (and 2.x)
+	grammar rules
+
+2016-11-22  rocky <rb@dustyfeet.com>
+
+	* pytest/test_docstring.py, uncompyle6/linenumbers.py,
+	uncompyle6/semantics/fragments.py, uncompyle6/semantics/helper.py,
+	uncompyle6/semantics/make_function.py,
+	uncompyle6/semantics/pysource.py: Split out print_docstring move from pysource.py to new helper.py
+
+2016-11-20  rocky <rb@dustyfeet.com>
+
+	* ChangeLog, NEWS, uncompyle6/version.py: Get ready for release
+	2.9.6
+
+2016-11-20  R. Bernstein <rocky@users.noreply.github.com>
+
+	* : Merge pull request #68 from rocky/line-mappings Line mappings
+
+2016-11-20  rocky <rb@dustyfeet.com>
+
+	* : Merge remote-tracking branch 'origin' into line-mappings
+
+2016-11-20  rocky <rb@dustyfeet.com>
+
+	* uncompyle6/parsers/parse27.py: Back off a test.  That means bugs in 2.7 still not fixed. Sigh.
+
+2016-11-20  rocky <rb@dustyfeet.com>
+
+	* pytest/test_fjt.py, uncompyle6/parsers/parse27.py,
+	uncompyle6/scanners/scanner2.py: more 2.7 control flow bug fixing
+
+2016-11-20  rocky <rb@dustyfeet.com>
+
+	* uncompyle6/scanners/scanner26.py: Pass debug in scanner26
+	find_targets
+
+2016-11-20  rocky <rb@dustyfeet.com>
+
+	* uncompyle6/scanners/scanner3.py: Add debug option on Python 3
+	find_jump_targets()
+
+2016-11-20  rocky <rb@dustyfeet.com>
+
+	* uncompyle6/semantics/pysource.py: A little closesr in PyPy 2.7
+	list comprehensions pysource.py: note need to handle line breaks in list comprehensions
+
+2016-11-20  rocky <rb@dustyfeet.com>
+
+	* pytest/test_fjt.py, uncompyle6/scanners/scanner2.py,
+	uncompyle6/scanners/scanner26.py, uncompyle6/scanners/scanner3.py: 
+	Start to improve detect_structure for 2.7 and 2.x Add debug flag to find_jump_targets to show the structure we found.
+	When there are control-flow bugs, it's often reflected here.  scanner3.py: make code make more similar to 2.x code
+
+2016-11-18  rocky <rb@dustyfeet.com>
+
+	* : commit d7f898b4fbf79d1f66eabadb25f0f9f0f38730cb Author: rocky
+	<rb@dustyfeet.com> Date:   Fri Nov 18 09:02:00 2016 -0500
+
+2016-11-17  R. Bernstein <rocky@users.noreply.github.com>
+
+	* : Merge pull request #67 from rocky/2.6-cf-ignore-if 2.6 cf ignore if
+
+2016-11-16  rocky <rb@dustyfeet.com>
+
+	* test/simple_source/bug26/03_if_vs_and.py, uncompyle6/main.py,
+	uncompyle6/semantics/check_ast.py, uncompyle6/semantics/pysource.py: 
+	More AST checking Small fixes in output format
+
+2016-11-15  rocky <rb@dustyfeet.com>
+
+	* uncompyle6/parsers/parse23.py, uncompyle6/parsers/parse26.py,
+	uncompyle6/scanners/scanner2.py: WIP Grammar changes - reinstatng
+	COME_FROMs around ignore_if's
+
+2016-11-14  rocky <rb@dustyfeet.com>
+
+	* MANIFEST.in: Revise MANIFEST.in with what we have
+
+2016-11-15  rocky <rb@dustyfeet.com>
+
+	* : commit 0f719d41fdf08d41de594abb1664ab42ff92bbdf Author: rocky
+	<rb@dustyfeet.com> Date:   Mon Nov 14 20:20:07 2016 -0500
+
+2016-11-14  rocky <rb@dustyfeet.com>
+
+	* uncompyle6/parsers/parse26.py, uncompyle6/scanners/scanner2.py: 
+	WIP remove COME_FROMs around ignore_if's
+
+2016-11-14  rocky <rb@dustyfeet.com>
+
+	* uncompyle6/parsers/parse26.py, uncompyle6/scanners/scanner2.py: 
+	WIP remove COME_FROMs around ignore_if's
+
+2016-11-14  rocky <rb@dustyfeet.com>
+
+	* uncompyle6/scanners/scanner2.py, uncompyle6/scanners/scanner26.py: 
+	Show line numbers in 2.6 "after" asm ..  start to understand some of the Python 2.6 bytecode parse failures.
+
 2016-11-13  rocky <rb@dustyfeet.com>
 
-	* uncompyle6/version.py: Get ready for release 2.9.5
+	* README.rst, uncompyle6/verify.py: Handle verify syntax errors...  Update README.rst stats
+
+2016-11-13  rocky <rb@dustyfeet.com>
+
+	* setup.py: Administrivia: Fixes #66
+
+2016-11-13  rocky <rb@dustyfeet.com>
+
+	* ChangeLog, NEWS, uncompyle6/version.py: Get ready for release
+	2.9.5
 
 2016-11-13  rocky <rb@dustyfeet.com>
 

MANIFEST.in

@@ -2,10 +2,16 @@ include README.rst
 include ChangeLog
 include HISTORY.md
 include LICENSE
+include Makefile
+include requirements.txt
+include requirements-dev.txt
 include DECOMPYLE-2.4-CHANGELOG.txt
 include __pkginfo__.py
 recursive-include uncompyle6 *.py
 include bin/uncompyle6
 include bin/pydisassemble
+include pytest/Makefile
+include test/Makefile
 recursive-include test *.py *.pyc
 recursive-include pytest *.py
+recursive-include pytest/testdata *

NEWS

@@ -1,3 +1,61 @@
+uncompyle6 2.9.9 2016-12-16
+
+- Remaining Python 3.5 ops handled
+  (this also means more Python 3.6 ops are handled)
+- Python 3.5 and 3.6 async and await handled
+- Python 3.0 decompilation improved
+- Python 3 annotations fixed
+- Better control-flow detection
+- Code cleanups and misc bug fixes
+
+uncompyle6 2.9.8 2016-12-16
+
+- Better control-flow detection
+- pseudo instruction THEN in 2.x
+  to disambiguate if from and
+- fix bug in --verify option
+- DRY (a little) control-flow detection
+- fix syntax in tuples with one element
+- if AST rule inheritence in Python 2.5
+- NAME_MODULE removal for Python <= 2.4
+- verifycall fixes for Python <= 2.4
+- more Python lint
+
+uncompyle6 2.9.7 2016-12-16
+
+- Start to handle 3.5/3.6 build_map_unpack_with_call
+- Some Python 3.6 bytecode to wordcode conversion fixes
+- option -g: show start-end range when possible
+- track print_docstring move to help (used in python 3.1)
+- verify: allow RETURN_VALUE to match RETURN_END_IF
+- some 3.2 compatibility
+- Better Python 3 control flow detection by adding Pseudo ELSE opcodes
+
+uncompyle6 2.9.6 2016-12-04
+
+- Shorten Python3 grammars with + and *
+  this requires spark parser 1.5.1
+- Add some AST reduction checks to improve
+  decompile accuracy. This too requires
+  spark parser 1.5.1
+
+uncompyle6 2.9.6 2016-11-20
+
+- Correct MANIFEST.in
+- More AST grammar checking
+- --linemapping option or linenumbers.line_number_mapping()
+  Shows correspondence of lines between source
+  and decompiled source
+- Some control flow adjustments in code for 2.x.
+  This is probably an improvement in 2.6 and before.
+  For 2.7 things are just shuffled around a little. Sigh.
+  Overall I think we are getting more precise in
+  or analysis even if it is not always reflected
+  in the results.
+- better control flow debugging output
+- Python 2 and 3 detect structure code is more similar
+- Handle Docstrings with embedded tiple quotes (""")
+
 uncompyle6 2.9.5 2016-11-13
 
 - Fix Python 3 bugs:

README.rst

@@ -44,8 +44,9 @@ Requirements
 ------------
 
 This project requires Python 2.6 or later, PyPy 3-2.4, or PyPy-5.0.1.
+Python versions 2.4-2.7 are supported in the python-2.4 branch.
 The bytecode files it can read has been tested on Python bytecodes from
-versions 2.1-2.7, and 3.2-3.6 and the above-mentioned PyPy versions.
+versions 1.5, 2.1-2.7, and 3.2-3.6 and the above-mentioned PyPy versions.
 
 Installation
 ------------
@@ -92,39 +93,58 @@ For usage help:
 
    $ uncompyle6 -h
 
+If you want strong verification of the correctness of the
+decompilation process, add the `--verify` option. But there are
+situations where this will indicate a failure, although the generated
+program is semantically equivalent. Using option `--weak-verify` will
+tell you if there is something definitely wrong. Generally, large
+swaths of code are decompiled correctly, if not the entire program.
+
+You can also cross compare the results with pycdc_ . Since they work
+differently, bugs here often aren't in that, and vice versa.
+
 
 Known Bugs/Restrictions
 -----------------------
 
 The biggest known and possibly fixable (but hard) problem has to do
-with handling control flow. In some cases we can detect an erroneous
+with handling control flow. All of the Python decompilers I have looked
+at have the same problem. In some cases we can detect an erroneous
 decompilation and report that.
 
-About 90% of the decompilation verifies from Python 2.3.7 to Python
-3.4.2 on the standard library packages I have on my system.
+About 90% of the decompilation of Python standard library packages in
+Python 2.7.12 verifies correctly. Over 99% of Python 2.7 and 3.3-3.5
+"weakly" verify. Python 2.6 drops down to 96% weakly verifying.
+Other versions drop off in quality too.
 
 *Verification* is the process of decompiling bytecode, compiling with
-a Python for that byecode version, and then comparing the bytecode
+a Python for that bytecode version, and then comparing the bytecode
 produced by the decompiled/compiled program. Some allowance is made
 for inessential differences. But other semantically equivalent
-differences are not caught. For example ``if x: foo()`` is
-equivalent to ``x and foo()`` and decompilation may turn one into the
-other. *Weak Verification* on the other hand doesn't check bytecode
-for equivalence but does check to see if the resulting decompiled
-source is a valid Python program by running the Python
-interpreter. Because the Python language has changed so much, for best
-results you should use the same Python Version in checking as used in
-the bytecode.
+differences are not caught. For example ``1 and 0`` is decompiled to
+the equivalent ``0``; remnants of the first true evaluation (1) is
+lost when Python compiles this. When Python next compiles ``0`` the
+resulting code is simpler.
+
+*Weak Verification*
+on the other hand doesn't check bytecode for equivalence but does
+check to see if the resulting decompiled source is a valid Python
+program by running the Python interpreter. Because the Python language
+has changed so much, for best results you should use the same Python
+Version in checking as used in the bytecode.
 
 Later distributions average about 200 files. There is some work to do
 on the lower end Python versions which is more difficult for us to
 handle since we don't have a Python interpreter for versions 1.5, 1.6,
 and 2.0.
 
-Python 3.0 support is weak; Python 3.5 largely works, but still has
-some bugs in it.  Python 3.6 changes things drastically by using word
-codes rather than byte codes. That has been addressed, but then it also
-changes function call opcodes and its semantics.
+In the Python 3 series, Python support is is strongest around 3.4 or
+3.3 and drops off as you move further away from those versions. Python
+3.5 largely works, but still has some bugs in it and is missing some
+opcodes.  Python 3.6 changes things drastically by using word codes
+rather than byte codes. That has been addressed, but then it also
+changes function call opcodes and its semantics and has more problems
+with control flow than 3.5 has.
 
 Currently not all Python magic numbers are supported. Specifically in
 some versions of Python, notably Python 3.6, the magic number has

__pkginfo__.py

@@ -12,14 +12,16 @@ copyright   = """
 Copyright (C) 2015, 2016 Rocky Bernstein <rb@dustyfeet.com>.
 """
 
-classifiers =  ['Development Status :: 4 - Beta',
+classifiers =  ['Development Status :: 5 - Production/Stable',
                 'Intended Audience :: Developers',
                 'Operating System :: OS Independent',
                 'Programming Language :: Python',
-                'Programming Language :: Python :: 2',
+                'Programming Language :: Python :: 2.4',
+                'Programming Language :: Python :: 2.5',
                 'Programming Language :: Python :: 2.6',
                 'Programming Language :: Python :: 2.7',
-                'Programming Language :: Python :: 3',
+                'Programming Language :: Python :: 3.1',
+                'Programming Language :: Python :: 3.2',
                 'Programming Language :: Python :: 3.3',
                 'Programming Language :: Python :: 3.4',
                 'Programming Language :: Python :: 3.5',
@@ -37,8 +39,8 @@ entry_points={
         'pydisassemble=uncompyle6.bin.pydisassemble:main',
     ]}
 ftp_url            = None
-install_requires   = ['spark-parser >= 1.4.0, < 1.5.0',
-                      'xdis >= 3.2.3, < 3.3.0']
+install_requires   = ['spark-parser >= 1.5.1, < 1.6.0',
+                      'xdis >= 3.2.4, < 3.3.0']
 license            = 'MIT'
 mailing_list       = 'python-debugger@googlegroups.com'
 modname            = 'uncompyle6'

circle.yml

@@ -1,6 +1,6 @@
 machine:
     python:
-        version: 2.7.8
+        version: 2.7.10
     environment:
         COMPILE: --compile
 

pytest/.gitignore

@@ -1 +1,2 @@
+/.hypothesis
 /__pycache__

pytest/test_docstring.py

@@ -0,0 +1,78 @@
+import sys
+from uncompyle6 import PYTHON3
+if PYTHON3:
+    from io import StringIO
+    minint = -sys.maxsize-1
+    maxint = sys.maxsize
+else:
+    from StringIO import StringIO
+    minint = -sys.maxint-1
+    maxint = sys.maxint
+from uncompyle6.semantics.helper import print_docstring
+
+class PrintFake():
+    def __init__(self):
+        self.pending_newlines = 0
+        self.f = StringIO()
+
+    def write(self, *data):
+        if (len(data) == 0) or (len(data) == 1 and data[0] == ''):
+            return
+        out = ''.join((str(j) for j in data))
+        n = 0
+        for i in out:
+            if i == '\n':
+                n += 1
+                if n == len(out):
+                    self.pending_newlines = max(self.pending_newlines, n)
+                    return
+            elif n:
+                self.pending_newlines = max(self.pending_newlines, n)
+                out = out[n:]
+                break
+            else:
+                break
+
+        if self.pending_newlines > 0:
+            self.f.write('\n'*self.pending_newlines)
+            self.pending_newlines = 0
+
+        for i in out[::-1]:
+            if i == '\n':
+                self.pending_newlines += 1
+            else:
+                break
+
+        if self.pending_newlines:
+            out = out[:-self.pending_newlines]
+        self.f.write(out)
+    def println(self, *data):
+        if data and not(len(data) == 1 and data[0] ==''):
+            self.write(*data)
+        self.pending_newlines = max(self.pending_newlines, 1)
+        return
+    pass
+
+def test_docstring():
+
+    for doc, expect in (
+            ("Now is the time",
+             '  """Now is the time"""'),
+           ("""
+Now is the time
+""",
+            '''  """
+  Now is the time
+  """''')
+
+            # (r'''func placeholder - ' and with ("""\nstring\n  """)''',
+            #  """  r'''func placeholder - ' and with (\"\"\"\nstring\n\"\"\")'''"""),
+            # (r"""func placeholder - ' and with ('''\nstring\n''') and \"\"\"\nstring\n\"\"\" """,
+            # """  r\"\"\"func placeholder - ' and with ('''\nstring\n''') and \"\"\"\nstring\n\"\"\" \"\"\"""")
+            ):
+
+        o = PrintFake()
+        # print(doc)
+        # print(expect)
+        print_docstring(o, '  ', doc)
+        assert expect == o.f.getvalue()

pytest/test_fjt.py

@@ -8,6 +8,18 @@ def bug(state, slotstate):
             for key, value in slotstate.items():
                 setattr(state, key, 2)
 
+# From 2.7 disassemble
+# Problem is not getting while, because
+# COME_FROM not added
+def bug_loop(disassemble, tb=None):
+    if tb:
+        try:
+            tb = 5
+        except AttributeError:
+            raise RuntimeError
+        while tb: tb = tb.tb_next
+    disassemble(tb)
+
 def test_if_in_for():
     code = bug.__code__
     scan = get_scanner(PYTHON_VERSION)
@@ -16,18 +28,35 @@ def test_if_in_for():
         n = scan.setup_code(code)
         scan.build_lines_data(code, n)
         scan.build_prev_op(n)
-        fjt = scan.find_jump_targets()
+        fjt = scan.find_jump_targets(False)
         assert {15: [3], 69: [66], 63: [18]} == fjt
         assert scan.structs == \
           [{'start': 0, 'end': 72, 'type': 'root'},
-           {'start': 18, 'end': 66, 'type': 'if-then'},
+           {'start': 15, 'end': 66, 'type': 'if-then'},
            {'start': 31, 'end': 59, 'type': 'for-loop'},
            {'start': 62, 'end': 63, 'type': 'for-else'}]
+
+        code = bug_loop.__code__
+        n = scan.setup_code(code)
+        scan.build_lines_data(code, n)
+        scan.build_prev_op(n)
+        fjt = scan.find_jump_targets(False)
+        assert{64: [42], 67: [42, 42], 42: [16, 41], 19: [6]} == fjt
+        assert scan.structs == [
+            {'start': 0, 'end': 80, 'type': 'root'},
+            {'start': 3, 'end': 64, 'type': 'if-then'},
+            {'start': 6, 'end': 15, 'type': 'try'},
+            {'start': 19, 'end': 38, 'type': 'except'},
+            {'start': 45, 'end': 67, 'type': 'while-loop'},
+            {'start': 70, 'end': 64, 'type': 'while-else'},
+            # previous bug was not mistaking while-loop for if-then
+            {'start': 48, 'end': 67, 'type': 'while-loop'}]
+
     elif 3.2 < PYTHON_VERSION <= 3.4:
         scan.code = array('B', code.co_code)
         scan.build_lines_data(code)
         scan.build_prev_op()
-        fjt  = scan.find_jump_targets()
+        fjt  = scan.find_jump_targets(False)
         assert {69: [66], 63: [18]} == fjt
         assert scan.structs == \
           [{'end': 72, 'type': 'root', 'start': 0},

pytest/test_grammar.py

@@ -1,6 +1,6 @@
 import re
 from uncompyle6 import PYTHON_VERSION, PYTHON3, IS_PYPY # , PYTHON_VERSION
-from uncompyle6.parser import get_python_parser
+from uncompyle6.parser import get_python_parser, python_parser
 from uncompyle6.scanner import get_scanner
 
 def test_grammar():
@@ -41,7 +41,7 @@ def test_grammar():
             """
             JUMP_BACK CONTINUE RETURN_END_IF
             COME_FROM COME_FROM_EXCEPT COME_FROM_LOOP COME_FROM_WITH
-            COME_FROM_FINALLY
+            COME_FROM_FINALLY ELSE
             LOAD_GENEXPR LOAD_ASSERT LOAD_SETCOMP LOAD_DICTCOMP
             LAMBDA_MARKER RETURN_LAST
             """.split())
@@ -53,3 +53,11 @@ def test_grammar():
         ignore_set.add('STORE_LOCALS')
         opcode_set = set(s.opc.opname).union(ignore_set)
         check_tokens(tokens, opcode_set)
+
+def test_dup_rule():
+    import inspect
+    python_parser(PYTHON_VERSION, inspect.currentframe().f_code,
+                  is_pypy=IS_PYPY,
+                  parser_debug={
+                      'dups': True, 'transition': False, 'reduce': False,
+                      'rules': False, 'errorstack': None, 'context': True})

setup.py

@@ -24,6 +24,6 @@ setup(
        py_modules         = py_modules,
        test_suite         = 'nose.collector',
        url                = web,
-       setup_requires     = ['nose>=1.0'],
+       tests_require     = ['nose>=1.0'],
        version            = VERSION,
        zip_safe           = zip_safe)

test/Makefile

@@ -22,9 +22,9 @@ check:
 #: Run working tests from Python 2.6 or 2.7
 check-2.6 check-2.7: check-bytecode-2 check-bytecode-3 check-bytecode-1  check-2.7-ok
 
-#: Run working tests from Python 3.1
+#: Run working tests from Python 3.0
 check-3.0: check-bytecode
-	@echo Python 3.0 testing not done yet
+	$(PYTHON) test_pythonlib.py --bytecode-3.0 --weak-verify $(COMPILE)
 
 #: Run working tests from Python 3.1
 check-3.1: check-bytecode
@@ -36,11 +36,11 @@ check-3.2: check-bytecode
 
 #: Run working tests from Python 3.3
 check-3.3: check-bytecode
-	$(PYTHON) test_pythonlib.py --bytecode-3.3 --weak-verify $(COMPILE)
+	$(PYTHON) test_pythonlib.py --bytecode-3.3 --verify $(COMPILE)
 
 #: Run working tests from Python 3.4
 check-3.4: check-bytecode check-3.4-ok check-2.7-ok
-	$(PYTHON) test_pythonlib.py --bytecode-3.4 --weak-verify $(COMPILE)
+	$(PYTHON) test_pythonlib.py --bytecode-3.4 --verify $(COMPILE)
 
 #: Run working tests from Python 3.5
 check-3.5: check-bytecode
@@ -100,11 +100,11 @@ check-bytecode-2.5:
 
 #: Check deparsing Python 2.6
 check-bytecode-2.6:
-	$(PYTHON) test_pythonlib.py --bytecode-2.6
+	$(PYTHON) test_pythonlib.py --bytecode-2.6 --weak-verify
 
 #: Check deparsing Python 2.7
 check-bytecode-2.7:
-	$(PYTHON) test_pythonlib.py --bytecode-2.7
+	$(PYTHON) test_pythonlib.py --bytecode-2.7 --verify
 
 #: Check deparsing Python 3.0
 check-bytecode-3.0:

test/simple_source/bug25/02_try_else.py

@@ -0,0 +1,12 @@
+# Python 2.5 bug
+# Was turning into tryelse when there in fact is no else.
+def options(self, section):
+    """Return a list of option names for the given section name."""
+    try:
+        opts = self._sections[section].copy()
+    except KeyError:
+        raise NoSectionError(section)
+    opts.update(self._defaults)
+    if '__name__' in opts:
+        del opts['__name__']
+    return opts.keys()

test/simple_source/bug26/03_elif_vs_continue.py

@@ -0,0 +1,18 @@
+# Bug was using continue fouling up 1st elif, by confusing
+# the "pass" for "continue" by not recognizing the if jump
+# around it. We fixed by ignoring what's done in Python 2.7
+# Better is better detection of control structures
+
+def _compile_charset(charset, flags, code, fixup=None):
+    # compile charset subprogram
+    emit = code.append
+    if fixup is None:
+        fixup = 1
+    for op, av in charset:
+        if op is flags:
+            pass
+        elif op is code:
+            emit(fixup(av))
+        else:
+            raise RuntimeError
+    emit(5)

test/simple_source/bug26/03_if_vs_and.py

@@ -0,0 +1,22 @@
+# From 2.6 decimal
+# Bug was not recognizing scope of if and
+# turning it into xc == 1 and xe *= yc
+def _power_exact(y, xc, yc, xe):
+    yc, ye = y.int, y.exp
+    while yc % 10 == 0:
+        yc //= 10
+        ye += 1
+
+    if xc == 1:
+        xe *= yc
+        while xe % 10 == 0:
+            xe //= 10
+            ye += 1
+        if ye < 0:
+            return None
+        exponent = xe * 10**ye
+        if y and xe:
+            xc = exponent
+        else:
+            xc = 0
+        return 5

test/simple_source/bug31/04_def_annotate.py

@@ -8,3 +8,12 @@ def open(file, mode = "r", buffering = None,
          encoding = None, errors = None,
          newline = None, closefd = True) -> "IOBase":
     return text
+
+def foo(x: 'an argument that defaults to 5' = 5):
+    print(x)
+
+def div(a: dict(type=float, help='the dividend'),
+        b: dict(type=float, help='the divisor (must be different than 0)')
+    ) -> dict(type=float, help='the result of dividing a by b'):
+    """Divide a by b"""
+    return a / b

test/simple_source/bug33/03_while_else.py

@@ -0,0 +1,8 @@
+# Bug from 3.4 threading. Bug is handling while/else
+def acquire(self):
+    with self._cond:
+        while self:
+            rc = False
+        else:
+            rc = True
+    return rc

test/simple_source/bug35/02_build_map_unpack_with_call.py

@@ -0,0 +1 @@
+f(**a, **b)

test/simple_source/bug35/03_async_await.py

@@ -0,0 +1,26 @@
+# Python 3.5+ async and await
+async def await_test(asyncio):
+    reader, writer = await asyncio.open_connection(80)
+    await bar()
+
+async def afor_test():
+
+    async for i in [1,2,3]:
+        x = i
+
+
+async def afor_else_test():
+
+    async for i in [1,2,3]:
+        x = i
+    else:
+        z = 4
+
+
+async def awith_test():
+    async with i:
+        print(i)
+
+async def awith_as_test():
+    async with 1 as i:
+        print(i)

test/simple_source/bug35/03_double_star_unpack.py

@@ -0,0 +1,9 @@
+# Bug in Python 3.5 is getting the two star'd arguments right.
+def sum(a,b,c,d):
+    return a + b + c + d
+
+args=(1,2)
+sum(*args, *args)
+
+# FIXME: this is handled incorrectly
+# (*c,) = (3,4)

test/simple_source/bug35/03_while-if-break.py

@@ -0,0 +1,7 @@
+# Python 3.5 and 3.6 break inside a
+# while True and if / break
+def display_date(loop):
+    while True:
+        if loop.time():
+            break
+        x = 5

test/simple_source/expression/03_tuple_assign.py

@@ -5,3 +5,8 @@ def some_function():
 def some_other_function():
     some_variable, = some_function()
     print(some_variable)
+
+empty_tup = ()
+one_item_tup = ("item1", )
+one_item_tup_without_parentheses = "item", 
+many_items_tup = ("item1", "item2", "item3")

test/simple_source/stmts/00_docstring.py

@@ -0,0 +1,7 @@
+# uncompyle2 bug was not escaping """ properly
+r'''func placeholder - with ("""\nstring\n""")'''
+def foo():
+    r'''func placeholder - ' and with ("""\nstring\n""")'''
+
+def bar():
+    r"""func placeholder - ' and with ('''\nstring\n''') and \"\"\"\nstring\n\"\"\" """

test/test_pyenvlib.py

@@ -33,7 +33,7 @@ TEST_VERSIONS=('2.3.7', '2.4.6', '2.5.6', '2.6.9',
                '2.7.10', '2.7.11', '2.7.12',
                '3.0.1', '3.1.5', '3.2.6',
                '3.3.5', '3.3.6',
-               '3.4.2', '3.5.1')
+               '3.4.2', '3.5.1', '3.6.0')
 
 target_base = '/tmp/py-dis/'
 lib_prefix = os.path.join(os.environ['HOME'], '.pyenv/versions')

tox.ini

@@ -6,13 +6,14 @@ filename = *.py
 ignore = C901,E113,E121,E122,E123,E124,E125,E126,E127,E128,E129,E201,E202,E203,E221,E222,E225,E226,E241,E242,E251,E261,E271,E272,E302,E401,E501,F401,E701,E702
 
 [tox]
-envlist = py26, py27, pypy
+envlist = py27, py34, pypy
 
 [testenv]
 deps =
-    requests>=0.8.8
-    mock>=1.0.1
-commands = python -W always setup.py nosetests {posargs}
+    hypothesis
+    pytest
+    flake8
+commands = python -W always make test {posargs}
 
 [testenv:py27]
 deps =

uncompyle6/__init__.py

@@ -47,7 +47,10 @@ import uncompyle6.semantics.pysource
 import uncompyle6.semantics.fragments
 
 # Export some functions
-from uncompyle6.main import uncompyle_file
+from uncompyle6.main import decompile_file
+
+# For compaitility
+uncompyle_file = decompile_file
 
 # Conventience functions so you can say:
 # from uncompyle6 import deparse_code

uncompyle6/bin/pydisassemble.py

@@ -62,7 +62,6 @@ Type -h for for full help.""" % program
             print(Usage_short, file=sys.stderr)
             sys.exit(1)
 
-
     for file in files:
         if os.path.exists(files[0]):
             disassemble_file(file, sys.stdout, native)

uncompyle6/bin/uncompile.py

@@ -5,7 +5,7 @@
 # Copyright (c) 2000-2002 by hartmut Goebel <h.goebel@crazy-compilers.com>
 #
 from __future__ import print_function
-import sys, os, getopt, tempfile, time
+import sys, os, getopt, time
 
 program, ext = os.path.splitext(os.path.basename(__file__))
 
@@ -35,7 +35,8 @@ Options:
   -p <integer>  use <integer> number of processes
   -r            recurse directories looking for .pyc and .pyo files
   --verify      compare generated source with input byte-code
-                (requires -o)
+  --linemaps    generated line number correspondencies between byte-code
+                and generated source output
   --help        show this message
 
 Debugging Options:
@@ -65,9 +66,9 @@ def usage():
 
 def main_bin():
     if not (sys.version_info[0:2] in ((2, 6), (2, 7),
-                                      (3, 2), (3, 3),
+                                      (3, 1), (3, 2), (3, 3),
                                       (3, 4), (3, 5), (3, 6))):
-        print('Error: %s requires Python 2.6, 2.7, 3.2, 3.3, 3.4, 3.5, or 3.6' % program,
+        print('Error: %s requires Python 2.6-2.7, or 3.1-3.6' % program,
               file=sys.stderr)
         sys.exit(-1)
 
@@ -81,8 +82,8 @@ def main_bin():
 
     try:
         opts, files = getopt.getopt(sys.argv[1:], 'hagtdrVo:c:p:',
-                                    'help asm grammar recurse timestamp tree verify version '
-                                    'showgrammar'.split(' '))
+                                    'help asm grammar linemaps recurse timestamp tree '
+                                    'verify version showgrammar'.split(' '))
     except getopt.GetoptError as e:
         print('%s: %s' % (os.path.basename(sys.argv[0]), e),  file=sys.stderr)
         sys.exit(-1)
@@ -97,6 +98,8 @@ def main_bin():
             sys.exit(0)
         elif opt == '--verify':
             options['do_verify'] = True
+        elif opt == '--linemaps':
+            options['do_linemaps'] = True
         elif opt in ('--asm', '-a'):
             options['showasm'] = 'after'
             options['do_verify'] = False
@@ -139,18 +142,13 @@ def main_bin():
     if src_base:
         sb_len = len( os.path.join(src_base, '') )
         files = [f[sb_len:] for f in files]
-        del sb_len
 
     if not files:
         print("No files given", file=sys.stderr)
         usage()
 
     if outfile == '-':
-        if 'do_verify' in options and options['do_verify'] and len(files) == 1:
-            junk, outfile = tempfile.mkstemp(suffix=".pyc",
-                                             prefix=files[0][0:-4]+'-')
-        else:
-            outfile = None # use stdout
+        outfile = None # use stdout
     elif outfile and os.path.isdir(outfile):
         out_base = outfile; outfile = None
     elif outfile and len(files) > 1:
@@ -226,7 +224,6 @@ def main_bin():
         except (KeyboardInterrupt, OSError):
             pass
 
-
     if timestamp:
         print(time.strftime(timestampfmt))
 

uncompyle6/linenumbers.py

@@ -0,0 +1,61 @@
+from collections import deque
+
+from xdis.code import iscode
+from xdis.load import load_file, load_module
+from xdis.main import get_opcode
+from xdis.bytecode import Bytecode, findlinestarts, offset2line
+
+def line_number_mapping(pyc_filename, src_filename):
+    (version, timestamp, magic_int, code1, is_pypy,
+     source_size) = load_module(pyc_filename)
+    try:
+        code2 = load_file(src_filename)
+    except SyntaxError as e:
+        return str(e)
+
+    queue = deque([code1, code2])
+
+    mappings = []
+
+    opc = get_opcode(version, is_pypy)
+    number_loop(queue, mappings, opc)
+    return sorted(mappings, key=lambda x: x[1])
+
+
+def number_loop(queue, mappings, opc):
+    while len(queue) > 0:
+        code1 = queue.popleft()
+        code2 = queue.popleft()
+        assert code1.co_name == code2.co_name
+        linestarts_orig = findlinestarts(code1)
+        linestarts_uncompiled = list(findlinestarts(code2))
+        mappings += [[line, offset2line(offset, linestarts_uncompiled)] for offset, line in linestarts_orig]
+        bytecode1 = Bytecode(code1, opc)
+        bytecode2 = Bytecode(code2, opc)
+        instr2s = bytecode2.get_instructions(code2)
+        seen = set([code1.co_name])
+        for instr in bytecode1.get_instructions(code1):
+            next_code1 = None
+            if iscode(instr.argval):
+                next_code1 = instr.argval
+            if next_code1:
+                next_code2 = None
+                while not next_code2:
+                    try:
+                        instr2 = next(instr2s)
+                        if iscode(instr2.argval):
+                            next_code2 = instr2.argval
+                            pass
+                    except StopIteration:
+                        break
+                    pass
+                if next_code2:
+                    assert next_code1.co_name == next_code2.co_name
+                    if next_code1.co_name not in seen:
+                        seen.add(next_code1.co_name)
+                        queue.append(next_code1)
+                        queue.append(next_code2)
+                        pass
+                    pass
+            pass
+        pass

uncompyle6/main.py

@@ -1,5 +1,5 @@
 from __future__ import print_function
-import datetime, os, sys
+import datetime, os, subprocess, sys, tempfile
 
 from uncompyle6 import verify, IS_PYPY
 from xdis.code import iscode
@@ -7,10 +7,11 @@ from uncompyle6.disas import check_object_path
 from uncompyle6.semantics import pysource
 from uncompyle6.parser import ParserError
 from uncompyle6.version import VERSION
+from uncompyle6.linenumbers import line_number_mapping
 
 from xdis.load import load_module
 
-def uncompyle(
+def decompile(
         bytecode_version, co, out=None, showasm=None, showast=False,
         timestamp=None, showgrammar=False, code_objects={},
         source_size=None, is_pypy=False, magic_int=None):
@@ -47,9 +48,10 @@ def uncompyle(
         # deparsing failed
         raise pysource.SourceWalkerError(str(e))
 
+# For compatiblity
+uncompyle = decompile
 
-
-def uncompyle_file(filename, outstream=None, showasm=None, showast=False,
+def decompile_file(filename, outstream=None, showasm=None, showast=False,
                    showgrammar=False):
     """
     decompile Python byte-code file (.pyc)
@@ -60,23 +62,27 @@ def uncompyle_file(filename, outstream=None, showasm=None, showast=False,
     (version, timestamp, magic_int, co, is_pypy,
      source_size) = load_module(filename, code_objects)
 
-
     if type(co) == list:
         for con in co:
-            uncompyle(version, con, outstream, showasm, showast,
+            decompile(version, con, outstream, showasm, showast,
                       timestamp, showgrammar, code_objects=code_objects,
                       is_pypy=is_pypy, magic_int=magic_int)
     else:
-        uncompyle(version, co, outstream, showasm, showast,
+        decompile(version, co, outstream, showasm, showast,
                   timestamp, showgrammar,
                   code_objects=code_objects, source_size=source_size,
                   is_pypy=is_pypy, magic_int=magic_int)
     co = None
 
+# For compatiblity
+uncompyle_file = decompile_file
+
+
 # FIXME: combine into an options parameter
 def main(in_base, out_base, files, codes, outfile=None,
          showasm=None, showast=False, do_verify=False,
-         showgrammar=False, raise_on_error=False):
+         showgrammar=False, raise_on_error=False,
+         do_linemaps=False):
     """
     in_base	base directory for input files
     out_base	base directory for output files (ignored when
@@ -99,15 +105,8 @@ def main(in_base, out_base, files, codes, outfile=None,
             pass
         return open(outfile, 'w')
 
-    of = outfile
     tot_files = okay_files = failed_files = verify_failed_files = 0
 
-    # for code in codes:
-    #    version = sys.version[:3] # "2.5"
-    #    with open(code, "r") as f:
-    #        co = compile(f.read(), "", "exec")
-    #    uncompyle(sys.version[:3], co, sys.stdout, showasm=showasm, showast=showast)
-
     for filename in files:
         infile = os.path.join(in_base, filename)
         if not os.path.exists(infile):
@@ -117,10 +116,22 @@ def main(in_base, out_base, files, codes, outfile=None,
 
         # print (infile, file=sys.stderr)
 
-        if of: # outfile was given as parameter
+        if outfile: # outfile was given as parameter
             outstream = _get_outstream(outfile)
         elif out_base is None:
             outstream = sys.stdout
+            if do_linemaps or do_verify:
+                prefix = os.path.basename(filename)
+                if prefix.endswith('.py'):
+                    prefix = prefix[:-len('.py')]
+                junk, outfile = tempfile.mkstemp(suffix=".py",
+                                             prefix=prefix)
+                # Unbuffer output if possible
+                buffering = -1 if sys.stdout.isatty() else 0
+                sys.stdout = os.fdopen(sys.stdout.fileno(), 'w', buffering)
+                tee = subprocess.Popen(["tee", outfile], stdin=subprocess.PIPE)
+                os.dup2(tee.stdin.fileno(), sys.stdout.fileno())
+                os.dup2(tee.stdin.fileno(), sys.stderr.fileno())
         else:
             if filename.endswith('.pyc'):
                 outfile = os.path.join(out_base, filename[0:-1])
@@ -131,15 +142,17 @@ def main(in_base, out_base, files, codes, outfile=None,
 
         # Try to uncompile the input file
         try:
-            uncompyle_file(infile, outstream, showasm, showast, showgrammar)
+            decompile_file(infile, outstream, showasm, showast, showgrammar)
             tot_files += 1
         except (ValueError, SyntaxError, ParserError, pysource.SourceWalkerError) as e:
-            sys.stderr.write("\n# file %s\n# %s" % (infile, e))
+            sys.stdout.write("\n")
+            sys.stderr.write("\n# file %s\n# %s\n" % (infile, e))
             failed_files += 1
         except KeyboardInterrupt:
             if outfile:
                 outstream.close()
                 os.remove(outfile)
+            sys.stdout.write("\n")
             sys.stderr.write("\nLast file: %s   " % (infile))
             raise
         # except:
@@ -152,7 +165,15 @@ def main(in_base, out_base, files, codes, outfile=None,
         #         sys.stderr.write("\n# Can't uncompile %s\n" % infile)
         else: # uncompile successful
             if outfile:
+                if do_linemaps:
+                    mapping = line_number_mapping(infile, outfile)
+                    outstream.write("\n\n## Line number correspondences\n")
+                    import pprint
+                    s = pprint.pformat(mapping, indent=2, width=80)
+                    s2 = '##' + '\n##'.join(s.split("\n")) + "\n"
+                    outstream.write(s2)
                 outstream.close()
+
                 if do_verify:
                     weak_verify = do_verify == 'weak'
                     try:
@@ -167,17 +188,16 @@ def main(in_base, out_base, files, codes, outfile=None,
                         print(e)
                         verify_failed_files += 1
                         os.rename(outfile, outfile + '_unverified')
+                        sys.stderr.write("### Error Verifying %s\n" % filename)
+                        sys.stderr.write(str(e) + "\n")
                         if not outfile:
-                            print("### Error Verifiying %s" % filename,  file=sys.stderr)
-                            print(e, file=sys.stderr)
                             if raise_on_error:
                                 raise
                             pass
                         pass
                 pass
             elif do_verify:
-                print("\n### uncompile successful, but no file to compare against",
-                      file=sys.stderr)
+                sys.stderr.write("\n### uncompile successful, but no file to compare against\n")
                 pass
             else:
                 okay_files += 1
@@ -211,11 +231,11 @@ def status_msg(do_verify, tot_files, okay_files, failed_files,
                verify_failed_files):
     if tot_files == 1:
         if failed_files:
-            return "decompile failed"
+            return "\n# decompile failed"
         elif verify_failed_files:
-            return "decompile verify failed"
+            return "\n# decompile verify failed"
         else:
-            return "Successfully decompiled file"
+            return "\n# Successfully decompiled file"
             pass
         pass
     mess = "decompiled %i files: %i okay, %i failed" % (tot_files, okay_files, failed_files)

uncompyle6/parser.py

@@ -69,6 +69,33 @@ class PythonParser(GenericASTBuilder):
         for i in dir(self):
             setattr(self, i, None)
 
+    def debug_reduce(self, rule, tokens, parent, i):
+        """Customized format and print for our kind of tokens
+        which gets called in debugging grammar reduce rules
+        """
+        def fix(c):
+            s = str(c)
+            i = s.find('_')
+            return s if i == -1 else s[:i]
+
+        prefix = ''
+        if parent and tokens:
+            p_token = tokens[parent]
+            if hasattr(p_token, 'linestart') and p_token.linestart:
+                prefix = 'L.%3d: ' % p_token.linestart
+            else:
+                prefix = '       '
+            if hasattr(p_token, 'offset'):
+                prefix += "%3s" % fix(p_token.offset)
+                if len(rule[1]) > 1:
+                    prefix += '-%-3s ' % fix(tokens[i-1].offset)
+                else:
+                    prefix += '     '
+        else:
+            prefix = '               '
+
+        print("%s%s ::= %s" % (prefix, rule[0], ' '.join(rule[1])))
+
     def error(self, instructions, index):
         # Find the last line boundary
         for start in range(index, -1, -1):
@@ -117,9 +144,9 @@ class PythonParser(GenericASTBuilder):
         # print >> sys.stderr, 'resolve', str(list)
         return GenericASTBuilder.resolve(self, list)
 
-    ##############################################
-    ## Common Python 2 and Python 3 grammar rules
-    ##############################################
+    ###############################################
+    #  Common Python 2 and Python 3 grammar rules #
+    ###############################################
     def p_start(self, args):
         '''
         # The start or goal symbol
@@ -138,8 +165,7 @@ class PythonParser(GenericASTBuilder):
         """
         passstmt ::=
 
-        _stmts ::= _stmts stmt
-        _stmts ::= stmt
+        _stmts ::= stmt+
 
         # statements with continue
         c_stmts ::= _stmts
@@ -246,13 +272,11 @@ class PythonParser(GenericASTBuilder):
 
         # Zero or more COME_FROMs
         # loops can have this
-        _come_from ::= _come_from COME_FROM
-        _come_from ::=
+        _come_from ::= COME_FROM*
 
         # Zero or one COME_FROM
         # And/or expressions have this
-        come_from_opt ::= COME_FROM
-        come_from_opt ::=
+        come_from_opt ::= COME_FROM?
         """
 
     def p_dictcomp(self, args):
@@ -425,7 +449,6 @@ class PythonParser(GenericASTBuilder):
         expr ::= unary_not
         expr ::= binary_subscr
         expr ::= binary_subscr2
-        expr ::= load_attr
         expr ::= get_iter
         expr ::= buildslice2
         expr ::= buildslice3
@@ -467,6 +490,8 @@ class PythonParser(GenericASTBuilder):
         _mklambda ::= load_closure mklambda
         _mklambda ::= mklambda
 
+        # "and" where the first part of the and is true,
+        # so there is only the 2nd part to evaluate
         and2 ::= _jump jmp_false COME_FROM expr COME_FROM
 
         expr ::= conditional
@@ -556,7 +581,7 @@ def parse(p, tokens, customize):
 
 
 def get_python_parser(
-        version, debug_parser={}, compile_mode='exec',
+        version, debug_parser=PARSER_DEFAULT_DEBUG, compile_mode='exec',
         is_pypy = False):
     """Returns parser object for Python version 2 or 3, 3.2, 3.5on,
     etc., depending on the parameters passed.  *compile_mode* is either
@@ -710,8 +735,8 @@ def python_parser(version, co, out=sys.stdout, showasm=False,
     maybe_show_asm(showasm, tokens)
 
     # For heavy grammar debugging
-    parser_debug = {'rules': True, 'transition': True, 'reduce' : True,
-                    'showstack': 'full'}
+    # parser_debug = {'rules': True, 'transition': True, 'reduce' : True,
+    #                 'showstack': 'full'}
     p = get_python_parser(version, parser_debug)
     return parse(p, tokens, customize)
 

uncompyle6/parsers/astnode.py

@@ -28,9 +28,9 @@ class AST(spark_AST):
         i = 0
         for node in self:
             if hasattr(node, '__repr1__'):
-               if enumerate_children:
+                if enumerate_children:
                     child =  node.__repr1__(indent, i)
-               else:
+                else:
                     child = node.__repr1__(indent, None)
             else:
                 inst = node.format(line_prefix='L.')

uncompyle6/parsers/parse2.py

@@ -9,7 +9,7 @@ e.g. 5, myvariable, "for", etc.  they are CPython Bytecode tokens,
 e.g. "LOAD_CONST 5", "STORE NAME myvariable", "SETUP_LOOP", etc.
 
 If we succeed in creating a parse tree, then we have a Python program
-that a later phase can tern into a sequence of ASCII text.
+that a later phase can turn into a sequence of ASCII text.
 """
 
 from __future__ import print_function
@@ -25,20 +25,18 @@ class Python2Parser(PythonParser):
         self.new_rules = set()
 
     def p_print2(self, args):
-        '''
+        """
         stmt ::= print_items_stmt
         stmt ::= print_nl
         stmt ::= print_items_nl_stmt
 
         print_items_stmt ::= expr PRINT_ITEM print_items_opt
         print_items_nl_stmt ::= expr PRINT_ITEM print_items_opt PRINT_NEWLINE_CONT
-        print_items_opt ::= print_items
-        print_items_opt ::=
-        print_items ::= print_items print_item
-        print_items ::= print_item
-        print_item ::= expr PRINT_ITEM_CONT
-        print_nl ::= PRINT_NEWLINE
-        '''
+        print_items_opt ::= print_items?
+        print_items     ::= print_item+
+        print_item      ::= expr PRINT_ITEM_CONT
+        print_nl        ::= PRINT_NEWLINE
+        """
 
     def p_stmt2(self, args):
         """
@@ -76,8 +74,6 @@ class Python2Parser(PythonParser):
         return_if_stmts ::= _stmts return_if_stmt
         return_if_stmt ::= ret_expr RETURN_END_IF
 
-        stmt ::= importstmt
-
         stmt ::= break_stmt
         break_stmt ::= BREAK_LOOP
 
@@ -171,8 +167,7 @@ class Python2Parser(PythonParser):
         try_middle   ::= jmp_abs COME_FROM except_stmts
                          END_FINALLY
 
-        except_stmts ::= except_stmts except_stmt
-        except_stmts ::= except_stmt
+        except_stmts ::= except_stmt+
 
         except_stmt ::= except_cond1 except_suite
         except_stmt ::= except
@@ -210,14 +205,6 @@ class Python2Parser(PythonParser):
         and  ::= expr jmp_false expr come_from_opt
         or   ::= expr jmp_true  expr come_from_opt
 
-        slice0 ::= expr SLICE+0
-        slice0 ::= expr DUP_TOP SLICE+0
-        slice1 ::= expr expr SLICE+1
-        slice1 ::= expr expr DUP_TOPX_2 SLICE+1
-        slice2 ::= expr expr SLICE+2
-        slice2 ::= expr expr DUP_TOPX_2 SLICE+2
-        slice3 ::= expr expr expr SLICE+3
-        slice3 ::= expr expr expr DUP_TOPX_3 SLICE+3
         unary_convert ::= expr UNARY_CONVERT
 
         # In Python 3, DUP_TOPX_2 is DUP_TOP_TWO
@@ -248,11 +235,10 @@ class Python2Parser(PythonParser):
         """
         inplace_op ::= INPLACE_DIVIDE
         binary_op  ::= BINARY_DIVIDE
-        binary_subscr2 ::= expr expr DUP_TOPX_2 BINARY_SUBSCR
         """
 
     def add_custom_rules(self, tokens, customize):
-        '''
+        """
         Special handling for opcodes such as those that take a variable number
         of arguments -- we add a new rule for each:
 
@@ -271,7 +257,7 @@ class Python2Parser(PythonParser):
             expr ::= expr {expr}^n CALL_FUNCTION_KW_n POP_TOP
 
         PyPy adds custom rules here as well
-        '''
+        """
         for opname, v in list(customize.items()):
             opname_base = opname[:opname.rfind('_')]
             if opname == 'PyPy':
@@ -286,7 +272,7 @@ class Python2Parser(PythonParser):
                 continue
             elif opname_base in ('BUILD_LIST', 'BUILD_TUPLE', 'BUILD_SET'):
                 thousands = (v//1024)
-                thirty32s = ((v//32)%32)
+                thirty32s = ((v//32) % 32)
                 if thirty32s > 0:
                     rule = "expr32 ::=%s" % (' expr' * 32)
                     self.add_unique_rule(rule, opname_base, v, customize)
@@ -296,7 +282,7 @@ class Python2Parser(PythonParser):
                                          opname_base, v, customize)
                     self.seen1024 = True
                 rule = ('build_list ::= ' + 'expr1024 '*thousands +
-                                        'expr32 '*thirty32s + 'expr '*(v%32) + opname)
+                                        'expr32 '*thirty32s + 'expr '*(v % 32) + opname)
             elif opname == 'LOOKUP_METHOD':
                 # A PyPy speciality - DRY with parse3
                 self.add_unique_rule("load_attr ::= expr LOOKUP_METHOD",
@@ -347,7 +333,7 @@ class Python2Parser(PythonParser):
                 # always be the case.
                 self.add_unique_rules([
                     "stmt ::= tryfinallystmt_pypy",
-                    "tryfinallystmt_pypy ::= SETUP_FINALLY suite_stmts_opt COME_FROM "
+                    "tryfinallystmt_pypy ::= SETUP_FINALLY suite_stmts_opt COME_FROM_FINALLY "
                         "suite_stmts_opt END_FINALLY"
                 ], customize)
                 continue
@@ -400,6 +386,26 @@ class Python2Parser(PythonParser):
             else:
                 raise Exception('unknown customize token %s' % opname)
             self.add_unique_rule(rule, opname_base, v, customize)
+            pass
+        self.check_reduce['augassign1'] = 'AST'
+        self.check_reduce['augassign2'] = 'AST'
+        self.check_reduce['_stmts'] = 'AST'
+        return
+
+    def reduce_is_invalid(self, rule, ast, tokens, first, last):
+        lhs = rule[0]
+        if lhs in ('augassign1', 'augassign2') and ast[0][0] == 'and':
+            return True
+        elif lhs == '_stmts':
+            for i, stmt in enumerate(ast):
+                if stmt == '_stmts':
+                    stmt = stmt[0]
+                assert stmt == 'stmt'
+                if stmt[0] == 'return_stmt':
+                    return i+1 != len(ast)
+                pass
+            return False
+        return False
 
 class Python2ParserSingle(Python2Parser, PythonParserSingle):
     pass

uncompyle6/parsers/parse23.py

@@ -18,6 +18,9 @@ class Python23Parser(Python24Parser):
         # of Python
         _while1test ::= SETUP_LOOP JUMP_FORWARD JUMP_IF_FALSE POP_TOP COME_FROM
 
+        while1stmt ::= _while1test l_stmts_opt JUMP_BACK
+                       POP_TOP POP_BLOCK COME_FROM
+
         while1stmt ::= _while1test l_stmts_opt JUMP_BACK
                        COME_FROM POP_TOP POP_BLOCK COME_FROM
 

uncompyle6/parsers/parse25.py

@@ -13,7 +13,7 @@ class Python25Parser(Python26Parser):
         self.customized = {}
 
     def p_misc25(self, args):
-        '''
+        """
         # If "return_if_stmt" is in a loop, a JUMP_BACK can be emitted. In 2.6 the
         # JUMP_BACK doesn't appear
 
@@ -33,7 +33,33 @@ class Python25Parser(Python26Parser):
 
         with_cleanup ::= LOAD_FAST DELETE_FAST WITH_CLEANUP END_FINALLY
         with_cleanup ::= LOAD_NAME DELETE_NAME WITH_CLEANUP END_FINALLY
-        '''
+        """
+
+    def add_custom_rules(self, tokens, customize):
+        super(Python25Parser, self).add_custom_rules(tokens, customize)
+        if self.version == 2.5:
+            self.check_reduce['tryelsestmt'] = 'tokens'
+
+    def reduce_is_invalid(self, rule, ast, tokens, first, last):
+        invalid = super(Python25Parser,
+                        self).reduce_is_invalid(rule, ast,
+                                                tokens, first, last)
+        if invalid:
+            return invalid
+        lhs = rule[0]
+        if lhs in ('tryelsestmt', ):
+            # The end of the else part of try/else come_from has to come
+            # from an END_FINALLY statement
+            if tokens[last-1].type.startswith('COME_FROM'):
+                end_finally_offset = int(tokens[last-1].pattr)
+                current  = first
+                while current < last:
+                    offset = tokens[current].offset
+                    if offset == end_finally_offset:
+                        return tokens[current].type != 'END_FINALLY'
+                    current += 1
+        return False
+
 
 class Python25ParserSingle(Python26Parser, PythonParserSingle):
     pass

uncompyle6/parsers/parse26.py

@@ -13,7 +13,6 @@ class Python26Parser(Python2Parser):
         super(Python26Parser, self).__init__(debug_parser)
         self.customized = {}
 
-
     def p_try_except26(self, args):
         """
         except_stmt  ::= except_cond3 except_suite
@@ -29,9 +28,9 @@ class Python26Parser(Python2Parser):
                          POP_TOP END_FINALLY
 
         try_middle   ::= jmp_abs COME_FROM except_stmts
-                         come_from_pop END_FINALLY
+                         POP_TOP END_FINALLY
 
-        trystmt      ::= SETUP_EXCEPT suite_stmts_opt come_from_pop
+        trystmt      ::= SETUP_EXCEPT suite_stmts_opt POP_TOP
                          try_middle
 
         # Sometimes we don't put in COME_FROM to the next statement
@@ -43,16 +42,22 @@ class Python26Parser(Python2Parser):
                          try_middle come_froms
 
         tryelsestmt    ::= SETUP_EXCEPT suite_stmts_opt POP_BLOCK
-                           try_middle else_suite come_froms
+                           try_middle else_suite COME_FROM
 
         _ifstmts_jump ::= c_stmts_opt JUMP_FORWARD COME_FROM POP_TOP
 
         except_suite ::= c_stmts_opt JUMP_FORWARD come_from_pop
+        except_suite ::= c_stmts_opt JUMP_FORWARD POP_TOP
+        except_suite ::= c_stmts_opt jmp_abs come_from_pop
 
         # Python 3 also has this.
         come_froms ::= come_froms COME_FROM
         come_froms ::= COME_FROM
 
+        # This is what happens after a jump where
+        # we start a new block. For reasons I don't fully
+        # understand, there is also a value on the top of the stack
+        come_from_pop   ::=  COME_FROM POP_TOP
         come_froms_pop ::= come_froms POP_TOP
 
         """
@@ -70,14 +75,15 @@ class Python26Parser(Python2Parser):
         jmp_true     ::= JUMP_IF_TRUE POP_TOP
         jmp_false    ::= JUMP_IF_FALSE POP_TOP
 
-        jf_pop       ::= JUMP_FORWARD come_from_pop
-        jf_pop       ::= JUMP_ABSOLUTE come_from_pop
-        jb_pop       ::= JUMP_BACK come_from_pop
+        jf_pop       ::= JUMP_FORWARD POP_TOP
+        jf_pop       ::= JUMP_ABSOLUTE POP_TOP
+        jb_pop       ::= JUMP_BACK POP_TOP
 
         jb_cont      ::= JUMP_BACK
         jb_cont      ::= CONTINUE
 
         jb_cf_pop ::= JUMP_BACK come_froms POP_TOP
+        jb_cf_pop ::= JUMP_BACK POP_TOP
         ja_cf_pop ::= JUMP_ABSOLUTE come_froms POP_TOP
         jf_cf_pop ::= JUMP_FORWARD come_froms POP_TOP
 
@@ -85,13 +91,12 @@ class Python26Parser(Python2Parser):
         jb_bp_come_from ::= JUMP_BACK bp_come_from
 
         _ifstmts_jump ::= c_stmts_opt jf_pop COME_FROM
-        _ifstmts_jump ::= c_stmts_opt JUMP_FORWARD COME_FROM come_from_pop
+        _ifstmts_jump ::= c_stmts_opt JUMP_FORWARD COME_FROM POP_TOP
         _ifstmts_jump ::= c_stmts_opt JUMP_FORWARD come_froms POP_TOP COME_FROM
 
         # This is what happens after a jump where
         # we start a new block. For reasons I don't fully
         # understand, there is also a value on the top of the stack
-        come_from_pop   ::=  COME_FROM POP_TOP
         come_froms_pop  ::=  come_froms POP_TOP
 
         """
@@ -108,16 +113,10 @@ class Python26Parser(Python2Parser):
 
         break_stmt ::= BREAK_LOOP JUMP_BACK
 
-        # Semantic actions want the else to be at position 3
-        ifelsestmt  ::= testexpr c_stmts_opt jf_cf_pop else_suite come_froms
-        ifelsestmt  ::= testexpr c_stmts_opt filler else_suitel come_froms POP_TOP
-
         # Semantic actions want else_suitel to be at index 3
         ifelsestmtl ::= testexpr c_stmts_opt jb_cf_pop else_suitel
         ifelsestmtc ::= testexpr c_stmts_opt ja_cf_pop else_suitec
 
-        iflaststmt  ::= testexpr c_stmts_opt JUMP_ABSOLUTE come_froms POP_TOP
-
         # Semantic actions want suite_stmts_opt to be at index 3
         withstmt ::= expr setupwith SETUP_FINALLY suite_stmts_opt
                      POP_BLOCK LOAD_CONST COME_FROM WITH_CLEANUP END_FINALLY
@@ -145,15 +144,38 @@ class Python26Parser(Python2Parser):
         whileelsestmt ::= SETUP_LOOP testexpr l_stmts_opt jb_pop POP_BLOCK
                           else_suite COME_FROM
 
-        return_stmt ::= ret_expr RETURN_END_IF come_from_pop
-        return_stmt ::= ret_expr RETURN_VALUE come_from_pop
-        return_if_stmt ::= ret_expr RETURN_END_IF come_from_pop
+        return_stmt ::= ret_expr RETURN_END_IF POP_TOP
+        return_stmt ::= ret_expr RETURN_VALUE POP_TOP
+        return_if_stmt ::= ret_expr RETURN_END_IF POP_TOP
 
         iflaststmtl ::= testexpr c_stmts_opt JUMP_BACK come_from_pop
         iflaststmt  ::= testexpr c_stmts_opt JUMP_ABSOLUTE come_from_pop
 
         while1stmt ::= SETUP_LOOP l_stmts_opt JUMP_BACK COME_FROM
 
+        ifstmt         ::= testexpr_then _ifstmts_jump
+
+        # Semantic actions want the else to be at position 3
+        ifelsestmt     ::= testexpr      c_stmts_opt jf_cf_pop else_suite come_froms
+        ifelsestmt     ::= testexpr_then c_stmts_opt jf_cf_pop else_suite come_froms
+        ifelsestmt     ::= testexpr      c_stmts_opt filler else_suitel come_froms POP_TOP
+        ifelsestmt     ::= testexpr_then c_stmts_opt filler else_suitel come_froms POP_TOP
+
+        # Semantic actions want else_suitel to be at index 3
+        ifelsestmtl    ::= testexpr_then c_stmts_opt jb_cf_pop else_suitel
+        ifelsestmtc    ::= testexpr_then c_stmts_opt ja_cf_pop else_suitec
+
+        iflaststmt     ::= testexpr_then c_stmts_opt JUMP_ABSOLUTE come_froms POP_TOP
+        iflaststmt     ::= testexpr      c_stmts_opt JUMP_ABSOLUTE come_froms POP_TOP
+
+        testexpr_then  ::= testtrue_then
+        testexpr_then  ::= testfalse_then
+        testtrue_then  ::= expr jmp_true_then
+        testfalse_then ::= expr jmp_false_then
+
+        jmp_false_then ::= JUMP_IF_FALSE THEN POP_TOP
+        jmp_true_then  ::= JUMP_IF_TRUE THEN POP_TOP
+
         # Common with 2.7
         while1stmt ::= SETUP_LOOP return_stmts bp_come_from
         while1stmt ::= SETUP_LOOP return_stmts COME_FROM
@@ -172,32 +194,35 @@ class Python26Parser(Python2Parser):
 
         list_iter  ::= list_if JUMP_BACK
         list_iter  ::= list_if JUMP_BACK COME_FROM POP_TOP
-	list_compr ::= BUILD_LIST_0 DUP_TOP
-		       designator list_iter del_stmt
-	list_compr ::= BUILD_LIST_0 DUP_TOP
-		       designator list_iter JUMP_BACK del_stmt
-	lc_body    ::= LOAD_NAME expr LIST_APPEND
+        list_compr ::= BUILD_LIST_0 DUP_TOP
+                       designator list_iter del_stmt
+        list_compr ::= BUILD_LIST_0 DUP_TOP
+	               designator list_iter JUMP_BACK del_stmt
+        lc_body    ::= LOAD_NAME expr LIST_APPEND
 	lc_body    ::= LOAD_FAST expr LIST_APPEND
 
         comp_for ::= SETUP_LOOP expr _for designator comp_iter jb_bp_come_from
 
         comp_body ::= gen_comp_body
 
+        for_block ::= l_stmts_opt _come_from POP_TOP JUMP_BACK
+
         # Make sure we keep indices the same as 2.7
         setup_loop_lf ::= SETUP_LOOP LOAD_FAST
         genexpr_func ::= setup_loop_lf FOR_ITER designator comp_iter jb_bp_come_from
-        genexpr_func ::= setup_loop_lf FOR_ITER designator comp_iter JUMP_BACK come_from_pop jb_bp_come_from
+        genexpr_func ::= setup_loop_lf FOR_ITER designator comp_iter JUMP_BACK come_from_pop
+                         jb_bp_come_from
         genexpr ::= LOAD_GENEXPR MAKE_FUNCTION_0 expr GET_ITER CALL_FUNCTION_1 COME_FROM
-
+        list_if ::= list_if ::= expr jmp_false_then list_iter
         '''
 
     def p_ret26(self, args):
         '''
-        ret_and  ::= expr jmp_false ret_expr_or_cond COME_FROM
-        ret_or   ::= expr jmp_true ret_expr_or_cond COME_FROM
-        ret_cond ::= expr jmp_false expr RETURN_END_IF POP_TOP ret_expr_or_cond
-        ret_cond ::= expr jmp_false expr ret_expr_or_cond
-        ret_cond_not ::= expr jmp_true expr RETURN_END_IF POP_TOP ret_expr_or_cond
+        ret_and      ::= expr jmp_false ret_expr_or_cond COME_FROM
+        ret_or       ::= expr jmp_true ret_expr_or_cond COME_FROM
+        ret_cond     ::= expr jmp_false_then expr RETURN_END_IF POP_TOP ret_expr_or_cond
+        ret_cond     ::= expr jmp_false_then expr ret_expr_or_cond
+        ret_cond_not ::= expr jmp_true_then  expr RETURN_END_IF POP_TOP ret_expr_or_cond
 
         return_if_stmt ::= ret_expr RETURN_END_IF POP_TOP
         return_stmt ::= ret_expr RETURN_VALUE POP_TOP
@@ -207,17 +232,37 @@ class Python26Parser(Python2Parser):
         '''
 
     def p_except26(self, args):
-        '''
-        except_suite ::= c_stmts_opt jmp_abs come_from_pop
-        '''
+        """
+        except_suite ::= c_stmts_opt jmp_abs POP_TOP
+        """
 
     def p_misc26(self, args):
-        '''
+        """
         conditional  ::= expr jmp_false expr jf_cf_pop expr come_from_opt
         and  ::= expr JUMP_IF_FALSE POP_TOP expr JUMP_IF_FALSE POP_TOP
         cmp_list ::= expr cmp_list1 ROT_TWO COME_FROM POP_TOP _come_from
-        '''
 
+        conditional_lambda ::= expr jmp_false_then return_if_stmt return_stmt LAMBDA_MARKER
+        """
+
+    def add_custom_rules(self, tokens, customize):
+        super(Python26Parser, self).add_custom_rules(tokens, customize)
+        self.check_reduce['and'] = 'AST'
+
+    def reduce_is_invalid(self, rule, ast, tokens, first, last):
+        invalid = super(Python26Parser,
+                        self).reduce_is_invalid(rule, ast,
+                                                tokens, first, last)
+        if invalid:
+            return invalid
+        if rule == ('and', ('expr', 'jmp_false', 'expr', '\\e_come_from_opt')):
+            # Test that jmp_false jumps to the end of "and"
+            # or that it jumps to the same place as the end of "and"
+            jmp_false = ast[1][0]
+            jmp_target = jmp_false.offset + jmp_false.attr + 3
+            return not (jmp_target == tokens[last].offset or
+                        tokens[last].pattr == jmp_false.pattr)
+        return False
 class Python26ParserSingle(Python2Parser, PythonParserSingle):
     pass
 
@@ -237,8 +282,8 @@ if __name__ == '__main__':
             """.split()))
         remain_tokens = set(tokens) - opcode_set
         import re
-        remain_tokens = set([re.sub('_\d+$','', t) for t in remain_tokens])
-        remain_tokens = set([re.sub('_CONT$','', t) for t in remain_tokens])
+        remain_tokens = set([re.sub('_\d+$', '', t) for t in remain_tokens])
+        remain_tokens = set([re.sub('_CONT$', '', t) for t in remain_tokens])
         remain_tokens = set(remain_tokens) - opcode_set
         print(remain_tokens)
         # print(sorted(p.rule2name.items()))

uncompyle6/parsers/parse27.py

@@ -31,6 +31,10 @@ class Python27Parser(Python2Parser):
 
     def p_try27(self, args):
         """
+        tryfinallystmt ::= SETUP_FINALLY suite_stmts_opt
+                           POP_BLOCK LOAD_CONST
+                           COME_FROM_FINALLY suite_stmts_opt END_FINALLY
+
         tryelsestmt    ::= SETUP_EXCEPT suite_stmts_opt POP_BLOCK
                            try_middle else_suite COME_FROM
 
@@ -45,7 +49,10 @@ class Python27Parser(Python2Parser):
 
     def p_jump27(self, args):
         """
-        _ifstmts_jump ::= c_stmts_opt JUMP_FORWARD COME_FROM
+        come_froms ::= come_froms COME_FROM
+        come_froms ::= COME_FROM
+
+        _ifstmts_jump ::= c_stmts_opt JUMP_FORWARD come_froms
         bp_come_from    ::= POP_BLOCK COME_FROM
 
         # FIXME: Common with 3.0+
@@ -77,15 +84,13 @@ class Python27Parser(Python2Parser):
         assert2       ::= assert_expr jmp_true LOAD_ASSERT expr RAISE_VARARGS_2
 
         withstmt ::= expr SETUP_WITH POP_TOP suite_stmts_opt
-                POP_BLOCK LOAD_CONST COME_FROM
+                POP_BLOCK LOAD_CONST COME_FROM_WITH
                 WITH_CLEANUP END_FINALLY
 
         withasstmt ::= expr SETUP_WITH designator suite_stmts_opt
-                POP_BLOCK LOAD_CONST COME_FROM
+                POP_BLOCK LOAD_CONST COME_FROM_WITH
                 WITH_CLEANUP END_FINALLY
 
-        while1stmt ::= SETUP_LOOP l_stmts_opt JUMP_BACK POP_BLOCK COME_FROM
-
         # Common with 2.6
         while1stmt ::= SETUP_LOOP return_stmts bp_come_from
         while1stmt ::= SETUP_LOOP return_stmts COME_FROM
@@ -110,8 +115,10 @@ if __name__ == '__main__':
             """.split()))
         remain_tokens = set(tokens) - opcode_set
         import re
-        remain_tokens = set([re.sub('_\d+$','', t) for t in remain_tokens])
-        remain_tokens = set([re.sub('_CONT$','', t) for t in remain_tokens])
+        remain_tokens = set([re.sub('_\d+$', '', t)
+                             for t in remain_tokens])
+        remain_tokens = set([re.sub('_CONT$', '', t)
+                             for t in remain_tokens])
         remain_tokens = set(remain_tokens) - opcode_set
         print(remain_tokens)
         # p.dumpGrammar()

uncompyle6/parsers/parse3.py

@@ -100,8 +100,7 @@ class Python3Parser(PythonParser):
         del_stmt ::= expr DELETE_ATTR
 
         kwarg   ::= LOAD_CONST expr
-        kwargs  ::= kwargs kwarg
-        kwargs  ::=
+        kwargs  ::= kwarg*
 
         classdef ::= build_class designator
 
@@ -139,16 +138,23 @@ class Python3Parser(PythonParser):
         iflaststmtl ::= testexpr c_stmts_opt JUMP_BACK
         iflaststmtl ::= testexpr c_stmts_opt JUMP_BACK COME_FROM_LOOP
 
+        # These are used to keep AST indices the same
+        jf_else    ::= JUMP_FORWARD ELSE
+        ja_else    ::= JUMP_ABSOLUTE ELSE
+
+        # Note: in if/else kinds of statements, we err on the side
+        # of missing "else" clauses. Therefore we include grammar
+        # rules with and without ELSE.
+
         ifelsestmt ::= testexpr c_stmts_opt JUMP_FORWARD else_suite COME_FROM
+        ifelsestmt ::= testexpr c_stmts_opt jf_else else_suite _come_from
 
         ifelsestmtc ::= testexpr c_stmts_opt JUMP_ABSOLUTE else_suitec
+        ifelsestmtc ::= testexpr c_stmts_opt ja_else else_suitec
 
         ifelsestmtr ::= testexpr return_if_stmts return_stmts
 
         ifelsestmtl ::= testexpr c_stmts_opt JUMP_BACK else_suitel
-        ifelsestmtl ::= testexpr c_stmts_opt JUMP_BACK else_suitel JUMP_BACK COME_FROM_LOOP
-        ifelsestmtl ::= testexpr c_stmts_opt JUMP_BACK else_suitel COME_FROM_LOOP
-
 
         # FIXME: this feels like a hack. Is it just 1 or two
         # COME_FROMs?  the parsed tree for this and even with just the
@@ -246,7 +252,6 @@ class Python3Parser(PythonParser):
         c_stmts_opt34 ::= JUMP_BACK JUMP_ABSOLUTE c_stmts_opt
         """
 
-
     def p_def_annotations3(self, args):
         """
         # Annotated functions
@@ -335,11 +340,12 @@ class Python3Parser(PythonParser):
         whilestmt         ::= SETUP_LOOP testexpr return_stmts          POP_BLOCK
                               COME_FROM_LOOP
 
+        while1elsestmt    ::= SETUP_LOOP          l_stmts     JUMP_BACK
+                              else_suite
+
         whileelsestmt     ::= SETUP_LOOP testexpr l_stmts_opt JUMP_BACK POP_BLOCK
                               else_suite COME_FROM_LOOP
 
-        while1elsestmt    ::= SETUP_LOOP          l_stmts     JUMP_BACK
-                              else_suite
 
         whileelselaststmt ::= SETUP_LOOP testexpr l_stmts_opt JUMP_BACK POP_BLOCK
                               else_suitec COME_FROM_LOOP
@@ -348,14 +354,13 @@ class Python3Parser(PythonParser):
 
         # FIXME: Python 3.? starts adding branch optimization? Put this starting there.
         while1stmt        ::= SETUP_LOOP l_stmts
+        while1stmt        ::= SETUP_LOOP l_stmts COME_FROM_LOOP
 
         # FIXME: investigate - can code really produce a NOP?
         whileTruestmt     ::= SETUP_LOOP l_stmts_opt JUMP_BACK NOP
                               COME_FROM_LOOP
         whileTruestmt     ::= SETUP_LOOP l_stmts_opt JUMP_BACK POP_BLOCK NOP
                               COME_FROM_LOOP
-        whileTruestmt     ::= SETUP_LOOP l_stmts_opt JUMP_BACK POP_BLOCK NOP
-                              COME_FROM_LOOP
         forstmt           ::= SETUP_LOOP expr _for designator for_block POP_BLOCK NOP
                               COME_FROM_LOOP
         """
@@ -370,16 +375,17 @@ class Python3Parser(PythonParser):
         '''
 
     def p_expr3(self, args):
-        '''
+        """
+        conditional    ::= expr jmp_false expr jf_else expr COME_FROM
+        conditionalnot ::= expr jmp_true  expr jf_else expr COME_FROM
+
+
         expr ::= LOAD_CLASSNAME
 
         # Python 3.4+
         expr ::= LOAD_CLASSDEREF
 
-        binary_subscr2 ::= expr expr DUP_TOP_TWO BINARY_SUBSCR
-        # Python3 drops slice0..slice3
-
-        '''
+        """
 
     @staticmethod
     def call_fn_name(token):
@@ -442,11 +448,20 @@ class Python3Parser(PythonParser):
         args_kw = (token.attr >> 8) & 0xff
         nak = ( len(opname)-len('CALL_FUNCTION') ) // 3
         token.type = self.call_fn_name(token)
-        rule = ('call_function ::= expr '
-                + ('pos_arg ' * args_pos)
-                + ('kwarg ' * args_kw)
-                + 'expr ' * nak + token.type)
+        rule = ('call_function ::= expr ' +
+                ('pos_arg ' * args_pos) +
+                ('kwarg ' * args_kw) +
+                'expr ' * nak + token.type)
         self.add_unique_rule(rule, token.type, args_pos, customize)
+        if self.version >= 3.5:
+            rule = ('async_call_function ::= expr ' +
+                    ('pos_arg ' * args_pos) +
+                    ('kwarg ' * args_kw) +
+                    'expr ' * nak + token.type +
+                    ' GET_AWAITABLE LOAD_CONST YIELD_FROM')
+            self.add_unique_rule(rule, token.type, args_pos, customize)
+            self.add_unique_rule('expr ::= async_call_function', token.type, args_pos, customize)
+
         rule = ('classdefdeco2 ::= LOAD_BUILD_CLASS mkfunc %s%s_%d'
                 %  (('expr ' * (args_pos-1)), opname, args_pos))
         self.add_unique_rule(rule, token.type, args_pos, customize)
@@ -470,8 +485,15 @@ class Python3Parser(PythonParser):
 
             # build_class (see load_build_class)
 
-            build_list  ::= {expr}^n BUILD_LIST_n
-            build_list  ::= {expr}^n BUILD_TUPLE_n
+            # Even the below say _list, in the semantic rules we
+            # disambiguate tuples, and sets from lists
+
+            build_list ::= {expr}^n BUILD_LIST_n
+            build_list ::= {expr}^n BUILD_TUPLE_n
+            build_list ::= {expr}^n BUILD_SET_n
+            build_list ::= {expr}^n BUILD_LIST_UNPACK_n
+            build_list ::= {expr}^n BUILD_SET_UNPACK_n
+            build_list ::= {expr}^n BUILD_TUPLE_UNPACK_n
 
             load_closure  ::= {LOAD_CLOSURE}^n BUILD_TUPLE_n
             # call_function (see custom_classfunc_rule)
@@ -544,7 +566,8 @@ class Python3Parser(PythonParser):
             elif opname_base in ('BUILD_LIST', 'BUILD_TUPLE', 'BUILD_SET'):
                 v = token.attr
                 rule = ('build_list ::= ' + 'expr1024 ' * int(v//1024) +
-                                        'expr32 ' * int((v//32)%32) + 'expr '*(v%32) + opname)
+                        'expr32 ' * int((v//32) % 32) +
+                        'expr ' * (v % 32) + opname)
                 self.add_unique_rule(rule, opname, token.attr, customize)
                 if opname_base == 'BUILD_TUPLE':
                     rule = ('load_closure ::= %s%s' % (('LOAD_CLOSURE ' * v), opname))
@@ -584,9 +607,10 @@ class Python3Parser(PythonParser):
                     self.add_unique_rule(rule, 'kvlist_n', 1, customize)
                     rule = "mapexpr ::=  BUILD_MAP_n kvlist_n"
                 elif self.version >= 3.5:
-                    rule = kvlist_n + ' ::= ' + 'expr ' * (token.attr*2)
-                    self.add_unique_rule(rule, opname, token.attr, customize)
-                    rule = "mapexpr ::=  %s %s" % (kvlist_n, opname)
+                    if  opname != 'BUILD_MAP_WITH_CALL':
+                        rule = kvlist_n + ' ::= ' + 'expr ' * (token.attr*2)
+                        self.add_unique_rule(rule, opname, token.attr, customize)
+                        rule = "mapexpr ::=  %s %s" % (kvlist_n, opname)
                 else:
                     rule = kvlist_n + ' ::= ' + 'expr expr STORE_MAP ' * token.attr
                     self.add_unique_rule(rule, opname, token.attr, customize)
@@ -628,6 +652,27 @@ class Python3Parser(PythonParser):
                     rule = ('mkfunc ::= kwargs %sexpr %s' %
                             ('pos_arg ' * args_pos, opname))
                 self.add_unique_rule(rule, opname, token.attr, customize)
+                if opname.startswith('MAKE_FUNCTION_A'):
+                    # rule = ('mkfunc2 ::= %s%sEXTENDED_ARG %s' %
+                    #         ('pos_arg ' * (args_pos), 'kwargs ' * (annotate_args-1), opname))
+                    self.add_unique_rule(rule, opname, token.attr, customize)
+                    if self.version >= 3.3:
+                        rule = ('mkfunc_annotate ::= %s%sannotate_tuple LOAD_CONST LOAD_CONST EXTENDED_ARG %s' %
+                                (('pos_arg ' * (args_pos)),
+                                 ('call_function ' * (annotate_args-1)), opname))
+                        self.add_unique_rule(rule, opname, token.attr, customize)
+                        rule = ('mkfunc_annotate ::= %s%sannotate_tuple LOAD_CONST LOAD_CONST EXTENDED_ARG %s' %
+                                (('pos_arg ' * (args_pos)),
+                                 ('annotate_arg ' * (annotate_args-1)), opname))
+                    else:
+                        rule = ('mkfunc_annotate ::= %s%sannotate_tuple LOAD_CONST EXTENDED_ARG %s' %
+                                (('pos_arg ' * (args_pos)),
+                                 ('annotate_arg ' * (annotate_args-1)), opname))
+                        self.add_unique_rule(rule, opname, token.attr, customize)
+                        rule = ('mkfunc_annotate ::= %s%sannotate_tuple LOAD_CONST EXTENDED_ARG %s' %
+                                (('pos_arg ' * (args_pos)),
+                                 ('call_function ' * (annotate_args-1)), opname))
+                    self.add_unique_rule(rule, opname, token.attr, customize)
             elif opname_base == 'CALL_METHOD':
                 # PyPy only - DRY with parse2
 
@@ -637,10 +682,10 @@ class Python3Parser(PythonParser):
 
                 # number of apply equiv arguments:
                 nak = ( len(opname_base)-len('CALL_METHOD') ) // 3
-                rule = ('call_function ::= expr '
-                        + ('pos_arg ' * args_pos)
-                        + ('kwarg ' * args_kw)
-                        + 'expr ' * nak + opname)
+                rule = ('call_function ::= expr ' +
+                        ('pos_arg ' * args_pos) +
+                        ('kwarg ' * args_kw) +
+                        'expr ' * nak + opname)
                 self.add_unique_rule(rule, opname, token.attr, customize)
             elif opname.startswith('MAKE_CLOSURE'):
                 # DRY with MAKE_FUNCTION
@@ -684,8 +729,45 @@ class Python3Parser(PythonParser):
                 rule = ('mkfunc ::= %sload_closure LOAD_CONST %s'
                         % ('expr ' * args_pos, opname))
                 self.add_unique_rule(rule, opname, token.attr, customize)
+                pass
+        self.check_reduce['augassign1'] = 'AST'
+        self.check_reduce['augassign2'] = 'AST'
+        self.check_reduce['while1stmt'] = 'noAST'
+        self.check_reduce['annotate_tuple'] = 'noAST'
+        self.check_reduce['kwarg'] = 'noAST'
+        # FIXME: remove parser errors caused by the below
+        # self.check_reduce['while1elsestmt'] = 'noAST'
         return
 
+    def reduce_is_invalid(self, rule, ast, tokens, first, last):
+        lhs = rule[0]
+        if lhs in ('augassign1', 'augassign2') and ast[0][0] == 'and':
+            return True
+        elif lhs == 'annotate_tuple':
+            return not isinstance(tokens[first].attr, tuple)
+        elif lhs == 'kwarg':
+            return not isinstance(tokens[first].attr, str)
+        elif lhs == 'while1elsestmt':
+            # if SETUP_LOOP target spans the else part, then this is
+            # not while1else. Also do for whileTrue?
+            last += 1
+            while isinstance(tokens[last].offset, str):
+                last += 1
+            return tokens[first].attr == tokens[last].offset
+        elif lhs == 'while1stmt':
+            if tokens[last] in ('COME_FROM_LOOP', 'JUMP_BACK'):
+                # jump_back should be right afer SETUP_LOOP. Test?
+                last += 1
+            while last < len(tokens) and isinstance(tokens[last].offset, str):
+                last += 1
+            if last < len(tokens):
+                offset = tokens[last].offset
+                assert tokens[first] == 'SETUP_LOOP'
+                if offset != tokens[first].attr:
+                    return True
+            return False
+        return False
+
 class Python30Parser(Python3Parser):
 
     def p_30(self, args):

uncompyle6/parsers/parse30.py

@@ -15,6 +15,11 @@ class Python30Parser(Python3Parser):
         stmt         ::= store_locals
         store_locals ::= LOAD_FAST STORE_LOCALS
 
+        # FIXME: combine with parse3.2
+        whileTruestmt     ::= SETUP_LOOP l_stmts_opt JUMP_BACK
+                              COME_FROM_LOOP
+        whileTruestmt     ::= SETUP_LOOP return_stmts
+                              COME_FROM_LOOP
 
         # In many ways Python 3.0 code generation is more like Python 2.6 than
         # it is 2.7 or 3.1. So we have a number of 2.6ish (and before) rules below

uncompyle6/parsers/parse31.py

@@ -36,16 +36,8 @@ class Python31Parser(Python32Parser):
 
     def add_custom_rules(self, tokens, customize):
         super(Python31Parser, self).add_custom_rules(tokens, customize)
-        for i, token in enumerate(tokens):
-            opname = token.type
-            if opname.startswith('MAKE_FUNCTION_A'):
-                args_pos, args_kw, annotate_args  = token.attr
-                # Check that there are 2 annotated params?
-                # rule = ('mkfunc2 ::= %s%sEXTENDED_ARG %s' %
-                #         ('pos_arg ' * (args_pos), 'kwargs ' * (annotate_args-1), opname))
-                rule = ('mkfunc_annotate ::= %s%sannotate_tuple LOAD_CONST EXTENDED_ARG %s' %
-                        (('pos_arg ' * (args_pos)),
-                         ('annotate_arg ' * (annotate_args-1)), opname))
-                self.add_unique_rule(rule, opname, token.attr, customize)
+        return
+    pass
+
 class Python31ParserSingle(Python31Parser, PythonParserSingle):
     pass

uncompyle6/parsers/parse32.py

@@ -10,9 +10,6 @@ from uncompyle6.parsers.parse3 import Python3Parser
 class Python32Parser(Python3Parser):
     def p_32to35(self, args):
         """
-        # In Python 3.2+, DUP_TOPX is DUP_TOP_TWO
-        binary_subscr2 ::= expr expr DUP_TOP_TWO BINARY_SUBSCR
-
         # Store locals is only in Python 3.0 to 3.3
         stmt ::= store_locals
         store_locals ::= LOAD_FAST STORE_LOCALS
@@ -22,6 +19,11 @@ class Python32Parser(Python3Parser):
                               COME_FROM_LOOP
         whileTruestmt     ::= SETUP_LOOP return_stmts
                               COME_FROM_LOOP
+
+        # Python 3.2+ has more loop optimization that removes
+        # JUMP_FORWARD in some cases, and hence we also don't
+        # see COME_FROM
+        _ifstmts_jump ::= c_stmts_opt
         """
     pass
 
@@ -44,6 +46,9 @@ class Python32Parser(Python3Parser):
                         (('pos_arg ' * (args_pos)),
                          ('annotate_arg ' * (annotate_args-1)), opname))
                 self.add_unique_rule(rule, opname, token.attr, customize)
+                pass
+            return
+        pass
 
 
 class Python32ParserSingle(Python32Parser, PythonParserSingle):

uncompyle6/parsers/parse33.py

@@ -19,14 +19,8 @@ class Python33Parser(Python32Parser):
         # actions that want c_stmts_opt at index 1
 
         iflaststmt    ::= testexpr c_stmts_opt33
-        iflaststmtl   ::= testexpr c_stmts_opt
         c_stmts_opt33 ::= JUMP_BACK JUMP_ABSOLUTE c_stmts_opt
         _ifstmts_jump ::= c_stmts_opt JUMP_FORWARD _come_from
-
-        # Python 3.3+ has more loop optimization that removes
-        # JUMP_FORWARD in some cases, and hence we also don't
-        # see COME_FROM
-        _ifstmts_jump ::= c_stmts_opt
         """
 
 class Python33ParserSingle(Python33Parser, PythonParserSingle):

uncompyle6/parsers/parse34.py

@@ -17,8 +17,6 @@ class Python34Parser(Python33Parser):
         """
         # Python 3.4+ optimizes the trailing two JUMPS away
 
-        for_block ::= l_stmts
-
         # Is this 3.4 only?
         yield_from ::= expr GET_ITER LOAD_CONST YIELD_FROM
         """
@@ -42,8 +40,8 @@ if __name__ == '__main__':
             """.split()))
         remain_tokens = set(tokens) - opcode_set
         import re
-        remain_tokens = set([re.sub('_\d+$','', t) for t in remain_tokens])
-        remain_tokens = set([re.sub('_CONT$','', t) for t in remain_tokens])
+        remain_tokens = set([re.sub('_\d+$', '',  t) for t in remain_tokens])
+        remain_tokens = set([re.sub('_CONT$', '', t) for t in remain_tokens])
         remain_tokens = set(remain_tokens) - opcode_set
         print(remain_tokens)
         # print(sorted(p.rule2name.items()))

uncompyle6/parsers/parse35.py

@@ -16,43 +16,122 @@ class Python35Parser(Python34Parser):
 
     def p_35on(self, args):
         """
+        # The number of canned instructions in new statements is mind boggling.
+        # I'm sure by the time Python 4 comes around these will be turned
+        # into special opcodes
+
+        # Python 3.5+ Await statement
+        stmt ::= await_stmt
+        await_stmt ::= call_function GET_AWAITABLE LOAD_CONST YIELD_FROM POP_TOP
+
         # Python 3.5+ has WITH_CLEANUP_START/FINISH
 
-        withstmt ::= expr SETUP_WITH exprlist suite_stmts_opt
-                    POP_BLOCK LOAD_CONST COME_FROM_WITH
-                    WITH_CLEANUP_START WITH_CLEANUP_FINISH END_FINALLY
+        withstmt    ::= expr
+                        SETUP_WITH exprlist suite_stmts_opt
+                        POP_BLOCK LOAD_CONST COME_FROM_WITH
+                        WITH_CLEANUP_START WITH_CLEANUP_FINISH END_FINALLY
 
-        withstmt ::= expr SETUP_WITH POP_TOP suite_stmts_opt
-                     POP_BLOCK LOAD_CONST COME_FROM_WITH
-                     WITH_CLEANUP_START WITH_CLEANUP_FINISH END_FINALLY
+        withstmt   ::= expr
+                       SETUP_WITH POP_TOP suite_stmts_opt
+                       POP_BLOCK LOAD_CONST COME_FROM_WITH
+                       WITH_CLEANUP_START WITH_CLEANUP_FINISH END_FINALLY
+
+        withasstmt ::= expr
+                       SETUP_WITH designator suite_stmts_opt
+                       POP_BLOCK LOAD_CONST COME_FROM_WITH
+                       WITH_CLEANUP_START WITH_CLEANUP_FINISH END_FINALLY
+
+
+        # Python 3.5+ async additions
+
+        stmt            ::= async_with_stmt
+        async_with_stmt ::= expr
+                            BEFORE_ASYNC_WITH GET_AWAITABLE LOAD_CONST YIELD_FROM
+                            SETUP_ASYNC_WITH POP_TOP suite_stmts_opt
+                            POP_BLOCK LOAD_CONST
+                            WITH_CLEANUP_START
+                            GET_AWAITABLE LOAD_CONST YIELD_FROM
+                            WITH_CLEANUP_FINISH END_FINALLY
+
+        stmt               ::= async_with_as_stmt
+        async_with_as_stmt ::= expr
+                               BEFORE_ASYNC_WITH GET_AWAITABLE LOAD_CONST YIELD_FROM
+                               SETUP_ASYNC_WITH designator suite_stmts_opt
+                               POP_BLOCK LOAD_CONST
+                               WITH_CLEANUP_START
+                               GET_AWAITABLE LOAD_CONST YIELD_FROM
+                               WITH_CLEANUP_FINISH END_FINALLY
+
+
+        stmt               ::= async_for_stmt
+        async_for_stmt     ::= SETUP_LOOP expr
+                               GET_AITER
+                               LOAD_CONST YIELD_FROM SETUP_EXCEPT GET_ANEXT LOAD_CONST
+                               YIELD_FROM
+                               designator
+                               POP_BLOCK JUMP_FORWARD COME_FROM_EXCEPT DUP_TOP
+                               LOAD_GLOBAL COMPARE_OP POP_JUMP_IF_FALSE
+                               POP_TOP POP_TOP POP_TOP POP_EXCEPT POP_BLOCK
+                               JUMP_ABSOLUTE END_FINALLY COME_FROM
+                               for_block POP_BLOCK JUMP_ABSOLUTE
+                               opt_come_from_loop
+
+        stmt               ::= async_forelse_stmt
+        async_forelse_stmt ::= SETUP_LOOP expr
+                               GET_AITER
+                               LOAD_CONST YIELD_FROM SETUP_EXCEPT GET_ANEXT LOAD_CONST
+                               YIELD_FROM
+                               designator
+                               POP_BLOCK JUMP_FORWARD COME_FROM_EXCEPT DUP_TOP
+                               LOAD_GLOBAL COMPARE_OP POP_JUMP_IF_FALSE
+                               POP_TOP POP_TOP POP_TOP POP_EXCEPT POP_BLOCK
+                               JUMP_ABSOLUTE END_FINALLY COME_FROM
+                               for_block POP_BLOCK JUMP_ABSOLUTE
+                               else_suite COME_FROM_LOOP
 
-        withasstmt ::= expr SETUP_WITH designator suite_stmts_opt
-                POP_BLOCK LOAD_CONST COME_FROM_WITH
-                WITH_CLEANUP_START WITH_CLEANUP_FINISH END_FINALLY
 
         inplace_op ::= INPLACE_MATRIX_MULTIPLY
         binary_op  ::= BINARY_MATRIX_MULTIPLY
 
         # Python 3.5+ does jump optimization
         # In <.3.5 the below is a JUMP_FORWARD to a JUMP_ABSOLUTE.
-        # in return_stmt, we will need the semantic actions in pysource.py
-        # to work out whether to dedent or not based on the presence of
-        # RETURN_END_IF vs RETURN_VALUE
+
+        return_if_stmt ::= ret_expr RETURN_END_IF POP_BLOCK
 
         ifelsestmtc ::= testexpr c_stmts_opt JUMP_FORWARD else_suitec
+        ifelsestmtc ::= testexpr c_stmts_opt jf_else else_suitec
+
         # ifstmt ::= testexpr c_stmts_opt
 
+        iflaststmt ::= testexpr c_stmts_opt JUMP_FORWARD
+
         # Python 3.3+ also has yield from. 3.5 does it
         # differently than 3.3, 3.4
 
-        expr ::= yield_from
         yield_from ::= expr GET_YIELD_FROM_ITER LOAD_CONST YIELD_FROM
 
-        # Python 3.4+ has more loop optimization that removes
-        # JUMP_FORWARD in some cases, and hence we also don't
-        # see COME_FROM
-        _ifstmts_jump ::= c_stmts_opt
+        _ifstmts_jump ::= c_stmts_opt COME_FROM
         """
+
+    def add_custom_rules(self, tokens, customize):
+        super(Python35Parser, self).add_custom_rules(tokens, customize)
+        for i, token in enumerate(tokens):
+            opname = token.type
+            if opname == 'BUILD_MAP_UNPACK_WITH_CALL':
+                nargs = token.attr % 256
+                map_unpack_n = "map_unpack_%s" % nargs
+                rule = map_unpack_n + ' ::= ' + 'expr ' * (nargs)
+                self.add_unique_rule(rule, opname, token.attr, customize)
+                rule = "unmapexpr ::=  %s %s" % (map_unpack_n, opname)
+                self.add_unique_rule(rule, opname, token.attr, customize)
+                call_token = tokens[i+1]
+                if self.version == 3.5:
+                    rule = 'call_function ::= expr unmapexpr ' + call_token.type
+                    self.add_unique_rule(rule, opname, token.attr, customize)
+                pass
+            pass
+        return
+
 class Python35ParserSingle(Python35Parser, PythonParserSingle):
     pass
 
@@ -72,8 +151,8 @@ if __name__ == '__main__':
             """.split()))
         remain_tokens = set(tokens) - opcode_set
         import re
-        remain_tokens = set([re.sub('_\d+$','', t) for t in remain_tokens])
-        remain_tokens = set([re.sub('_CONT$','', t) for t in remain_tokens])
+        remain_tokens = set([re.sub('_\d+$', '', t) for t in remain_tokens])
+        remain_tokens = set([re.sub('_CONT$', '', t) for t in remain_tokens])
         remain_tokens = set(remain_tokens) - opcode_set
         print(remain_tokens)
         # print(sorted(p.rule2name.items()))

uncompyle6/parsers/parse36.py

@@ -17,8 +17,13 @@ class Python36Parser(Python35Parser):
     def p_36misc(self, args):
         """
         fstring_multi ::= fstring_expr_or_strs BUILD_STRING
-        fstring_expr_or_strs ::= fstring_expr_or_strs fstring_expr_or_str
-        fstring_expr_or_strs ::= fstring_expr_or_str
+        fstring_expr_or_strs ::= fstring_expr_or_str+
+
+        func_args36   ::= expr BUILD_TUPLE_0
+        call_function ::= func_args36 unmapexpr CALL_FUNCTION_EX
+
+        withstmt ::= expr SETUP_WITH POP_TOP suite_stmts_opt POP_BLOCK LOAD_CONST
+                     WITH_CLEANUP_START WITH_CLEANUP_FINISH END_FINALLY
         """
 
     def add_custom_rules(self, tokens, customize):
@@ -47,6 +52,7 @@ class Python36Parser(Python35Parser):
                 """ % (fstring_expr_or_str_n, fstring_expr_or_str_n, "fstring_expr_or_str " * v)
                 self.add_unique_doc_rules(rules_str, customize)
 
+
 class Python36ParserSingle(Python36Parser, PythonParserSingle):
     pass
 
@@ -66,8 +72,8 @@ if __name__ == '__main__':
             """.split()))
         remain_tokens = set(tokens) - opcode_set
         import re
-        remain_tokens = set([re.sub('_\d+$','', t) for t in remain_tokens])
-        remain_tokens = set([re.sub('_CONT$','', t) for t in remain_tokens])
+        remain_tokens = set([re.sub('_\d+$', '', t) for t in remain_tokens])
+        remain_tokens = set([re.sub('_CONT$', '', t) for t in remain_tokens])
         remain_tokens = set(remain_tokens) - opcode_set
         print(remain_tokens)
         # print(sorted(p.rule2name.items()))

uncompyle6/scanner.py

@@ -27,7 +27,8 @@ if PYTHON3:
     intern = sys.intern
     L65536 = 65536
 
-    def long(l): l
+    def long(l):
+        return l
 else:
     L65536 = long(65536) # NOQA
 
@@ -225,9 +226,9 @@ class Scanner(object):
         for given opcode <op>.
         """
         if op < self.opc.HAVE_ARGUMENT:
-            return 1
+            return 2 if self.version >= 3.6 else 1
         else:
-            return 3
+            return 2 if self.version >= 3.6 else 3
 
     def remove_mid_line_ifs(self, ifs):
         """

uncompyle6/scanners/scanner15.py

@@ -23,5 +23,5 @@ class Scanner15(scan.Scanner21):
         self.opc = opcode_15
         self.opname = opcode_15.opname
         self.version = 1.5
-        self.genexpr_name = '<generator expression>';
+        self.genexpr_name = '<generator expression>'
         return

uncompyle6/scanners/scanner2.py

@@ -25,6 +25,7 @@ from __future__ import print_function
 from collections import namedtuple
 from array import array
 
+from uncompyle6.scanner import op_has_argument
 from xdis.code import iscode
 
 import uncompyle6.scanner as scan
@@ -36,7 +37,7 @@ class Scanner2(scan.Scanner):
         self.jump_forward = frozenset([self.opc.JUMP_ABSOLUTE, self.opc.JUMP_FORWARD])
         # This is the 2.5+ default
         # For <2.5 it is <generator expression>
-        self.genexpr_name = '<genexpr>';
+        self.genexpr_name = '<genexpr>'
 
     @staticmethod
     def unmangle_name(name, classname):
@@ -67,7 +68,6 @@ class Scanner2(scan.Scanner):
             varnames = co.co_varnames
         return free, names, varnames
 
-
     def ingest(self, co, classname=None, code_objects={}, show_asm=None):
         """
         Pick out tokens from an uncompyle6 code object, and transform them,
@@ -92,24 +92,19 @@ class Scanner2(scan.Scanner):
             for instr in bytecode.get_instructions(co):
                 print(instr._disassemble())
 
-        # from xdis.bytecode import Bytecode
-        # bytecode = Bytecode(co, self.opc)
-        # for instr in bytecode.get_instructions(co):
-        #     print(instr._disassemble())
-
         # Container for tokens
         tokens = []
 
         customize = {}
         if self.is_pypy:
-            customize['PyPy'] = 1;
+            customize['PyPy'] = 1
 
         Token = self.Token # shortcut
 
-        n = self.setup_code(co)
+        codelen = self.setup_code(co)
 
-        self.build_lines_data(co, n)
-        self.build_prev_op(n)
+        self.build_lines_data(co, codelen)
+        self.build_prev_op(codelen)
 
         free, names, varnames = self.unmangle_code_names(co, classname)
         self.names = names
@@ -118,7 +113,7 @@ class Scanner2(scan.Scanner):
         # turn 'LOAD_GLOBAL' to 'LOAD_ASSERT'.
         # 'LOAD_ASSERT' is used in assert statements.
         self.load_asserts = set()
-        for i in self.op_range(0, n):
+        for i in self.op_range(0, codelen):
             # We need to detect the difference between:
             #   raise AssertionError
             #  and
@@ -137,13 +132,13 @@ class Scanner2(scan.Scanner):
                 if names[self.get_argument(i+3)] == 'AssertionError':
                     self.load_asserts.add(i+3)
 
-        jump_targets = self.find_jump_targets()
+        jump_targets = self.find_jump_targets(show_asm)
         # contains (code, [addrRefToCode])
 
         last_stmt = self.next_stmt[0]
         i = self.next_stmt[last_stmt]
         replace = {}
-        while i < n-1:
+        while i < codelen - 1:
             if self.lines[last_stmt].next > i:
                 # Distinguish "print ..." from "print ...,"
                 if self.code[last_stmt] == self.opc.PRINT_ITEM:
@@ -155,7 +150,7 @@ class Scanner2(scan.Scanner):
             i = self.next_stmt[i]
 
         extended_arg = 0
-        for offset in self.op_range(0, n):
+        for offset in self.op_range(0, codelen):
             if offset in jump_targets:
                 jump_idx = 0
                 # We want to process COME_FROMs to the same offset to be in *descending*
@@ -164,18 +159,30 @@ class Scanner2(scan.Scanner):
                 # we sort them). That way, specific COME_FROM tags will match up
                 # properly. For example, a "loop" with an "if" nested in it should have the
                 # "loop" tag last so the grammar rule matches that properly.
+                # last_offset = -1
                 for jump_offset  in sorted(jump_targets[offset], reverse=True):
+                    # if jump_offset == last_offset:
+                    #     continue
+                    # last_offset = jump_offset
+                    come_from_name = 'COME_FROM'
+                    op_name = self.opc.opname[self.code[jump_offset]]
+                    if op_name.startswith('SETUP_') and self.version == 2.7:
+                        come_from_type = op_name[len('SETUP_'):]
+                        if come_from_type not in ('LOOP', 'EXCEPT'):
+                            come_from_name = 'COME_FROM_%s' % come_from_type
+                        pass
                     tokens.append(Token(
-                        'COME_FROM', None, repr(jump_offset),
+                        come_from_name, None, repr(jump_offset),
                         offset="%s_%d" % (offset, jump_idx),
                         has_arg = True))
                     jump_idx += 1
+                    pass
 
             op = self.code[offset]
-            opname = self.opc.opname[op]
+            op_name = self.opc.opname[op]
 
             oparg = None; pattr = None
-            has_arg = (op >= self.opc.HAVE_ARGUMENT)
+            has_arg = op_has_argument(op, self.opc)
             if has_arg:
                 oparg = self.get_argument(offset) + extended_arg
                 extended_arg = 0
@@ -187,14 +194,14 @@ class Scanner2(scan.Scanner):
                     if iscode(const):
                         oparg = const
                         if const.co_name == '<lambda>':
-                            assert opname == 'LOAD_CONST'
-                            opname = 'LOAD_LAMBDA'
+                            assert op_name == 'LOAD_CONST'
+                            op_name = 'LOAD_LAMBDA'
                         elif const.co_name == '<genexpr>':
-                            opname = 'LOAD_GENEXPR'
+                            op_name = 'LOAD_GENEXPR'
                         elif const.co_name == '<dictcomp>':
-                            opname = 'LOAD_DICTCOMP'
+                            op_name = 'LOAD_DICTCOMP'
                         elif const.co_name == '<setcomp>':
-                            opname = 'LOAD_SETCOMP'
+                            op_name = 'LOAD_SETCOMP'
                         # verify() uses 'pattr' for comparison, since 'attr'
                         # now holds Code(const) and thus can not be used
                         # for comparison (todo: think about changing this)
@@ -230,20 +237,20 @@ class Scanner2(scan.Scanner):
                     self.code[self.prev[offset]] == self.opc.LOAD_CLOSURE:
                     continue
                 else:
-                    if self.is_pypy and not oparg and opname == 'BUILD_MAP':
-                        opname = 'BUILD_MAP_n'
+                    if self.is_pypy and not oparg and op_name == 'BUILD_MAP':
+                        op_name = 'BUILD_MAP_n'
                     else:
-                        opname = '%s_%d' % (opname, oparg)
+                        op_name = '%s_%d' % (op_name, oparg)
                     if op != self.opc.BUILD_SLICE:
-                        customize[opname] = oparg
-            elif self.is_pypy and opname in ('LOOKUP_METHOD',
+                        customize[op_name] = oparg
+            elif self.is_pypy and op_name in ('LOOKUP_METHOD',
                                              'JUMP_IF_NOT_DEBUG',
                                              'SETUP_EXCEPT',
                                              'SETUP_FINALLY'):
                 # The value in the dict is in special cases in semantic actions, such
                 # as CALL_FUNCTION. The value is not used in these cases, so we put
                 # in arbitrary value 0.
-                customize[opname] = 0
+                customize[op_name] = 0
             elif op == self.opc.JUMP_ABSOLUTE:
                 # Further classify JUMP_ABSOLUTE into backward jumps
                 # which are used in loops, and "CONTINUE" jumps which
@@ -262,16 +269,16 @@ class Scanner2(scan.Scanner):
                         and self.code[offset+3] not in (self.opc.END_FINALLY,
                                                         self.opc.POP_BLOCK)
                         and offset not in self.not_continue):
-                        opname = 'CONTINUE'
+                        op_name = 'CONTINUE'
                     else:
-                        opname = 'JUMP_BACK'
+                        op_name = 'JUMP_BACK'
 
             elif op == self.opc.LOAD_GLOBAL:
                 if offset in self.load_asserts:
-                    opname = 'LOAD_ASSERT'
+                    op_name = 'LOAD_ASSERT'
             elif op == self.opc.RETURN_VALUE:
                 if offset in self.return_end_ifs:
-                    opname = 'RETURN_END_IF'
+                    op_name = 'RETURN_END_IF'
 
             if offset in self.linestartoffsets:
                 linestart = self.linestartoffsets[offset]
@@ -280,7 +287,7 @@ class Scanner2(scan.Scanner):
 
             if offset not in replace:
                 tokens.append(Token(
-                    opname, oparg, pattr, offset, linestart, op,
+                    op_name, oparg, pattr, offset, linestart, op,
                     has_arg, self.opc))
             else:
                 tokens.append(Token(
@@ -352,7 +359,7 @@ class Scanner2(scan.Scanner):
             j+=1
         return
 
-    def build_stmt_indices(self):
+    def build_statement_indices(self):
         code = self.code
         start = 0
         end = len(code)
@@ -429,10 +436,10 @@ class Scanner2(scan.Scanner):
         slist += [end] * (end-len(slist))
 
     def next_except_jump(self, start):
-        '''
+        """
         Return the next jump that was generated by an except SomeException:
         construct in a try...except...else clause or None if not found.
-        '''
+        """
 
         if self.code[start] == self.opc.DUP_TOP:
             except_match = self.first_instr(start, len(self.code), self.opc.PJIF)
@@ -443,6 +450,11 @@ class Scanner2(scan.Scanner):
                 if self.version < 2.7 and self.code[jmp] in self.jump_forward:
                     self.not_continue.add(jmp)
                     jmp = self.get_target(jmp)
+                    prev_offset = self.prev[except_match]
+                    # COMPARE_OP argument should be "exception match" or 10
+                    if (self.code[prev_offset] == self.opc.COMPARE_OP and
+                        self.code[prev_offset+1] != 10):
+                        return None
                     if jmp not in self.pop_jump_if | self.jump_forward:
                         self.ignore_if.add(except_match)
                         return None
@@ -466,13 +478,11 @@ class Scanner2(scan.Scanner):
             elif op in self.setup_ops:
                 count_SETUP_ += 1
 
-    def detect_structure(self, pos, op):
-        '''
+    def detect_control_flow(self, offset, op):
+        """
         Detect type of block structures and their boundaries to fix optimized jumps
         in python2.3+
-        '''
-
-        # TODO: check the struct boundaries more precisely -Dan
+        """
 
         code = self.code
 
@@ -480,12 +490,15 @@ class Scanner2(scan.Scanner):
         parent = self.structs[0]
         start  = parent['start']
         end    = parent['end']
+
+        # Pick inner-most parent for our offset
         for struct in self.structs:
-            _start = struct['start']
-            _end   = struct['end']
-            if (_start <= pos < _end) and (_start >= start and _end <= end):
-                start  = _start
-                end    = _end
+            current_start = struct['start']
+            current_end   = struct['end']
+            if ((current_start <= offset < current_end)
+                and (current_start >= start and current_end <= end)):
+                start  = current_start
+                end    = current_end
                 parent = struct
 
         if op == self.opc.SETUP_LOOP:
@@ -495,14 +508,16 @@ class Scanner2(scan.Scanner):
             # Try to find the jump_back instruction of the loop.
             # It could be a return instruction.
 
-            start = pos+3
-            target = self.get_target(pos, op)
+            start = offset+3
+            target = self.get_target(offset, op)
             end    = self.restrict_to_parent(target, parent)
+            self.setup_loop_targets[offset] = target
+            self.setup_loops[target] = offset
 
             if target != end:
-                self.fixed_jumps[pos] = end
+                self.fixed_jumps[offset] = end
 
-            (line_no, next_line_byte) = self.lines[pos]
+            (line_no, next_line_byte) = self.lines[offset]
             jump_back = self.last_instr(start, end, self.opc.JUMP_ABSOLUTE,
                                         next_line_byte, False)
 
@@ -566,10 +581,10 @@ class Scanner2(scan.Scanner):
                 if end > jump_back+4 and code[end] in self.jump_forward:
                     if code[jump_back+4] in self.jump_forward:
                         if self.get_target(jump_back+4) == self.get_target(end):
-                            self.fixed_jumps[pos] = jump_back+4
+                            self.fixed_jumps[offset] = jump_back+4
                             end = jump_back+4
-                elif target < pos:
-                    self.fixed_jumps[pos] = jump_back+4
+                elif target < offset:
+                    self.fixed_jumps[offset] = jump_back+4
                     end = jump_back+4
 
                 target = self.get_target(jump_back, self.opc.JUMP_ABSOLUTE)
@@ -585,7 +600,7 @@ class Scanner2(scan.Scanner):
                     else:
                         test = self.prev[next_line_byte]
 
-                    if test == pos:
+                    if test == offset:
                         loop_type = 'while 1'
                     elif self.code[test] in self.opc.hasjabs + self.opc.hasjrel:
                         self.ignore_if.add(test)
@@ -602,15 +617,15 @@ class Scanner2(scan.Scanner):
                                        'start': jump_back+3,
                                        'end':   end})
         elif op == self.opc.SETUP_EXCEPT:
-            start  = pos+3
-            target = self.get_target(pos, op)
+            start  = offset+3
+            target = self.get_target(offset, op)
             end    = self.restrict_to_parent(target, parent)
             if target != end:
-                self.fixed_jumps[pos] = end
+                self.fixed_jumps[offset] = end
                 # print target, end, parent
             # Add the try block
             self.structs.append({'type':  'try',
-                                   'start': start,
+                                   'start': start-3,
                                    'end':   end-4})
             # Now isolate the except and else blocks
             end_else = start_else = self.get_target(self.prev[end])
@@ -654,15 +669,17 @@ class Scanner2(scan.Scanner):
                 self.fixed_jumps[i] = i+1
 
         elif op in self.pop_jump_if:
-            target = self.get_target(pos, op)
+            target = self.get_target(offset, op)
             rtarget = self.restrict_to_parent(target, parent)
 
             # Do not let jump to go out of parent struct bounds
             if target != rtarget and parent['type'] == 'and/or':
-                self.fixed_jumps[pos] = rtarget
+                self.fixed_jumps[offset] = rtarget
                 return
 
-            start = pos+3
+            jump_if_offset = offset
+
+            start = offset+3
             pre = self.prev
 
             # Does this jump to right after another conditional jump that is
@@ -677,43 +694,47 @@ class Scanner2(scan.Scanner):
                 op_testset = self.pop_jump_if_or_pop | self.pop_jump_if
 
             if ( code[pre[target]] in op_testset
-                 and (target > pos) ):
-                self.fixed_jumps[pos] = pre[target]
+                 and (target > offset) ):
+                self.fixed_jumps[offset] = pre[target]
                 self.structs.append({'type':  'and/or',
                                        'start': start,
                                        'end':   pre[target]})
                 return
 
+            # The op offset just before the target jump offset is important
+            # in making a determination of what we have. Save that.
+            pre_rtarget = pre[rtarget]
+
             # Is it an "and" inside an "if" or "while" block
             if op == self.opc.PJIF:
 
                 # Search for other POP_JUMP_IF_FALSE targetting the same op,
                 # in current statement, starting from current offset, and filter
                 # everything inside inner 'or' jumps and midline ifs
-                match = self.rem_or(start, self.next_stmt[pos], self.opc.PJIF, target)
+                match = self.rem_or(start, self.next_stmt[offset], self.opc.PJIF, target)
 
                 # If we still have any offsets in set, start working on it
                 if match:
-                    if code[pre[rtarget]] in self.jump_forward \
-                            and pre[rtarget] not in self.stmts \
-                            and self.restrict_to_parent(self.get_target(pre[rtarget]), parent) == rtarget:
-                        if code[pre[pre[rtarget]]] == self.opc.JUMP_ABSOLUTE \
-                                and self.remove_mid_line_ifs([pos]) \
-                                and target == self.get_target(pre[pre[rtarget]]) \
-                                and (pre[pre[rtarget]] not in self.stmts or self.get_target(pre[pre[rtarget]]) > pre[pre[rtarget]])\
-                                and 1 == len(self.remove_mid_line_ifs(self.rem_or(start, pre[pre[rtarget]], self.pop_jump_if, target))):
+                    if code[pre_rtarget] in self.jump_forward \
+                            and pre_rtarget not in self.stmts \
+                            and self.restrict_to_parent(self.get_target(pre_rtarget), parent) == rtarget:
+                        if code[pre[pre_rtarget]] == self.opc.JUMP_ABSOLUTE \
+                                and self.remove_mid_line_ifs([offset]) \
+                                and target == self.get_target(pre[pre_rtarget]) \
+                                and (pre[pre_rtarget] not in self.stmts or self.get_target(pre[pre_rtarget]) > pre[pre_rtarget])\
+                                and 1 == len(self.remove_mid_line_ifs(self.rem_or(start, pre[pre_rtarget], self.pop_jump_if, target))):
                             pass
-                        elif code[pre[pre[rtarget]]] == self.opc.RETURN_VALUE \
-                                and self.remove_mid_line_ifs([pos]) \
+                        elif code[pre[pre_rtarget]] == self.opc.RETURN_VALUE \
+                                and self.remove_mid_line_ifs([offset]) \
                                 and 1 == (len(set(self.remove_mid_line_ifs(self.rem_or(start,
-                                                                                       pre[pre[rtarget]],
+                                                                                       pre[pre_rtarget],
                                                                                        self.pop_jump_if, target)))
-                                              | set(self.remove_mid_line_ifs(self.rem_or(start, pre[pre[rtarget]],
-                                                            (self.opc.PJIF, self.opc.PJIT, self.opc.JUMP_ABSOLUTE), pre[rtarget], True))))):
+                                              | set(self.remove_mid_line_ifs(self.rem_or(start, pre[pre_rtarget],
+                                                            (self.opc.PJIF, self.opc.PJIT, self.opc.JUMP_ABSOLUTE), pre_rtarget, True))))):
                             pass
                         else:
                             fix = None
-                            jump_ifs = self.all_instr(start, self.next_stmt[pos], self.opc.PJIF)
+                            jump_ifs = self.all_instr(start, self.next_stmt[offset], self.opc.PJIF)
                             last_jump_good = True
                             for j in jump_ifs:
                                 if target == self.get_target(j):
@@ -722,83 +743,178 @@ class Scanner2(scan.Scanner):
                                         break
                                 else:
                                     last_jump_good = False
-                            self.fixed_jumps[pos] = fix or match[-1]
+                            self.fixed_jumps[offset] = fix or match[-1]
                             return
                     else:
                         if (self.version < 2.7
                             and parent['type'] in ('root', 'for-loop', 'if-then',
-                                                   'if-else', 'try')):
-                            self.fixed_jumps[pos] = rtarget
+                                                   'else', 'try')):
+                            self.fixed_jumps[offset] = rtarget
                         else:
                             # note test for < 2.7 might be superflous although informative
                             # for 2.7 a different branch is taken and the below code is handled
                             # under: elif op in self.pop_jump_if_or_pop
                             # below
-                            self.fixed_jumps[pos] = match[-1]
+                            self.fixed_jumps[offset] = match[-1]
                         return
             else: # op != self.opc.PJIT
-                if self.version < 2.7 and code[pos+3] == self.opc.POP_TOP:
-                    assert_pos = pos + 4
+                if self.version < 2.7 and code[offset+3] == self.opc.POP_TOP:
+                    assert_offset = offset + 4
                 else:
-                    assert_pos = pos + 3
-                if (assert_pos) in self.load_asserts:
-                    if code[pre[rtarget]] == self.opc.RAISE_VARARGS:
+                    assert_offset = offset + 3
+                if (assert_offset) in self.load_asserts:
+                    if code[pre_rtarget] == self.opc.RAISE_VARARGS:
                         return
-                    self.load_asserts.remove(assert_pos)
+                    self.load_asserts.remove(assert_offset)
 
-                next = self.next_stmt[pos]
-                if pre[next] == pos:
+                next = self.next_stmt[offset]
+                if pre[next] == offset:
                     pass
                 elif code[next] in self.jump_forward and target == self.get_target(next):
                     if code[pre[next]] == self.opc.PJIF:
-                        if code[next] == self.opc.JUMP_FORWARD or target != rtarget or code[pre[pre[rtarget]]] not in (self.opc.JUMP_ABSOLUTE, self.opc.RETURN_VALUE):
-                            self.fixed_jumps[pos] = pre[next]
+                        if code[next] == self.opc.JUMP_FORWARD or target != rtarget or code[pre[pre_rtarget]] not in (self.opc.JUMP_ABSOLUTE, self.opc.RETURN_VALUE):
+                            self.fixed_jumps[offset] = pre[next]
                             return
                 elif code[next] == self.opc.JUMP_ABSOLUTE and code[target] in self.jump_forward:
                     next_target = self.get_target(next)
                     if self.get_target(target) == next_target:
-                        self.fixed_jumps[pos] = pre[next]
+                        self.fixed_jumps[offset] = pre[next]
                         return
                     elif code[next_target] in self.jump_forward and self.get_target(next_target) == self.get_target(target):
-                        self.fixed_jumps[pos] = pre[next]
+                        self.fixed_jumps[offset] = pre[next]
                         return
 
             # don't add a struct for a while test, it's already taken care of
-            if pos in self.ignore_if:
+            if offset in self.ignore_if:
                 return
 
-            if code[pre[rtarget]] == self.opc.JUMP_ABSOLUTE and pre[rtarget] in self.stmts \
-                    and pre[rtarget] != pos and pre[pre[rtarget]] != pos:
-                if code[rtarget] == self.opc.JUMP_ABSOLUTE and code[rtarget+3] == self.opc.POP_BLOCK:
-                    if code[pre[pre[rtarget]]] != self.opc.JUMP_ABSOLUTE:
-                        pass
-                    elif self.get_target(pre[pre[rtarget]]) != target:
-                        pass
+            if self.version == 2.7:
+                if code[pre_rtarget] == self.opc.JUMP_ABSOLUTE and pre_rtarget in self.stmts \
+                        and pre_rtarget != offset and pre[pre_rtarget] != offset:
+                    if code[rtarget] == self.opc.JUMP_ABSOLUTE and code[rtarget+3] == self.opc.POP_BLOCK:
+                        if code[pre[pre_rtarget]] != self.opc.JUMP_ABSOLUTE:
+                            pass
+                        elif self.get_target(pre[pre_rtarget]) != target:
+                            pass
+                        else:
+                            rtarget = pre_rtarget
                     else:
-                        rtarget = pre[rtarget]
-                else:
-                    rtarget = pre[rtarget]
+                        rtarget = pre_rtarget
 
-            # Does the "if" jump just beyond a jump op, then this is probably an if statement
-            pre_rtarget = pre[rtarget]
+            # Does the "jump if" jump beyond a jump op?
+            # That is, we have something like:
+            #  POP_JUMP_IF_FALSE HERE
+            #  ...
+            # JUMP_FORWARD
+            # HERE:
+            #
+            # If so, this can be block inside an "if" statement
+            # or a conditional assignment like:
+            #   x = 1 if x else 2
+            #
+            # There are other contexts we may need to consider
+            # like whether the target is "END_FINALLY"
+            # or if the condition jump is to a forward location
             code_pre_rtarget = code[pre_rtarget]
+
             if code_pre_rtarget in self.jump_forward:
                 if_end = self.get_target(pre_rtarget)
 
                 # Is this a loop and not an "if" statment?
-                if (if_end < pre_rtarget) and (code[pre[if_end]] == self.opc.SETUP_LOOP):
-                    if(if_end > start):
+                if (if_end < pre_rtarget) and (pre[if_end] in self.setup_loop_targets):
+
+                    if (if_end > start):
                         return
+                    else:
+                        # We still have the case in 2.7 that the next instruction
+                        # is a jump to a SETUP_LOOP target.
+                        next_offset = target + self.op_size(self.code[target])
+                        next_op = self.code[next_offset]
+                        if self.opc.opname[next_op] == 'JUMP_FORWARD':
+                            jump_target = self.get_target(next_offset, next_op)
+                            if jump_target in self.setup_loops:
+                                self.structs.append({'type':  'while-loop',
+                                       'start': jump_if_offset,
+                                       'end':   jump_target})
+                                self.fixed_jumps[jump_if_offset] = jump_target
+                                return
 
                 end = self.restrict_to_parent(if_end, parent)
 
-                self.structs.append({'type':  'if-then',
-                                       'start': start,
-                                       'end':   pre_rtarget})
+                if_then_maybe = None
+
+                if 2.2 <= self.version <= 2.6:
+                    # Take the JUMP_IF target. In an "if/then", it will be
+                    # a POP_TOP instruction and the instruction before it
+                    # will be a JUMP_FORWARD to just after the POP_TOP.
+                    # For example:
+                    # Good:
+                    # 3  JUMP_IF_FALSE        33  'to 39'
+                    # ..
+                    # 36  JUMP_FORWARD          1  'to 40'
+                    # 39  POP_TOP
+                    # 40 ...
+                    # example:
+
+                    # BAD (is an "and"):
+                    # 28  JUMP_IF_FALSE         4  'to 35'
+                    # ...
+                    # 32  JUMP_ABSOLUTE        40  'to 40' # should be 36 or there should
+                    #                                      # be a COME_FROM at the pop top
+                    #                                      # before 40 to 35
+                    # 35  POP_TOP
+                    # 36 ...
+                    # 39  POP_TOP
+                    # 39_0  COME_FROM 3
+                    # 40 ...
+
+                    if self.opc.opname[code[jump_if_offset]].startswith('JUMP_IF'):
+                        jump_if_target = code[jump_if_offset+1]
+                        if self.opc.opname[code[jump_if_target + jump_if_offset + 3]] == 'POP_TOP':
+                            jump_inst = jump_if_target + jump_if_offset
+                            jump_offset = code[jump_inst+1]
+                            jump_op = self.opc.opname[code[jump_inst]]
+                            if (jump_op == 'JUMP_FORWARD' and jump_offset == 1):
+                                self.structs.append({'type':  'if-then',
+                                                     'start': start-3,
+                                                     'end':   pre_rtarget})
+
+                                self.thens[start] = end
+                            elif jump_op == 'JUMP_ABSOLUTE':
+                                if_then_maybe = {'type':  'if-then',
+                                                 'start': start-3,
+                                                 'end':   pre_rtarget}
+
+                elif self.version == 2.7:
+                    self.structs.append({'type':  'if-then',
+                                         'start': start-3,
+                                         'end':   pre_rtarget})
+
                 self.not_continue.add(pre_rtarget)
 
                 if rtarget < end:
-                    self.structs.append({'type':  'if-else',
+                    # We have an "else" block  of some kind.
+                    # Is it associated with "if_then_maybe" seen above?
+                    # These will be linked in this funny way:
+
+                    # 198  JUMP_IF_FALSE        18  'to 219'
+                    # 201  POP_TOP
+                    # ...
+                    # 216  JUMP_ABSOLUTE       256  'to 256'
+                    # 219  POP_TOP
+                    # ...
+                    # 252  JUMP_FORWARD          1  'to 256'
+                    # 255  POP_TOP
+                    # 256
+                    if if_then_maybe and jump_op == 'JUMP_ABSOLUTE':
+                        jump_target = self.get_target(jump_inst, code[jump_inst])
+                        if self.opc.opname[code[end]] == 'JUMP_FORWARD':
+                            end_target = self.get_target(end, code[end])
+                            if jump_target == end_target:
+                                self.structs.append(if_then_maybe)
+                                self.thens[start] = end
+
+                    self.structs.append({'type':  'else',
                                        'start': rtarget,
                                        'end':   end})
             elif code_pre_rtarget == self.opc.RETURN_VALUE:
@@ -806,50 +922,63 @@ class Scanner2(scan.Scanner):
                     self.structs.append({'type':  'if-then',
                                            'start': start,
                                            'end':   rtarget})
+                    self.thens[start] = rtarget
                     if self.version == 2.7 or code[pre_rtarget+1] != self.opc.JUMP_FORWARD:
                         self.return_end_ifs.add(pre_rtarget)
 
         elif op in self.pop_jump_if_or_pop:
-            target = self.get_target(pos, op)
-            self.fixed_jumps[pos] = self.restrict_to_parent(target, parent)
+            target = self.get_target(offset, op)
+            self.fixed_jumps[offset] = self.restrict_to_parent(target, parent)
 
-    def find_jump_targets(self):
-        '''
+    def find_jump_targets(self, debug):
+        """
         Detect all offsets in a byte code which are jump targets
-        where we might insert a COME_FROM instruction.
+        where we might insert a pseudo "COME_FROM" instruction.
+        "COME_FROM" instructions are used in detecting overall
+        control flow. The more detailed information about the
+        control flow is captured in self.structs.
+        Since this stuff is tricky, consult self.structs when
+        something goes amiss.
 
         Return the list of offsets. An instruction can be jumped
         to in from multiple instructions.
-        '''
-
-        n = len(self.code)
+        """
+        code = self.code
+        n = len(code)
         self.structs = [{'type':  'root',
                            'start': 0,
                            'end':   n-1}]
-        self.loops = []  # All loop entry points
-        self.fixed_jumps = {} # Map fixed jumps to their real destination
-        self.ignore_if = set()
-        self.build_stmt_indices()
+        # All loop entry points
+        self.loops = []
 
-        # Containers filled by detect_structure()
+        # Map fixed jumps to their real destination
+        self.fixed_jumps = {}
+        self.ignore_if = set()
+        self.build_statement_indices()
+
+        # Containers filled by detect_control_flow()
         self.not_continue = set()
         self.return_end_ifs = set()
+        self.setup_loop_targets = {}  # target given setup_loop offset
+        self.setup_loops = {}  # setup_loop offset given target
+        self.thens = {} # JUMP_IF's that separate the 'then' part of an 'if'
 
         targets = {}
         for offset in self.op_range(0, n):
-            op = self.code[offset]
+            op = code[offset]
 
             # Determine structures and fix jumps in Python versions
             # since 2.3
-            self.detect_structure(offset, op)
+            self.detect_control_flow(offset, op)
 
-            if op >= self.opc.HAVE_ARGUMENT:
+            if op_has_argument(op, self.opc):
                 label = self.fixed_jumps.get(offset)
                 oparg = self.get_argument(offset)
 
                 if label is None:
-                    if (op in self.opc.hasjrel and
-                        (self.version < 2.0 or op != self.opc.FOR_ITER)):
+                    if op in self.opc.hasjrel and self.opc.opname[op] != 'FOR_ITER':
+                        # if (op in self.opc.hasjrel and
+                        #     (self.version < 2.0 or op != self.opc.FOR_ITER)):
                         label = offset + 3 + oparg
                     elif self.version == 2.7 and op in self.opc.hasjabs:
                         if op in (self.opc.JUMP_IF_FALSE_OR_POP,
@@ -859,7 +988,6 @@ class Scanner2(scan.Scanner):
                                 pass
                             pass
 
-
                 # FIXME: All the < 2.7 conditions are is horrible. We need a better way.
                 if label is not None and label != -1:
                     # In Python < 2.7, the POP_TOP in:
@@ -867,23 +995,38 @@ class Scanner2(scan.Scanner):
                     # does now start a new statement
                     # Otherwise, we have want to add a "COME_FROM"
                     if not (self.version < 2.7 and
-                            self.code[label] == self.opc.POP_TOP and
-                            self.code[self.prev[label]] == self.opc.RETURN_VALUE):
+                            code[label] == self.opc.POP_TOP and
+                            code[self.prev[label]] == self.opc.RETURN_VALUE):
                         # In Python < 2.7, don't add a COME_FROM, for:
                         #     JUMP_FORWARD, END_FINALLY
                         # or:
                         #     JUMP_FORWARD, POP_TOP, END_FINALLY
                         if not (self.version < 2.7 and op == self.opc.JUMP_FORWARD
-                                and ((self.code[offset+3] == self.opc.END_FINALLY)
-                                     or (self.code[offset+3] == self.opc.POP_TOP
-                                         and self.code[offset+4] == self.opc.END_FINALLY))):
-                            targets[label] = targets.get(label, []) + [offset]
+                                and ((code[offset+3] == self.opc.END_FINALLY)
+                                     or (code[offset+3] == self.opc.POP_TOP
+                                         and code[offset+4] == self.opc.END_FINALLY))):
+
+                            # FIXME: rocky: I think we need something like this...
+                            if offset not in set(self.ignore_if) or self.version == 2.7:
+                                source = (self.setup_loops[label]
+                                          if label in self.setup_loops else offset)
+                                targets[label] = targets.get(label, []) + [source]
                             pass
+
                         pass
                     pass
             elif op == self.opc.END_FINALLY and offset in self.fixed_jumps and self.version == 2.7:
                 label = self.fixed_jumps[offset]
                 targets[label] = targets.get(label, []) + [offset]
+                pass
+            pass
+
+        # DEBUG:
+        if debug in ('both', 'after'):
+            print(targets)
+            import pprint as pp
+            pp.pprint(self.structs)
+
         return targets
 
     # FIXME: combine with scanner3.py code and put into scanner.py

uncompyle6/scanners/scanner21.py

@@ -23,5 +23,5 @@ class Scanner21(scan.Scanner22):
         self.opc = opcode_21
         self.opname = opcode_21.opname
         self.version = 2.1
-        self.genexpr_name = '<generator expression>';
+        self.genexpr_name = '<generator expression>'
         return

uncompyle6/scanners/scanner22.py

@@ -23,7 +23,7 @@ class Scanner22(scan.Scanner23):
         self.opc = opcode_22
         self.opname = opcode_22.opname
         self.version = 2.2
-        self.genexpr_name = '<generator expression>';
+        self.genexpr_name = '<generator expression>'
         self.parent_ingest = self.ingest
         self.ingest = self.ingest22
         return

uncompyle6/scanners/scanner23.py

@@ -2,9 +2,8 @@
 """
 Python 2.3 bytecode scanner/deparser
 
-This overlaps Python's 2.3's dis module, but it can be run from
-Python 3 and other versions of Python. Also, we save token
-information for later use in deparsing.
+This massages tokenized 2.3 bytecode to make it more amenable for
+grammar parsing.
 """
 
 import uncompyle6.scanners.scanner24 as scan
@@ -13,17 +12,17 @@ import uncompyle6.scanners.scanner24 as scan
 from xdis.opcodes import opcode_23
 JUMP_OPs = opcode_23.JUMP_OPs
 
-# We base this off of 2.5 instead of the other way around
+# We base this off of 2.4 instead of the other way around
 # because we cleaned things up this way.
 # The history is that 2.7 support is the cleanest,
 # then from that we got 2.6 and so on.
 class Scanner23(scan.Scanner24):
-    def __init__(self, show_asm):
+    def __init__(self, show_asm=False):
         scan.Scanner24.__init__(self, show_asm)
         self.opc = opcode_23
         self.opname = opcode_23.opname
         # These are the only differences in initialization between
         # 2.3-2.6
         self.version = 2.3
-        self.genexpr_name = '<generator expression>';
+        self.genexpr_name = '<generator expression>'
         return

uncompyle6/scanners/scanner24.py

@@ -2,9 +2,8 @@
 """
 Python 2.4 bytecode scanner/deparser
 
-This overlaps Python's 2.4's dis module, but it can be run from
-Python 3 and other versions of Python. Also, we save token
-information for later use in deparsing.
+This massages tokenized 2.7 bytecode to make it more amenable for
+grammar parsing.
 """
 
 import uncompyle6.scanners.scanner25 as scan
@@ -18,12 +17,12 @@ JUMP_OPs = opcode_24.JUMP_OPs
 # The history is that 2.7 support is the cleanest,
 # then from that we got 2.6 and so on.
 class Scanner24(scan.Scanner25):
-    def __init__(self, show_asm):
+    def __init__(self, show_asm=False):
         scan.Scanner25.__init__(self, show_asm)
         # These are the only differences in initialization between
         # 2.4, 2.5 and 2.6
         self.opc = opcode_24
         self.opname = opcode_24.opname
         self.version = 2.4
-        self.genexpr_name = '<generator expression>';
+        self.genexpr_name = '<generator expression>'
         return

uncompyle6/scanners/scanner25.py

@@ -18,7 +18,7 @@ JUMP_OPs = opcode_25.JUMP_OPs
 # The history is that 2.7 support is the cleanest,
 # then from that we got 2.6 and so on.
 class Scanner25(scan.Scanner26):
-    def __init__(self, show_asm):
+    def __init__(self, show_asm=False):
         # There are no differences in initialization between
         # 2.5 and 2.6
         self.opc = opcode_25

uncompyle6/scanners/scanner26.py

@@ -94,35 +94,32 @@ class Scanner26(scan.Scanner2):
             for instr in bytecode.get_instructions(co):
                 print(instr._disassemble())
 
-        # from xdis.bytecode import Bytecode
-        # bytecode = Bytecode(co, self.opc)
-        # for instr in bytecode.get_instructions(co):
-        #     print(instr._disassemble())
-
         # Container for tokens
         tokens = []
 
         customize = {}
+        if self.is_pypy:
+            customize['PyPy'] = 1
+
         Token = self.Token # shortcut
 
-        n = self.setup_code(co)
+        codelen = self.setup_code(co)
 
-        self.build_lines_data(co, n)
-        self.build_prev_op(n)
+        self.build_lines_data(co, codelen)
+        self.build_prev_op(codelen)
 
         free, names, varnames = self.unmangle_code_names(co, classname)
         self.names = names
 
-        codelen = len(self.code)
-
         # Scan for assertions. Later we will
         # turn 'LOAD_GLOBAL' to 'LOAD_ASSERT'.
         # 'LOAD_ASSERT' is used in assert statements.
         self.load_asserts = set()
-        for i in self.op_range(0, n):
-            # We need to detect the difference between
-            # "raise AssertionError" and
-            # "assert"
+        for i in self.op_range(0, codelen):
+            # We need to detect the difference between:
+            #   raise AssertionError
+            #  and
+            #   assert ...
             if (self.code[i] == self.opc.JUMP_IF_TRUE and
                 i + 4 < codelen and
                 self.code[i+3] == self.opc.POP_TOP and
@@ -130,7 +127,7 @@ class Scanner26(scan.Scanner2):
                 if names[self.get_argument(i+4)] == 'AssertionError':
                     self.load_asserts.add(i+4)
 
-        jump_targets = self.find_jump_targets()
+        jump_targets = self.find_jump_targets(show_asm)
         # contains (code, [addrRefToCode])
 
         last_stmt = self.next_stmt[0]
@@ -167,6 +164,11 @@ class Scanner26(scan.Scanner2):
                         offset="%s_%d" % (offset, jump_idx),
                         has_arg = True))
                     jump_idx += 1
+            elif offset in self.thens:
+                tokens.append(Token(
+                    'THEN', None, self.thens[offset],
+                    offset="%s_0" % offset,
+                    has_arg = True))
 
             has_arg = (op >= self.opc.HAVE_ARGUMENT)
             if has_arg:
@@ -233,7 +235,7 @@ class Scanner26(scan.Scanner2):
                     if op != self.opc.BUILD_SLICE:
                         customize[op_name] = oparg
             elif op == self.opc.JUMP_ABSOLUTE:
-                # Further classifhy JUMP_ABSOLUTE into backward jumps
+                # Further classify JUMP_ABSOLUTE into backward jumps
                 # which are used in loops, and "CONTINUE" jumps which
                 # may appear in a "continue" statement.  The loop-type
                 # and continue-type jumps will help us classify loop
@@ -254,6 +256,9 @@ class Scanner26(scan.Scanner2):
                         #   if x: continue
                         # the "continue" is not on a new line.
                         if tokens[-1].type == 'JUMP_BACK':
+                            # We need 'intern' since we have
+                            # already have processed the previous
+                            # token.
                             tokens[-1].type = intern('CONTINUE')
 
             elif op == self.opc.LOAD_GLOBAL:
@@ -279,9 +284,9 @@ class Scanner26(scan.Scanner2):
                 pass
             pass
 
-        if show_asm:
+        if show_asm in ('both', 'after'):
             for t in tokens:
-                print(t)
+                print(t.format(line_prefix='L.'))
             print()
         return tokens, customize
 

uncompyle6/scanners/scanner3.py

@@ -128,7 +128,6 @@ class Scanner3(Scanner):
             varargs_ops.add(self.opc.CALL_METHOD)
         self.varargs_ops = frozenset(varargs_ops)
 
-
     def opName(self, offset):
         return self.opc.opname[self.code[offset]]
 
@@ -149,7 +148,7 @@ class Scanner3(Scanner):
         """
 
         show_asm = self.show_asm if not show_asm else show_asm
-        # show_asm = 'after'
+        # show_asm = 'both'
         if show_asm in ('both', 'before'):
             bytecode = Bytecode(co, self.opc)
             for instr in bytecode.get_instructions(co):
@@ -160,7 +159,7 @@ class Scanner3(Scanner):
 
         customize = {}
         if self.is_pypy:
-            customize['PyPy'] = 1;
+            customize['PyPy'] = 1
 
         self.code = array('B', co.co_code)
         self.build_lines_data(co)
@@ -199,7 +198,8 @@ class Scanner3(Scanner):
 
         # Get jump targets
         # Format: {target offset: [jump offsets]}
-        jump_targets = self.find_jump_targets()
+        jump_targets = self.find_jump_targets(show_asm)
+        last_op_was_break = False
 
         for inst in bytecode:
 
@@ -226,6 +226,14 @@ class Scanner3(Scanner):
                     jump_idx += 1
                     pass
                 pass
+            elif inst.offset in self.else_start:
+                end_offset = self.else_start[inst.offset]
+                tokens.append(Token('ELSE',
+                                    None, repr(end_offset),
+                                    offset='%s' % (inst.offset),
+                                    has_arg = True, opc=self.opc))
+
+                pass
 
             pattr =  inst.argrepr
             opname = inst.opname
@@ -314,27 +322,38 @@ class Scanner3(Scanner):
                 if target <= inst.offset:
                     next_opname = self.opname[self.code[inst.offset+3]]
                     if (inst.offset in self.stmts and
-                        next_opname not in ('END_FINALLY', 'POP_BLOCK',
+                        (self.version != 3.0 or (hasattr(inst, 'linestart'))) and
+                        (next_opname not in ('END_FINALLY', 'POP_BLOCK',
                                             # Python 3.0 only uses POP_TOP
-                                            'POP_TOP')
-                        and inst.offset not in self.not_continue):
-                        opname = 'CONTINUE'
+                                            'POP_TOP'))):
+                        if (self.version >= 3.5 or
+                            (inst.offset not in self.not_continue) or
+                            (tokens[-1].type == 'RETURN_VALUE')):
+                            opname = 'CONTINUE'
+                            pass
                     else:
                         opname = 'JUMP_BACK'
                         # FIXME: this is a hack to catch stuff like:
                         #   if x: continue
                         # the "continue" is not on a new line.
-                        # There are other situations were we don't catch
+                        # There are other situations where we don't catch
                         # CONTINUE as well.
-                        if tokens[-1].type == 'JUMP_BACK':
-                            tokens[-1].type = intern('CONTINUE')
-
+                        if tokens[-1].type == 'JUMP_BACK' and tokens[-1].attr <= argval:
+                            if tokens[-2].type == 'BREAK_LOOP':
+                                del tokens[-1]
+                            else:
+                                # intern is used because we are changing the *previous* token
+                                tokens[-1].type = intern('CONTINUE')
+                    if last_op_was_break and opname == 'CONTINUE':
+                        last_op_was_break = False
+                        continue
             elif op == self.opc.RETURN_VALUE:
                 if inst.offset in self.return_end_ifs:
                     opname = 'RETURN_END_IF'
             elif inst.offset in self.load_asserts:
                 opname = 'LOAD_ASSERT'
 
+            last_op_was_break = opname == 'BREAK_LOOP'
             tokens.append(
                 Token(
                     type_ = opname,
@@ -401,14 +420,15 @@ class Scanner3(Scanner):
             for _ in range(self.op_size(op)):
                 self.prev_op.append(offset)
 
-    def find_jump_targets(self):
+    def find_jump_targets(self, debug):
         """
-        Detect all offsets in a byte code which are jump targets.
+        Detect all offsets in a byte code which are jump targets
+        where we might insert a COME_FROM instruction.
 
         Return the list of offsets.
 
-        This procedure is modelled after dis.findlabels(), but here
-        for each target the number of jumps is counted.
+        Return the list of offsets. An instruction can be jumped
+        to in from multiple instructions.
         """
         code = self.code
         n = len(code)
@@ -423,10 +443,13 @@ class Scanner3(Scanner):
         self.fixed_jumps = {}
         self.ignore_if = set()
         self.build_statement_indices()
+        self.else_start = {}
 
-        # Containers filled by detect_structure()
+        # Containers filled by detect_control_flow()
         self.not_continue = set()
         self.return_end_ifs = set()
+        self.setup_loop_targets = {}  # target given setup_loop offset
+        self.setup_loops = {}  # setup_loop offset given target
 
         targets = {}
         for offset in self.op_range(0, n):
@@ -434,16 +457,20 @@ class Scanner3(Scanner):
 
             # Determine structures and fix jumps in Python versions
             # since 2.3
-            self.detect_structure(offset, targets)
+            self.detect_control_flow(offset, targets)
 
             has_arg = (op >= op3.HAVE_ARGUMENT)
             if has_arg:
                 label = self.fixed_jumps.get(offset)
-                oparg = code[offset+1] + code[offset+2] * 256
+                if self.version >= 3.6:
+                    oparg = code[offset+1]
+                else:
+                    oparg = code[offset+1] + code[offset+2] * 256
+                next_offset = offset + self.op_size(op)
 
                 if label is None:
                     if op in op3.hasjrel and op != self.opc.FOR_ITER:
-                        label = offset + 3 + oparg
+                        label = next_offset + oparg
                     elif op in op3.hasjabs:
                         if op in self.jump_if_pop:
                             if oparg > offset:
@@ -454,6 +481,13 @@ class Scanner3(Scanner):
             elif op == self.opc.END_FINALLY and offset in self.fixed_jumps:
                 label = self.fixed_jumps[offset]
                 targets[label] = targets.get(label, []) + [offset]
+                pass
+            pass
+        # DEBUG:
+        if debug in ('both', 'after'):
+            import pprint as pp
+            pp.pprint(self.structs)
+
         return targets
 
     def build_statement_indices(self):
@@ -466,7 +500,7 @@ class Scanner3(Scanner):
         prelim = self.all_instr(start, end, self.statement_opcodes)
 
         # Initialize final container with statements with
-        # preliminnary data
+        # preliminary data
         stmts = self.stmts = set(prelim)
 
         # Same for opcode sequences
@@ -543,12 +577,18 @@ class Scanner3(Scanner):
         Get target offset for op located at given <offset>.
         """
         op = self.code[offset]
-        target = self.code[offset+1] + self.code[offset+2] * 256
-        if op in op3.hasjrel:
-            target += offset + 3
+        if self.version  >= 3.6:
+            target = self.code[offset+1]
+            if op in self.opc.hasjrel:
+                target += offset + 2
+        else:
+            target = self.code[offset+1] + self.code[offset+2] * 256
+            if op in self.opc.hasjrel:
+                target += offset + 3
+
         return target
 
-    def detect_structure(self, offset, targets):
+    def detect_control_flow(self, offset, targets):
         """
         Detect structures and their boundaries to fix optimized jumps
         in python2.3+
@@ -575,36 +615,53 @@ class Scanner3(Scanner):
                 parent = struct
 
         if op == self.opc.SETUP_LOOP:
-
             # We categorize loop types: 'for', 'while', 'while 1' with
             # possibly suffixes '-loop' and '-else'
             # Try to find the jump_back instruction of the loop.
             # It could be a return instruction.
 
-            start = offset+3
+            if self.version <= 3.5:
+                start = offset+3
+            else:
+                start = offset+2
             target = self.get_target(offset)
             end    = self.restrict_to_parent(target, parent)
+            self.setup_loop_targets[offset] = target
+            self.setup_loops[target] = offset
 
             if target != end:
                 self.fixed_jumps[offset] = end
             (line_no, next_line_byte) = self.lines[offset]
             jump_back = self.last_instr(start, end, self.opc.JUMP_ABSOLUTE,
-                                          next_line_byte, False)
+                                            next_line_byte, False)
 
-            if jump_back and jump_back != self.prev_op[end] and self.is_jump_forward(jump_back+3):
-                if (code[self.prev_op[end]] == self.opc.RETURN_VALUE
-                    or (code[self.prev_op[end]] == self.opc.POP_BLOCK
-                         and code[self.prev_op[self.prev_op[end]]] == self.opc.RETURN_VALUE)):
+            if jump_back:
+                jump_forward_offset = jump_back+3
+            else:
+                jump_forward_offset = None
+
+            return_val_offset1 = self.prev[self.prev[end]]
+
+            if (jump_back and jump_back != self.prev_op[end]
+                and self.is_jump_forward(jump_forward_offset)):
+                if (code[self.prev_op[end]] == self.opc.RETURN_VALUE or
+                    (code[self.prev_op[end]] == self.opc.POP_BLOCK
+                     and code[return_val_offset1] == self.opc.RETURN_VALUE)):
                     jump_back = None
-            if not jump_back: # loop suite ends in return. wtf right?
-                jump_back = self.last_instr(start, end, self.opc.RETURN_VALUE) + 1
+            if not jump_back:
+                jump_back = self.last_instr(start, end, self.opc.RETURN_VALUE)
                 if not jump_back:
                     return
+
+                jump_back += 2
+                if_offset = None
                 if code[self.prev_op[next_line_byte]] not in POP_JUMP_TF:
-                    loop_type = 'for'
-                else:
+                    if_offset = self.prev[next_line_byte]
+                if if_offset:
                     loop_type = 'while'
-                    self.ignore_if.add(self.prev_op[next_line_byte])
+                    self.ignore_if.add(if_offset)
+                else:
+                    loop_type = 'for'
                 target = next_line_byte
                 end = jump_back + 3
             else:
@@ -618,6 +675,7 @@ class Scanner3(Scanner):
                 elif target < offset:
                     self.fixed_jumps[offset] = jump_back+4
                     end = jump_back+4
+
                 target = self.get_target(jump_back)
 
                 if code[target] in (self.opc.FOR_ITER, self.opc.GET_ITER):
@@ -625,6 +683,7 @@ class Scanner3(Scanner):
                 else:
                     loop_type = 'while'
                     test = self.prev_op[next_line_byte]
+
                     if test == offset:
                         loop_type = 'while 1'
                     elif self.code[test] in op3.hasjabs+op3.hasjrel:
@@ -665,38 +724,40 @@ class Scanner3(Scanner):
                                      'end': prev_op[target]})
                 return
 
-            # Is it an "and" inside an "if" block
+            # The op offset just before the target jump offset is important
+            # in making a determination of what we have. Save that.
+            pre_rtarget = prev_op[rtarget]
+
+            # Is it an "and" inside an "if" or "while" block
             if op == self.opc.POP_JUMP_IF_FALSE:
+
                 # Search for another POP_JUMP_IF_FALSE targetting the same op,
                 # in current statement, starting from current offset, and filter
                 # everything inside inner 'or' jumps and midline ifs
                 match = self.rem_or(start, self.next_stmt[offset],
                                     self.opc.POP_JUMP_IF_FALSE, target)
-                ## We can't remove mid-line ifs because line structures have changed
-                ## from restructBytecode().
-                ##  match = self.remove_mid_line_ifs(match)
 
                 # If we still have any offsets in set, start working on it
                 if match:
-                    is_jump_forward = self.is_jump_forward(prev_op[rtarget])
-                    if (is_jump_forward and prev_op[rtarget] not in self.stmts and
-                        self.restrict_to_parent(self.get_target(prev_op[rtarget]), parent) == rtarget):
-                        if (code[prev_op[prev_op[rtarget]]] == self.opc.JUMP_ABSOLUTE
+                    is_jump_forward = self.is_jump_forward(pre_rtarget)
+                    if (is_jump_forward and pre_rtarget not in self.stmts and
+                        self.restrict_to_parent(self.get_target(pre_rtarget), parent) == rtarget):
+                        if (code[prev_op[pre_rtarget]] == self.opc.JUMP_ABSOLUTE
                             and self.remove_mid_line_ifs([offset]) and
-                            target == self.get_target(prev_op[prev_op[rtarget]]) and
-                            (prev_op[prev_op[rtarget]] not in self.stmts or
-                             self.get_target(prev_op[prev_op[rtarget]]) > prev_op[prev_op[rtarget]]) and
-                            1 == len(self.remove_mid_line_ifs(self.rem_or(start, prev_op[prev_op[rtarget]], POP_JUMP_TF, target)))):
+                            target == self.get_target(prev_op[pre_rtarget]) and
+                            (prev_op[pre_rtarget] not in self.stmts or
+                             self.get_target(prev_op[pre_rtarget]) > prev_op[pre_rtarget]) and
+                            1 == len(self.remove_mid_line_ifs(self.rem_or(start, prev_op[pre_rtarget], POP_JUMP_TF, target)))):
                             pass
-                        elif (code[prev_op[prev_op[rtarget]]] == self.opc.RETURN_VALUE
+                        elif (code[prev_op[pre_rtarget]] == self.opc.RETURN_VALUE
                               and self.remove_mid_line_ifs([offset]) and
-                              1 == (len(set(self.remove_mid_line_ifs(self.rem_or(start, prev_op[prev_op[rtarget]],
+                              1 == (len(set(self.remove_mid_line_ifs(self.rem_or(start, prev_op[pre_rtarget],
                                                                                  POP_JUMP_TF, target))) |
-                                    set(self.remove_mid_line_ifs(self.rem_or(start, prev_op[prev_op[rtarget]],
+                                    set(self.remove_mid_line_ifs(self.rem_or(start, prev_op[pre_rtarget],
                                                                              (self.opc.POP_JUMP_IF_FALSE,
                                                                               self.opc.POP_JUMP_IF_TRUE,
                                                                               self.opc.JUMP_ABSOLUTE),
-                                                                             prev_op[rtarget], True)))))):
+                                                                             pre_rtarget, True)))))):
                             pass
                         else:
                             fix = None
@@ -724,7 +785,7 @@ class Scanner3(Scanner):
                     if code[prev_op[next]] == self.opc.POP_JUMP_IF_FALSE:
                         if (code[next] == self.opc.JUMP_FORWARD
                             or target != rtarget
-                            or code[prev_op[prev_op[rtarget]]] not in
+                            or code[prev_op[pre_rtarget]] not in
                             (self.opc.JUMP_ABSOLUTE, self.opc.RETURN_VALUE)):
                             self.fixed_jumps[offset] = prev_op[next]
                             return
@@ -737,38 +798,62 @@ class Scanner3(Scanner):
             if offset in self.ignore_if:
                 return
 
-            if (code[prev_op[rtarget]] == self.opc.JUMP_ABSOLUTE and
-                prev_op[rtarget] in self.stmts and
-                prev_op[rtarget] != offset and
-                prev_op[prev_op[rtarget]] != offset and
+            if (code[pre_rtarget] == self.opc.JUMP_ABSOLUTE and
+                pre_rtarget in self.stmts and
+                pre_rtarget != offset and
+                prev_op[pre_rtarget] != offset and
                 not (code[rtarget] == self.opc.JUMP_ABSOLUTE and
                      code[rtarget+3] == self.opc.POP_BLOCK and
-                     code[prev_op[prev_op[rtarget]]] != self.opc.JUMP_ABSOLUTE)):
-                rtarget = prev_op[rtarget]
+                     code[prev_op[pre_rtarget]] != self.opc.JUMP_ABSOLUTE)):
+                rtarget = pre_rtarget
 
-            # Does the "if" jump just beyond a jump op, then this can be
-            # a block inside an "if" statement
-            if self.is_jump_forward(prev_op[rtarget]):
-                if_end = self.get_target(prev_op[rtarget])
+            # Does the "jump if" jump beyond a jump op?
+            # That is, we have something like:
+            #  POP_JUMP_IF_FALSE HERE
+            #  ...
+            # JUMP_FORWARD
+            # HERE:
+            #
+            # If so, this can be block inside an "if" statement
+            # or a conditional assignment like:
+            #   x = 1 if x else 2
+            #
+            # There are other contexts we may need to consider
+            # like whether the target is "END_FINALLY"
+            # or if the condition jump is to a forward location
+            if self.is_jump_forward(pre_rtarget):
+                if_end = self.get_target(pre_rtarget)
 
-                # Is this a loop and not an "if" statement?
-                if ((if_end < prev_op[rtarget]) and
+                # If the jump target is back, we are looping
+                if (if_end < pre_rtarget and
                     (code[prev_op[if_end]] == self.opc.SETUP_LOOP)):
-                    if(if_end > start):
+                    if (if_end > start):
                         return
 
                 end = self.restrict_to_parent(if_end, parent)
 
                 self.structs.append({'type': 'if-then',
                                      'start': start,
-                                     'end': prev_op[rtarget]})
-                self.not_continue.add(prev_op[rtarget])
+                                     'end': pre_rtarget})
+                self.not_continue.add(pre_rtarget)
 
-                if rtarget < end:
-                    self.structs.append({'type': 'if-else',
+                if rtarget < end and (
+                        code[rtarget] not in (self.opc.END_FINALLY,
+                                              self.opc.JUMP_ABSOLUTE) and
+                        code[prev_op[pre_rtarget]] not in (self.opc.POP_EXCEPT,
+                                                        self.opc.END_FINALLY)):
+                    self.structs.append({'type': 'else',
                                          'start': rtarget,
                                          'end': end})
-            elif code[prev_op[rtarget]] == self.opc.RETURN_VALUE:
+                    self.else_start[rtarget] = end
+            elif self.is_jump_back(pre_rtarget):
+                if_end = rtarget
+                self.structs.append({'type': 'if-then',
+                                     'start': start,
+                                     'end': pre_rtarget})
+                self.not_continue.add(pre_rtarget)
+            elif code[pre_rtarget] in (self.opc.RETURN_VALUE,
+                                       self.opc.BREAK_LOOP):
                 self.structs.append({'type': 'if-then',
                                      'start': start,
                                      'end': rtarget})
@@ -798,8 +883,21 @@ class Scanner3(Scanner):
                                 return
                             pass
                     pass
-                self.return_end_ifs.add(prev_op[rtarget])
+                if code[pre_rtarget] == self.opc.RETURN_VALUE:
+                    self.return_end_ifs.add(pre_rtarget)
+                else:
+                    self.fixed_jumps[offset] = rtarget
+                    self.not_continue.add(pre_rtarget)
 
+
+        elif op == self.opc.SETUP_EXCEPT:
+            target = self.get_target(offset)
+            end    = self.restrict_to_parent(target, parent)
+            self.fixed_jumps[offset] = end
+        elif op == self.opc.SETUP_FINALLY:
+            target = self.get_target(offset)
+            end    = self.restrict_to_parent(target, parent)
+            self.fixed_jumps[offset] = end
         elif op in self.jump_if_pop:
             target = self.get_target(offset)
             if target > offset:
@@ -837,6 +935,16 @@ class Scanner3(Scanner):
                 pass
         return
 
+    def is_jump_back(self, offset):
+        """
+        Return True if the code at offset is some sort of jump back.
+        That is, it is ether "JUMP_FORWARD" or an absolute jump that
+        goes forward.
+        """
+        if self.code[offset] != self.opc.JUMP_ABSOLUTE:
+            return False
+        return offset > self.get_target(offset)
+
     def next_except_jump(self, start):
         """
         Return the next jump that was generated by an except SomeException:
@@ -857,7 +965,9 @@ class Scanner3(Scanner):
             op = self.code[i]
             if op == self.opc.END_FINALLY:
                 if count_END_FINALLY == count_SETUP_:
-                    assert self.code[self.prev_op[i]] in (JUMP_ABSOLUTE, JUMP_FORWARD, RETURN_VALUE)
+                    assert self.code[self.prev_op[i]] in (JUMP_ABSOLUTE,
+                                                          JUMP_FORWARD,
+                                                          RETURN_VALUE)
                     self.not_continue.add(self.prev_op[i])
                     return self.prev_op[i]
                 count_END_FINALLY += 1

uncompyle6/scanners/scanner30.py

@@ -1,4 +1,4 @@
-#  Copyright (c) 2016 by Rocky Bernstein
+#  Copyright (c) 2016, 2017 by Rocky Bernstein
 """
 Python 3.0 bytecode scanner/deparser
 
@@ -12,6 +12,8 @@ from __future__ import print_function
 from xdis.opcodes import opcode_30 as opc
 JUMP_OPs = map(lambda op: opc.opname[op], opc.hasjrel + opc.hasjabs)
 
+JUMP_TF = frozenset([opc.JUMP_IF_FALSE, opc.JUMP_IF_TRUE])
+
 from uncompyle6.scanners.scanner3 import Scanner3
 class Scanner30(Scanner3):
 
@@ -20,6 +22,373 @@ class Scanner30(Scanner3):
         return
     pass
 
+    def detect_control_flow(self, offset, targets):
+        """
+        Detect structures and their boundaries to fix optimized jumps
+        Python 3.0 is more like Python 2.6 than it is Python 3.x.
+        So we have a special routine here.
+        """
+
+        code = self.code
+        op = code[offset]
+
+        # Detect parent structure
+        parent = self.structs[0]
+        start = parent['start']
+        end = parent['end']
+
+        # Pick inner-most parent for our offset
+        for struct in self.structs:
+            current_start = struct['start']
+            current_end   = struct['end']
+            if ((current_start <= offset < current_end)
+                and (current_start >= start and current_end <= end)):
+                start = current_start
+                end = current_end
+                parent = struct
+
+        if op == self.opc.SETUP_LOOP:
+            # We categorize loop types: 'for', 'while', 'while 1' with
+            # possibly suffixes '-loop' and '-else'
+            # Try to find the jump_back instruction of the loop.
+            # It could be a return instruction.
+
+            start = offset+3
+            target = self.get_target(offset)
+            end    = self.restrict_to_parent(target, parent)
+            self.setup_loop_targets[offset] = target
+            self.setup_loops[target] = offset
+
+            if target != end:
+                self.fixed_jumps[offset] = end
+
+            (line_no, next_line_byte) = self.lines[offset]
+            jump_back = self.last_instr(start, end, self.opc.JUMP_ABSOLUTE,
+                                            next_line_byte, False)
+
+            if jump_back:
+                jump_forward_offset = jump_back+3
+            else:
+                jump_forward_offset = None
+
+            return_val_offset1 = self.prev[self.prev[end]]
+
+            if (jump_back and jump_back != self.prev_op[end]
+                and self.is_jump_forward(jump_forward_offset)):
+                if (code[self.prev_op[end]] == self.opc.RETURN_VALUE or
+                    (code[self.prev_op[end]] == self.opc.POP_BLOCK
+                     and code[return_val_offset1] == self.opc.RETURN_VALUE)):
+                    jump_back = None
+            if not jump_back:
+                # loop suite ends in return
+                jump_back = self.last_instr(start, end, self.opc.RETURN_VALUE)
+                if not jump_back:
+                    return
+
+                jump_back += 2
+                if_offset = None
+                if code[self.prev_op[next_line_byte]] not in JUMP_TF:
+                    if_offset = self.prev[next_line_byte]
+                if if_offset:
+                    loop_type = 'while'
+                    self.ignore_if.add(if_offset)
+                else:
+                    loop_type = 'for'
+                target = next_line_byte
+                end = jump_back + 3
+            else:
+                if self.get_target(jump_back) >= next_line_byte:
+                    jump_back = self.last_instr(start, end, self.opc.JUMP_ABSOLUTE, start, False)
+                if end > jump_back+4 and self.is_jump_forward(end):
+                    if self.is_jump_forward(jump_back+4):
+                        if self.get_target(jump_back+4) == self.get_target(end):
+                            self.fixed_jumps[offset] = jump_back+4
+                            end = jump_back+4
+                elif target < offset:
+                    self.fixed_jumps[offset] = jump_back+4
+                    end = jump_back+4
+
+                target = self.get_target(jump_back)
+
+                if code[target] in (self.opc.FOR_ITER, self.opc.GET_ITER):
+                    loop_type = 'for'
+                else:
+                    loop_type = 'while'
+                    test = self.prev_op[next_line_byte]
+
+                    if test == offset:
+                        loop_type = 'while 1'
+                    elif self.code[test] in opc.hasjabs+opc.hasjrel:
+                        self.ignore_if.add(test)
+                        test_target = self.get_target(test)
+                        if test_target > (jump_back+3):
+                            jump_back = test_target
+                self.not_continue.add(jump_back)
+            self.loops.append(target)
+            self.structs.append({'type': loop_type + '-loop',
+                                   'start': target,
+                                   'end':   jump_back})
+            if jump_back+3 != end:
+                self.structs.append({'type': loop_type + '-else',
+                                       'start': jump_back+3,
+                                       'end':   end})
+        elif op in JUMP_TF:
+            start = offset + self.op_size(op)
+            target = self.get_target(offset)
+            rtarget = self.restrict_to_parent(target, parent)
+            prev_op = self.prev_op
+
+            # Do not let jump to go out of parent struct bounds
+            if target != rtarget and parent['type'] == 'and/or':
+                self.fixed_jumps[offset] = rtarget
+                return
+
+            # Does this jump to right after another conditional jump that is
+            # not myself?  If so, it's part of a larger conditional.
+            # rocky: if we have a conditional jump to the next instruction, then
+            # possibly I am "skipping over" a "pass" or null statement.
+
+            if ((code[prev_op[target]] in self.pop_jump_if_pop) and
+                (target > offset) and prev_op[target] != offset):
+                self.fixed_jumps[offset] = prev_op[target]
+                self.structs.append({'type': 'and/or',
+                                     'start': start,
+                                     'end': prev_op[target]})
+                return
+
+            # The op offset just before the target jump offset is important
+            # in making a determination of what we have. Save that.
+            pre_rtarget = prev_op[rtarget]
+
+            # Is it an "and" inside an "if" or "while" block
+            if op == opc.JUMP_IF_FALSE:
+
+                # Search for another JUMP_IF_FALSE targetting the same op,
+                # in current statement, starting from current offset, and filter
+                # everything inside inner 'or' jumps and midline ifs
+                match = self.rem_or(start, self.next_stmt[offset],
+                                    opc.JUMP_IF_FALSE, target)
+
+                # If we still have any offsets in set, start working on it
+                if match:
+                    is_jump_forward = self.is_jump_forward(pre_rtarget)
+                    if (is_jump_forward and pre_rtarget not in self.stmts and
+                        self.restrict_to_parent(self.get_target(pre_rtarget), parent) == rtarget):
+                        if (code[prev_op[pre_rtarget]] == self.opc.JUMP_ABSOLUTE
+                            and self.remove_mid_line_ifs([offset]) and
+                            target == self.get_target(prev_op[pre_rtarget]) and
+                            (prev_op[pre_rtarget] not in self.stmts or
+                             self.get_target(prev_op[pre_rtarget]) > prev_op[pre_rtarget]) and
+                            1 == len(self.remove_mid_line_ifs(self.rem_or(start, prev_op[pre_rtarget], JUMP_TF, target)))):
+                            pass
+                        elif (code[prev_op[pre_rtarget]] == self.opc.RETURN_VALUE
+                              and self.remove_mid_line_ifs([offset]) and
+                              1 == (len(set(self.remove_mid_line_ifs(self.rem_or(start, prev_op[pre_rtarget],
+                                                                                 JUMP_TF, target))) |
+                                    set(self.remove_mid_line_ifs(self.rem_or(start, prev_op[pre_rtarget],
+                                                                             (opc.JUMP_IF_FALSE,
+                                                                              opc.JUMP_IF_TRUE,
+                                                                              opc.JUMP_ABSOLUTE),
+                                                                             pre_rtarget, True)))))):
+                            pass
+                        else:
+                            fix = None
+                            jump_ifs = self.all_instr(start, self.next_stmt[offset],
+                                                      opc.JUMP_IF_FALSE)
+                            last_jump_good = True
+                            for j in jump_ifs:
+                                if target == self.get_target(j):
+                                    if self.lines[j].next == j + 3 and last_jump_good:
+                                        fix = j
+                                        break
+                                else:
+                                    last_jump_good = False
+                            self.fixed_jumps[offset] = fix or match[-1]
+                            return
+                    else:
+                        self.fixed_jumps[offset] = match[-1]
+                        return
+            # op == JUMP_IF_TRUE
+            else:
+                next = self.next_stmt[offset]
+                if prev_op[next] == offset:
+                    pass
+                elif self.is_jump_forward(next) and target == self.get_target(next):
+                    if code[prev_op[next]] == opc.JUMP_IF_FALSE:
+                        if (code[next] == self.opc.JUMP_FORWARD
+                            or target != rtarget
+                            or code[prev_op[pre_rtarget]] not in
+                            (self.opc.JUMP_ABSOLUTE, self.opc.RETURN_VALUE)):
+                            self.fixed_jumps[offset] = prev_op[next]
+                            return
+                elif (code[next] == self.opc.JUMP_ABSOLUTE and self.is_jump_forward(target) and
+                      self.get_target(target) == self.get_target(next)):
+                    self.fixed_jumps[offset] = prev_op[next]
+                    return
+
+            # Don't add a struct for a while test, it's already taken care of
+            if offset in self.ignore_if:
+                return
+
+            if (code[pre_rtarget] == self.opc.JUMP_ABSOLUTE and
+                pre_rtarget in self.stmts and
+                pre_rtarget != offset and
+                prev_op[pre_rtarget] != offset and
+                not (code[rtarget] == self.opc.JUMP_ABSOLUTE and
+                     code[rtarget+3] == self.opc.POP_BLOCK and
+                     code[prev_op[pre_rtarget]] != self.opc.JUMP_ABSOLUTE)):
+                rtarget = pre_rtarget
+
+            # Does the "jump if" jump beyond a jump op?
+            # That is, we have something like:
+            #  JUMP_IF_FALSE HERE
+            #  ...
+            # JUMP_FORWARD
+            # HERE:
+            #
+            # If so, this can be block inside an "if" statement
+            # or a conditional assignment like:
+            #   x = 1 if x else 2
+            #
+            # There are other contexts we may need to consider
+            # like whether the target is "END_FINALLY"
+            # or if the condition jump is to a forward location
+            if self.is_jump_forward(pre_rtarget):
+                if_end = self.get_target(pre_rtarget)
+
+                # If the jump target is back, we are looping
+                if (if_end < pre_rtarget and
+                    (code[prev_op[if_end]] == self.opc.SETUP_LOOP)):
+                    if (if_end > start):
+                        return
+
+                end = self.restrict_to_parent(if_end, parent)
+
+                self.structs.append({'type': 'if-then',
+                                     'start': start,
+                                     'end': pre_rtarget})
+                self.not_continue.add(pre_rtarget)
+
+                # if rtarget < end and (
+                #         code[rtarget] not in (self.opc.END_FINALLY,
+                #                               self.opc.JUMP_ABSOLUTE) and
+                #         code[prev_op[pre_rtarget]] not in (self.opc.POP_EXCEPT,
+                #                                         self.opc.END_FINALLY)):
+                #     self.structs.append({'type': 'else',
+                #                          'start': rtarget,
+                #                          'end': end})
+                #     self.else_start[rtarget] = end
+            elif self.is_jump_back(pre_rtarget):
+                if_end = rtarget
+                self.structs.append({'type': 'if-then',
+                                     'start': start,
+                                     'end': pre_rtarget})
+                self.not_continue.add(pre_rtarget)
+            elif code[pre_rtarget] in (self.opc.RETURN_VALUE,
+                                       self.opc.BREAK_LOOP):
+                self.structs.append({'type': 'if-then',
+                                     'start': start,
+                                     'end': rtarget})
+                # It is important to distingish if this return is inside some sort
+                # except block return
+                jump_prev = prev_op[offset]
+                if self.is_pypy and code[jump_prev] == self.opc.COMPARE_OP:
+                    if self.opc.cmp_op[code[jump_prev+1]] == 'exception match':
+                        return
+                if self.version >= 3.5:
+                    # Python 3.5 may remove as dead code a JUMP
+                    # instruction after a RETURN_VALUE. So we check
+                    # based on seeing SETUP_EXCEPT various places.
+                    if code[rtarget] == self.opc.SETUP_EXCEPT:
+                        return
+                    # Check that next instruction after pops and jump is
+                    # not from SETUP_EXCEPT
+                    next_op = rtarget
+                    if code[next_op] == self.opc.POP_BLOCK:
+                        next_op += self.op_size(self.code[next_op])
+                    if code[next_op] == self.opc.JUMP_ABSOLUTE:
+                        next_op += self.op_size(self.code[next_op])
+                    if next_op in targets:
+                        for try_op in targets[next_op]:
+                            come_from_op = code[try_op]
+                            if come_from_op == self.opc.SETUP_EXCEPT:
+                                return
+                            pass
+                    pass
+                if code[pre_rtarget] == self.opc.RETURN_VALUE:
+                    self.return_end_ifs.add(pre_rtarget)
+                else:
+                    self.fixed_jumps[offset] = rtarget
+                    self.not_continue.add(pre_rtarget)
+
+
+        elif op == self.opc.SETUP_EXCEPT:
+            target = self.get_target(offset)
+            end    = self.restrict_to_parent(target, parent)
+            self.fixed_jumps[offset] = end
+        elif op == self.opc.SETUP_FINALLY:
+            target = self.get_target(offset)
+            end    = self.restrict_to_parent(target, parent)
+            self.fixed_jumps[offset] = end
+        elif op in self.jump_if_pop:
+            target = self.get_target(offset)
+            if target > offset:
+                unop_target = self.last_instr(offset, target, self.opc.JUMP_FORWARD, target)
+                if unop_target and code[unop_target+3] != self.opc.ROT_TWO:
+                    self.fixed_jumps[offset] = unop_target
+                else:
+                    self.fixed_jumps[offset] = self.restrict_to_parent(target, parent)
+                    pass
+                pass
+        elif self.version >= 3.5:
+            # 3.5+ has Jump optimization which too often causes RETURN_VALUE to get
+            # misclassified as RETURN_END_IF. Handle that here.
+            # In RETURN_VALUE, JUMP_ABSOLUTE, RETURN_VALUE is never RETURN_END_IF
+            if op == self.opc.RETURN_VALUE:
+                if (offset+1 < len(code) and code[offset+1] == self.opc.JUMP_ABSOLUTE and
+                    offset in self.return_end_ifs):
+                    self.return_end_ifs.remove(offset)
+                    pass
+                pass
+            elif op == self.opc.JUMP_FORWARD:
+                # If we have:
+                #   JUMP_FORWARD x, [non-jump, insns], RETURN_VALUE, x:
+                # then RETURN_VALUE is not RETURN_END_IF
+                rtarget = self.get_target(offset)
+                rtarget_prev = self.prev[rtarget]
+                if (code[rtarget_prev] == self.opc.RETURN_VALUE and
+                    rtarget_prev in self.return_end_ifs):
+                    i = rtarget_prev
+                    while i != offset:
+                        if code[i] in [opc.JUMP_FORWARD, opc.JUMP_ABSOLUTE]:
+                            return
+                        i = self.prev[i]
+                    self.return_end_ifs.remove(rtarget_prev)
+                pass
+        return
+
+    def rem_or(self, start, end, instr, target=None, include_beyond_target=False):
+        """
+        Find offsets of all requested <instr> between <start> and <end>,
+        optionally <target>ing specified offset, and return list found
+        <instr> offsets which are not within any POP_JUMP_IF_TRUE jumps.
+        """
+        assert(start>=0 and end<=len(self.code) and start <= end)
+
+        # Find all offsets of requested instructions
+        instr_offsets = self.all_instr(start, end, instr, target, include_beyond_target)
+        # Get all JUMP_IF_TRUE (or) offsets
+        pjit_offsets = self.all_instr(start, end, opc.JUMP_IF_TRUE)
+        filtered = []
+        for pjit_offset in pjit_offsets:
+            pjit_tgt = self.get_target(pjit_offset) - 3
+            for instr_offset in instr_offsets:
+                if instr_offset <= pjit_offset or instr_offset >= pjit_tgt:
+                    filtered.append(instr_offset)
+            instr_offsets = filtered
+            filtered = []
+        return instr_offsets
+
 if __name__ == "__main__":
     from uncompyle6 import PYTHON_VERSION
     if PYTHON_VERSION == 3.0:

uncompyle6/scanners/tok.py

@@ -29,7 +29,7 @@ class Token:
         self.pattr = pattr
         self.offset = offset
         self.linestart = linestart
-        if has_arg == False:
+        if has_arg is False:
             self.attr = None
             self.pattr = None
         self.opc = opc
@@ -66,7 +66,8 @@ class Token:
             pattr = self.pattr
             if self.opc:
                 if self.op in self.opc.hasjrel:
-                    pattr = "to " + self.pattr
+                    if not self.pattr.startswith('to '):
+                        pattr = "to " + self.pattr
                 elif self.op in self.opc.hasjabs:
                     self.pattr= str(self.pattr)
                     if not self.pattr.startswith('to '):

uncompyle6/semantics/check_ast.py

@@ -12,9 +12,9 @@ def checker(ast, in_loop, errors):
     in_loop = in_loop or ast.type in ('while1stmt', 'whileTruestmt',
                                       'whilestmt', 'whileelsestmt',
                                       'for_block')
-    if ast.type == 'augassign1' and ast[0][0] == 'and':
-        text = str(ast[0])
-        error_text = '\n# improper augmented assigment:\n#\t' + '\n# '.join(text.split("\n"))
+    if ast.type in ('augassign1', 'augassign2') and ast[0][0] == 'and':
+        text = str(ast)
+        error_text = '\n# improper augmented assigment (e.g. +=, *=, ...):\n#\t' + '\n# '.join(text.split("\n")) + '\n'
         errors.append(error_text)
 
     for node in ast:

uncompyle6/semantics/consts.py

@@ -0,0 +1,347 @@
+#  Copyright (c) 2017 by Rocky Bernstein
+"""Constants used in pysource.py"""
+
+import re, sys
+from uncompyle6.parsers.astnode import AST
+from uncompyle6 import PYTHON3
+from uncompyle6.scanners.tok import Token, NoneToken
+
+if PYTHON3:
+    minint = -sys.maxsize-1
+    maxint = sys.maxsize
+else:
+    minint = -sys.maxint-1
+    maxint = sys.maxint
+
+
+LINE_LENGTH = 80
+
+# Some ASTs used for comparing code fragments (like 'return None' at
+# the end of functions).
+
+RETURN_LOCALS = AST('return_stmt',
+                    [ AST('ret_expr', [AST('expr', [ Token('LOAD_LOCALS') ])]),
+                      Token('RETURN_VALUE')])
+
+NONE = AST('expr', [ NoneToken ] )
+
+RETURN_NONE = AST('stmt',
+                  [ AST('return_stmt',
+                        [ NONE, Token('RETURN_VALUE')]) ])
+
+PASS = AST('stmts',
+           [ AST('sstmt',
+                 [ AST('stmt',
+                       [ AST('passstmt', [])])])])
+
+ASSIGN_DOC_STRING = lambda doc_string: \
+  AST('stmt',
+      [ AST('assign',
+            [ AST('expr', [ Token('LOAD_CONST', pattr=doc_string) ]),
+              AST('designator', [ Token('STORE_NAME', pattr='__doc__')])
+            ])])
+
+NAME_MODULE = AST('stmt',
+                [ AST('assign',
+                    [ AST('expr',
+                          [Token('LOAD_NAME', pattr='__name__', offset=0, has_arg=True)]),
+                      AST('designator',
+                          [ Token('STORE_NAME', pattr='__module__', offset=3, has_arg=True)])
+                      ])])
+
+# God intended \t, but Python has decided to use 4 spaces.
+# If you want real tabs, use Go.
+# TAB = '\t'
+TAB = ' ' * 4
+INDENT_PER_LEVEL = ' ' # additional intent per pretty-print level
+
+TABLE_R = {
+    'STORE_ATTR':	( '%c.%[1]{pattr}', 0),
+#   'STORE_SUBSCR':	( '%c[%c]', 0, 1 ),
+    'DELETE_ATTR':	( '%|del %c.%[-1]{pattr}\n', 0 ),
+#   'EXEC_STMT':	( '%|exec %c in %[1]C\n', 0, (0,maxint,', ') ),
+}
+
+TABLE_R0 = {
+#    'BUILD_LIST':	( '[%C]',      (0,-1,', ') ),
+#    'BUILD_TUPLE':	( '(%C)',      (0,-1,', ') ),
+#    'CALL_FUNCTION':	( '%c(%P)', 0, (1,-1,', ') ),
+}
+TABLE_DIRECT = {
+    'BINARY_ADD':	( '+' ,),
+    'BINARY_SUBTRACT':	( '-' ,),
+    'BINARY_MULTIPLY':	( '*' ,),
+    'BINARY_DIVIDE':	( '/' ,),
+    'BINARY_MATRIX_MULTIPLY':	( '@' ,),
+    'BINARY_TRUE_DIVIDE':	( '/' ,),
+    'BINARY_FLOOR_DIVIDE':	( '//' ,),
+    'BINARY_MODULO':	( '%%',),
+    'BINARY_POWER':	( '**',),
+    'BINARY_LSHIFT':	( '<<',),
+    'BINARY_RSHIFT':	( '>>',),
+    'BINARY_AND':	( '&' ,),
+    'BINARY_OR':	( '|' ,),
+    'BINARY_XOR':	( '^' ,),
+    'INPLACE_ADD':	( '+=' ,),
+    'INPLACE_SUBTRACT':	( '-=' ,),
+    'INPLACE_MULTIPLY':	( '*=' ,),
+    'INPLACE_MATRIX_MULTIPLY':	( '@=' ,),
+    'INPLACE_DIVIDE':	( '/=' ,),
+    'INPLACE_TRUE_DIVIDE':	( '/=' ,),
+    'INPLACE_FLOOR_DIVIDE':	( '//=' ,),
+    'INPLACE_MODULO':	( '%%=',),
+    'INPLACE_POWER':	( '**=',),
+    'INPLACE_LSHIFT':	( '<<=',),
+    'INPLACE_RSHIFT':	( '>>=',),
+    'INPLACE_AND':	( '&=' ,),
+    'INPLACE_OR':	( '|=' ,),
+    'INPLACE_XOR':	( '^=' ,),
+    'binary_expr':	( '%c %c %c', 0, -1, 1 ),
+
+    'UNARY_POSITIVE':	( '+',),
+    'UNARY_NEGATIVE':	( '-',),
+    'UNARY_INVERT':	( '~%c'),
+    'unary_expr':   ( '%c%c', 1, 0),
+
+    'unary_not':	( 'not %c', 0 ),
+    'unary_convert':	( '`%c`', 0 ),
+    'get_iter':	( 'iter(%c)', 0 ),
+    'slice0':		( '%c[:]', 0 ),
+    'slice1':		( '%c[%p:]', 0, (1, 100) ),
+    'slice2':		( '%c[:%p]', 0, (1, 100) ),
+    'slice3':		( '%c[%p:%p]', 0, (1, 100), (2, 100) ),
+
+    'IMPORT_FROM':	( '%{pattr}', ),
+    'load_attr':	( '%c.%[1]{pattr}', 0),
+    'LOAD_FAST':	( '%{pattr}', ),
+    'LOAD_NAME':	( '%{pattr}', ),
+    'LOAD_CLASSNAME':	( '%{pattr}', ),
+    'LOAD_GLOBAL':	( '%{pattr}', ),
+    'LOAD_DEREF':	( '%{pattr}', ),
+    'LOAD_LOCALS':	( 'locals()', ),
+    'LOAD_ASSERT':  ( '%{pattr}', ),
+#   'LOAD_CONST':	( '%{pattr}', ),	# handled by n_LOAD_CONST
+    'DELETE_FAST':	( '%|del %{pattr}\n', ),
+    'DELETE_NAME':	( '%|del %{pattr}\n', ),
+    'DELETE_GLOBAL':	( '%|del %{pattr}\n', ),
+    'delete_subscr':	( '%|del %c[%c]\n', 0, 1,),
+    'binary_subscr':	( '%c[%p]', 0, (1, 100)),
+    'binary_subscr2':	( '%c[%p]', 0, (1, 100)),
+    'store_subscr':	( '%c[%c]', 0, 1),
+    'STORE_FAST':	( '%{pattr}', ),
+    'STORE_NAME':	( '%{pattr}', ),
+    'STORE_GLOBAL':	( '%{pattr}', ),
+    'STORE_DEREF':	( '%{pattr}', ),
+    'unpack':		( '%C%,', (1, maxint, ', ') ),
+
+    # This nonterminal we create on the fly in semantic routines
+    'unpack_w_parens':	( '(%C%,)', (1, maxint, ', ') ),
+
+    'unpack_list':	( '[%C]', (1, maxint, ', ') ),
+    'build_tuple2':	( '%P', (0, -1, ', ', 100) ),
+
+    # 'list_compr':	( '[ %c ]', -2),	# handled by n_list_compr
+    'list_iter':	( '%c', 0),
+    'list_for':		( ' for %c in %c%c', 2, 0, 3 ),
+    'list_if':		( ' if %c%c', 0, 2 ),
+    'list_if_not':		( ' if not %p%c', (0, 22), 2 ),
+    'lc_body':		( '', ),	# ignore when recusing
+
+    'comp_iter':	( '%c', 0),
+    'comp_if':		( ' if %c%c', 0, 2 ),
+    'comp_ifnot':	( ' if not %p%c', (0, 22), 2 ),
+    'comp_body':	( '', ),	# ignore when recusing
+    'set_comp_body':    ( '%c', 0 ),
+    'gen_comp_body':    ( '%c', 0 ),
+    'dict_comp_body':   ( '%c:%c', 1, 0 ),
+
+    'assign':		( '%|%c = %p\n', -1, (0, 200) ),
+
+    # The 2nd parameter should have a = suffix.
+    # There is a rule with a 4th parameter "designator"
+    # which we don't use here.
+    'augassign1':	( '%|%c %c %c\n', 0, 2, 1),
+
+    'augassign2':	( '%|%c.%[2]{pattr} %c %c\n', 0, -3, -4),
+    'designList':	( '%c = %c', 0, -1 ),
+    'and':          	( '%c and %c', 0, 2 ),
+    'ret_and':        	( '%c and %c', 0, 2 ),
+    'and2':          	( '%c', 3 ),
+    'or':           	( '%c or %c', 0, 2 ),
+    'ret_or':           	( '%c or %c', 0, 2 ),
+    'conditional':  ( '%p if %p else %p', (2, 27), (0, 27), (4, 27)),
+    'ret_cond':     ( '%p if %p else %p', (2, 27), (0, 27), (-1, 27)),
+    'conditionalnot':  ( '%p if not %p else %p', (2, 27), (0, 22), (4, 27)),
+    'ret_cond_not':  ( '%p if not %p else %p', (2, 27), (0, 22), (-1, 27)),
+    'conditional_lambda':  ( '(%c if %c else %c)', 2, 0, 3),
+    'return_lambda':    ('%c', 0),
+    'compare':		( '%p %[-1]{pattr} %p', (0, 19), (1, 19) ),
+    'cmp_list':		( '%p %p', (0, 20), (1, 19)),
+    'cmp_list1':	( '%[3]{pattr} %p %p', (0, 19), (-2, 19)),
+    'cmp_list2':	( '%[1]{pattr} %p', (0, 19)),
+#   'classdef': 	(), # handled by n_classdef()
+    'funcdef':  	( '\n\n%|def %c\n', -2), # -2 to handle closures
+    'funcdefdeco':  	( '\n\n%c', 0),
+    'mkfuncdeco':  	( '%|@%c\n%c', 0, 1),
+    'mkfuncdeco0':  	( '%|def %c\n', 0),
+    'classdefdeco':  	( '\n\n%c', 0),
+    'classdefdeco1':  	( '%|@%c\n%c', 0, 1),
+    'kwarg':    	( '%[0]{pattr}=%c', 1),
+    'kwargs':    	( '%D', (0, maxint, ', ') ),
+
+    'assert_expr_or': ( '%c or %c', 0, 2 ),
+    'assert_expr_and':    ( '%c and %c', 0, 2 ),
+    'print_items_stmt': ( '%|print %c%c,\n', 0, 2),  # Python 2 only
+    'print_items_nl_stmt': ( '%|print %c%c\n', 0, 2),
+    'print_item':  ( ', %c', 0),
+    'print_nl':	( '%|print\n', ),
+    'print_to':		( '%|print >> %c, %c,\n', 0, 1 ),
+    'print_to_nl':	( '%|print >> %c, %c\n', 0, 1 ),
+    'print_nl_to':	( '%|print >> %c\n', 0 ),
+    'print_to_items':	( '%C', (0, 2, ', ') ),
+
+    'call_stmt':	( '%|%p\n', (0, 200)),
+    'break_stmt':	( '%|break\n', ),
+    'continue_stmt':	( '%|continue\n', ),
+
+    'raise_stmt0':	( '%|raise\n', ),
+    'raise_stmt1':	( '%|raise %c\n', 0),
+    'raise_stmt3':	( '%|raise %c, %c, %c\n', 0, 1, 2),
+#    'yield':	( 'yield %c', 0),
+#    'return_stmt':	( '%|return %c\n', 0),
+
+    'ifstmt':		( '%|if %c:\n%+%c%-', 0, 1 ),
+    'iflaststmt':		( '%|if %c:\n%+%c%-', 0, 1 ),
+    'iflaststmtl':		( '%|if %c:\n%+%c%-', 0, 1 ),
+    'testtrue':         ( 'not %p', (0, 22) ),
+
+    'ifelsestmt':	( '%|if %c:\n%+%c%-%|else:\n%+%c%-', 0, 1, 3 ),
+    'ifelsestmtc':	( '%|if %c:\n%+%c%-%|else:\n%+%c%-', 0, 1, 3 ),
+    'ifelsestmtl':	( '%|if %c:\n%+%c%-%|else:\n%+%c%-', 0, 1, 3 ),
+    'ifelifstmt':	( '%|if %c:\n%+%c%-%c', 0, 1, 3 ),
+    'elifelifstmt':	( '%|elif %c:\n%+%c%-%c', 0, 1, 3 ),
+    'elifstmt':		( '%|elif %c:\n%+%c%-', 0, 1 ),
+    'elifelsestmt':	( '%|elif %c:\n%+%c%-%|else:\n%+%c%-', 0, 1, 3 ),
+    'ifelsestmtr':	( '%|if %c:\n%+%c%-%|else:\n%+%c%-', 0, 1, 2 ),
+    'elifelsestmtr':	( '%|elif %c:\n%+%c%-%|else:\n%+%c%-\n\n', 0, 1, 2 ),
+
+    'whileTruestmt':	( '%|while True:\n%+%c%-\n\n', 1 ),
+    'whilestmt':	( '%|while %c:\n%+%c%-\n\n', 1, 2 ),
+    'while1stmt':	( '%|while 1:\n%+%c%-\n\n', 1 ),
+    'while1elsestmt':  ( '%|while 1:\n%+%c%-%|else:\n%+%c%-\n\n', 1, -2 ),
+    'whileelsestmt':	( '%|while %c:\n%+%c%-%|else:\n%+%c%-\n\n', 1, 2, -2 ),
+    'whileelselaststmt':	( '%|while %c:\n%+%c%-%|else:\n%+%c%-', 1, 2, -2 ),
+    'forstmt':		( '%|for %c in %c:\n%+%c%-\n\n', 3, 1, 4 ),
+    'forelsestmt':	(
+        '%|for %c in %c:\n%+%c%-%|else:\n%+%c%-\n\n', 3, 1, 4, -2),
+    'forelselaststmt':	(
+        '%|for %c in %c:\n%+%c%-%|else:\n%+%c%-', 3, 1, 4, -2),
+    'forelselaststmtl':	(
+        '%|for %c in %c:\n%+%c%-%|else:\n%+%c%-\n\n', 3, 1, 4, -2),
+    'trystmt':		( '%|try:\n%+%c%-%c\n\n', 1, 3 ),
+    'tryelsestmt':	( '%|try:\n%+%c%-%c%|else:\n%+%c%-\n\n', 1, 3, 4 ),
+    'tryelsestmtc':	( '%|try:\n%+%c%-%c%|else:\n%+%c%-', 1, 3, 4 ),
+    'tryelsestmtl':	( '%|try:\n%+%c%-%c%|else:\n%+%c%-', 1, 3, 4 ),
+    'tf_trystmt':	( '%c%-%c%+', 1, 3 ),
+    'tf_tryelsestmt':	( '%c%-%c%|else:\n%+%c', 1, 3, 4 ),
+    'tryfinallystmt':	( '%|try:\n%+%c%-%|finally:\n%+%c%-\n\n', 1, 5 ),
+    'except':           ( '%|except:\n%+%c%-', 3 ),
+    'except_cond1':	( '%|except %c:\n', 1 ),
+    'except_suite':     ( '%+%c%-%C', 0, (1, maxint, '') ),
+    'except_suite_finalize':     ( '%+%c%-%C', 1, (3, maxint, '') ),
+    'withstmt':     ( '%|with %c:\n%+%c%-', 0, 3),
+    'withasstmt':   ( '%|with %c as %c:\n%+%c%-', 0, 2, 3),
+    'passstmt':		( '%|pass\n', ),
+    'STORE_FAST':	( '%{pattr}', ),
+    'kv':		( '%c: %c', 3, 1 ),
+    'kv2':		( '%c: %c', 1, 2 ),
+    'mapexpr':		( '{%[1]C}', (0, maxint, ', ') ),
+    'importstmt': ( '%|import %c\n', 2),
+    'importfrom': ( '%|from %[2]{pattr} import %c\n', 3 ),
+    'importstar': ( '%|from %[2]{pattr} import *\n', ),
+}
+
+
+MAP_DIRECT = (TABLE_DIRECT, )
+MAP_R0 = (TABLE_R0, -1, 0)
+MAP_R = (TABLE_R, -1)
+
+MAP = {
+    'stmt':		MAP_R,
+    'call_function':	MAP_R,
+    'del_stmt':		MAP_R,
+    'designator':	MAP_R,
+    'exprlist':		MAP_R0,
+}
+
+PRECEDENCE = {
+    'build_list':           0,
+    'mapexpr':              0,
+    'unary_convert':        0,
+    'dictcomp':             0,
+    'setcomp':              0,
+    'list_compr':           0,
+    'genexpr':              0,
+
+    'load_attr':            2,
+    'binary_subscr':        2,
+    'binary_subscr2':       2,
+    'slice0':               2,
+    'slice1':               2,
+    'slice2':               2,
+    'slice3':               2,
+    'buildslice2':          2,
+    'buildslice3':          2,
+    'call_function':        2,
+
+    'BINARY_POWER':         4,
+
+    'unary_expr':           6,
+
+    'BINARY_MULTIPLY':      8,
+    'BINARY_DIVIDE':        8,
+    'BINARY_TRUE_DIVIDE':   8,
+    'BINARY_FLOOR_DIVIDE':  8,
+    'BINARY_MODULO':        8,
+
+    'BINARY_ADD':           10,
+    'BINARY_SUBTRACT':      10,
+
+    'BINARY_LSHIFT':        12,
+    'BINARY_RSHIFT':        12,
+
+    'BINARY_AND':           14,
+
+    'BINARY_XOR':           16,
+
+    'BINARY_OR':            18,
+
+    'cmp':                  20,
+
+    'unary_not':            22,
+
+    'and':                  24,
+    'ret_and':              24,
+
+    'or':                   26,
+    'ret_or':               26,
+
+    'conditional':          28,
+    'conditionalnot':       28,
+    'ret_cond':             28,
+    'ret_cond_not':         28,
+
+    '_mklambda':            30,
+    'yield':                101,
+    'yield_from':           101
+}
+
+ASSIGN_TUPLE_PARAM = lambda param_name: \
+             AST('expr', [ Token('LOAD_FAST', pattr=param_name) ])
+
+escape = re.compile(r'''
+            (?P<prefix> [^%]* )
+            % ( \[ (?P<child> -? \d+ ) \] )?
+                ((?P<type> [^{] ) |
+                 ( [{] (?P<expr> [^}]* ) [}] ))
+        ''', re.VERBOSE)

uncompyle6/semantics/fragments.py

@@ -61,6 +61,7 @@ from uncompyle6.semantics import pysource
 from uncompyle6 import parser
 from uncompyle6.scanner import Token, Code, get_scanner
 from uncompyle6.semantics.check_ast import checker
+from uncompyle6.semantics.helper import print_docstring
 
 from uncompyle6.show import (
     maybe_show_asm,
@@ -68,17 +69,22 @@ from uncompyle6.show import (
     maybe_show_ast_param_default,
 )
 
-from uncompyle6.semantics.pysource import AST, INDENT_PER_LEVEL, NONE, PRECEDENCE, \
-     ParserError, TABLE_DIRECT, escape, find_globals, minint, MAP
+from uncompyle6.parsers.astnode import AST
+
+from uncompyle6.semantics.pysource import (
+    ParserError, find_globals, StringIO)
+
+from uncompyle6.semantics.consts import (
+    INDENT_PER_LEVEL, NONE, PRECEDENCE,
+    TABLE_DIRECT, escape, minint, MAP
+    )
 
 from uncompyle6.semantics.make_function import find_all_globals, find_none
 
 if PYTHON3:
     from itertools import zip_longest
-    from io import StringIO
 else:
     from itertools import izip_longest as zip_longest
-    from StringIO import StringIO
 
 
 from spark_parser import DEFAULT_DEBUG as PARSER_DEFAULT_DEBUG
@@ -96,14 +102,14 @@ TABLE_DIRECT_FRAGMENT = {
     'importstmt':	( '%|import %c%x\n', 2, (2, (0, 1)), ),
     'importfrom':	( '%|from %[2]{pattr}%x import %c\n', (2, (0, 1)), 3),
     'importmultiple':   ( '%|import%b %c%c\n', 0, 2, 3 ),
-    'list_for':	  	(' for %c%x in %c%c', 2, (2,(1,)), 0, 3 ),
-    'forstmt':	  	( '%|for%b %c%x in %c:\n%+%c%-\n\n', 0, 3, (3, (2,)), 1, 4 ),
+    'list_for':	  	(' for %c%x in %c%c', 2, (2, (1, )), 0, 3 ),
+    'forstmt':	  	( '%|for%b %c%x in %c:\n%+%c%-\n\n', 0, 3, (3, (2, )), 1, 4 ),
     'forelsestmt': 	(
         '%|for %c in %c%x:\n%+%c%-%|else:\n%+%c%-\n\n', 3, (3, (2,)), 1, 4, -2),
     'forelselaststmt':	(
         '%|for %c%x in %c:\n%+%c%-%|else:\n%+%c%-', 3, (3, (2,)), 1, 4, -2),
     'forelselaststmtl':	(
-        '%|for %c%x in %c:\n%+%c%-%|else:\n%+%c%-\n\n', 3, (3, (2,)), 1, 4, -2),
+        '%|for %c%x in %c:\n%+%c%-%|else:\n%+%c%-\n\n', 3, (3, (2, )), 1, 4, -2),
 
     'whilestmt':	( '%|while%b %c:\n%+%c%-\n\n', 0, 1, 2 ),
     'whileelsestmt':	( '%|while%b %c:\n%+%c%-%|else:\n%+%c%-\n\n', 0, 1, 2, -2 ),
@@ -162,7 +168,7 @@ class FragmentsWalker(pysource.SourceWalker, object):
                  None)
 
     def set_pos_info(self, node, start, finish, name=None):
-        if name == None: name = self.name
+        if name is None: name = self.name
         if hasattr(node, 'offset'):
             self.offsets[name, node.offset] = \
               NodeInfo(node = node, start = start, finish = finish)
@@ -582,7 +588,7 @@ class FragmentsWalker(pysource.SourceWalker, object):
         self.set_pos_info(node[-3], start, len(self.f.getvalue()))
         start = len(self.f.getvalue())
         self.preorder(ast[iter_index])
-        self.set_pos_info(iter_index, start, len(self.f.getvalue()))
+        self.set_pos_info(ast[iter_index], start, len(self.f.getvalue()))
         self.prec = p
 
     def comprehension_walk3(self, node, iter_index, code_index=-5):
@@ -621,7 +627,7 @@ class FragmentsWalker(pysource.SourceWalker, object):
             n = ast[iter_index]
             assert n == 'list_iter'
 
-        ## FIXME: I'm not totally sure this is right.
+        # FIXME: I'm not totally sure this is right.
 
         # find innermost node
         if_node = None
@@ -1687,7 +1693,7 @@ class FragmentsWalker(pysource.SourceWalker, object):
 
         if len(code.co_consts)>0 and code.co_consts[0] is not None and not isLambda: # ugly
             # docstring exists, dump it
-            self.print_docstring(indent, code.co_consts[0])
+            print_docstring(self, indent, code.co_consts[0])
 
         code._tokens = None # save memory
         assert ast == 'stmts'

uncompyle6/semantics/helper.py

@@ -0,0 +1,135 @@
+import sys
+from uncompyle6 import PYTHON3
+if PYTHON3:
+    minint = -sys.maxsize-1
+    maxint = sys.maxsize
+else:
+    minint = -sys.maxint-1
+    maxint = sys.maxint
+
+def print_docstring(self, indent, docstring):
+    try:
+        if docstring.find('"""') == -1:
+            quote = '"""'
+        else:
+            quote = "'''"
+    except:
+        return False
+    self.write(indent)
+    if not PYTHON3 and not isinstance(docstring, str):
+        # Must be unicode in Python2
+        self.write('u')
+        docstring = repr(docstring.expandtabs())[2:-1]
+    else:
+        docstring = repr(docstring.expandtabs())[1:-1]
+
+    for (orig, replace) in (('\\\\', '\t'),
+                            ('\\r\\n', '\n'),
+                            ('\\n', '\n'),
+                            ('\\r', '\n'),
+                            ('\\"', '"'),
+                            ("\\'", "'")):
+        docstring = docstring.replace(orig, replace)
+
+    # Do a raw string if there are backslashes but no other escaped characters:
+    # also check some edge cases
+    if ('\t' in docstring
+        and '\\' not in docstring
+        and len(docstring) >= 2
+        and docstring[-1] != '\t'
+        and (docstring[-1] != '"'
+             or docstring[-2] == '\t')):
+        self.write('r') # raw string
+        # restore backslashes unescaped since raw
+        docstring = docstring.replace('\t', '\\')
+    else:
+        # Escape '"' if it's the last character, so it doesn't
+        # ruin the ending triple quote
+        if len(docstring) and docstring[-1] == '"':
+            docstring = docstring[:-1] + '\\"'
+        # Restore escaped backslashes
+        docstring = docstring.replace('\t', '\\\\')
+    # Escape triple quote when needed
+    if quote == '""""':
+        docstring = docstring.replace('"""', '\\"\\"\\"')
+    lines = docstring.split('\n')
+    calculate_indent = maxint
+    for line in lines[1:]:
+        stripped = line.lstrip()
+        if len(stripped) > 0:
+            calculate_indent = min(calculate_indent, len(line) - len(stripped))
+    calculate_indent = min(calculate_indent, len(lines[-1]) - len(lines[-1].lstrip()))
+    # Remove indentation (first line is special):
+    trimmed = [lines[0]]
+    if calculate_indent < maxint:
+        trimmed += [line[calculate_indent:] for line in lines[1:]]
+
+    self.write(quote)
+    if len(trimmed) == 0:
+        self.println(quote)
+    elif len(trimmed) == 1:
+        self.println(trimmed[0], quote)
+    else:
+        self.println(trimmed[0])
+        for line in trimmed[1:-1]:
+            self.println( indent, line )
+        self.println(indent, trimmed[-1], quote)
+    return True
+
+# if __name__ == '__main__':
+#     if PYTHON3:
+#         from io import StringIO
+#     else:
+#         from StringIO import StringIO
+#     class PrintFake():
+#         def __init__(self):
+#             self.pending_newlines = 0
+#             self.f = StringIO()
+
+#         def write(self, *data):
+#             if (len(data) == 0) or (len(data) == 1 and data[0] == ''):
+#                 return
+#             out = ''.join((str(j) for j in data))
+#             n = 0
+#             for i in out:
+#                 if i == '\n':
+#                     n += 1
+#                     if n == len(out):
+#                         self.pending_newlines = max(self.pending_newlines, n)
+#                         return
+#                 elif n:
+#                     self.pending_newlines = max(self.pending_newlines, n)
+#                     out = out[n:]
+#                     break
+#                 else:
+#                     break
+
+#             if self.pending_newlines > 0:
+#                 self.f.write('\n'*self.pending_newlines)
+#                 self.pending_newlines = 0
+
+#             for i in out[::-1]:
+#                 if i == '\n':
+#                     self.pending_newlines += 1
+#                 else:
+#                     break
+
+#             if self.pending_newlines:
+#                 out = out[:-self.pending_newlines]
+#             self.f.write(out)
+#         def println(self, *data):
+#             if data and not(len(data) == 1 and data[0] ==''):
+#                 self.write(*data)
+#             self.pending_newlines = max(self.pending_newlines, 1)
+#             return
+#         pass
+
+#     for doc in (
+#             "Now is the time",
+#             r'''func placeholder - with ("""\nstring\n""")''',
+#             r'''func placeholder - ' and with ("""\nstring\n""")''',
+#             r"""func placeholder - ' and with ('''\nstring\n''') and \"\"\"\nstring\n\"\"\" """
+#             ):
+#         o = PrintFake()
+#         print_docstring(o, '  ', doc)
+#         print(o.f.getvalue())

uncompyle6/semantics/make_function.py

@@ -8,6 +8,7 @@ from uncompyle6.scanner import Code
 from uncompyle6.parsers.astnode import AST
 from uncompyle6 import PYTHON3
 from uncompyle6.semantics.parser_error import ParserError
+from uncompyle6.semantics.helper import print_docstring
 
 if PYTHON3:
     from itertools import zip_longest
@@ -37,7 +38,7 @@ def find_globals(node, globs):
 def find_none(node):
     for n in node:
         if isinstance(n, AST):
-            if not n in ('return_stmt', 'return_if_stmt'):
+            if n not in ('return_stmt', 'return_if_stmt'):
                 if find_none(n):
                     return True
         elif n.type == 'LOAD_CONST' and n.pattr is None:
@@ -70,7 +71,11 @@ def make_function3_annotate(self, node, isLambda, nested=1,
     # MAKE_FUNCTION_... or MAKE_CLOSURE_...
     assert node[-1].type.startswith('MAKE_')
 
-    annotate_tuple = node[annotate_last]
+    annotate_tuple = None
+    for annotate_last in range(len(node)-1, -1, -1):
+        if node[annotate_last] == 'annotate_tuple':
+            annotate_tuple = node[annotate_last]
+            break
     annotate_args = {}
 
     if (annotate_tuple == 'annotate_tuple'
@@ -147,13 +152,19 @@ def make_function3_annotate(self, node, isLambda, nested=1,
     suffix = ''
     for param in paramnames[:i]:
         self.write(suffix, param)
-        if param in annotate_args:
-            value, string = annotate_args[param]
-            if string:
-                self.write(': "%s"' % value)
-            else:
-                self.write(': %s' % value)
         suffix = ', '
+        if param in annotate_tuple[0].attr:
+            p = annotate_tuple[0].attr.index(param)
+            self.write(': ')
+            self.preorder(node[p])
+            if (line_number != self.line_number):
+                suffix = ",\n" + indent
+                line_number = self.line_number
+            # value, string = annotate_args[param]
+            # if string:
+            #     self.write(': "%s"' % value)
+            # else:
+            #     self.write(': %s' % value)
 
     suffix = ', ' if i > 0 else ''
     for n in node:
@@ -162,7 +173,10 @@ def make_function3_annotate(self, node, isLambda, nested=1,
             param = paramnames[i]
             self.write(param)
             if param in annotate_args:
-                self.write(':"%s' % annotate_args[param])
+                aa = annotate_args[param]
+                if isinstance(aa, tuple):
+                    aa = aa[0]
+                self.write(': "%s"' % aa)
             self.write('=')
             i += 1
             self.preorder(n)
@@ -188,6 +202,9 @@ def make_function3_annotate(self, node, isLambda, nested=1,
         i = 0
         for n in node[0]:
             if n == 'kwarg':
+                if (line_number != self.line_number):
+                    self.write("\n" + indent)
+                    line_number = self.line_number
                 self.write('%s=' % n[0].pattr)
                 self.preorder(n[1])
                 if i < last:
@@ -206,19 +223,24 @@ def make_function3_annotate(self, node, isLambda, nested=1,
         self.write(": ")
     else:
         self.write(')')
-        if 'return' in annotate_args:
-            value, string = annotate_args['return']
-            if string:
-                self.write(' -> "%s"' % value)
-            else:
-                self.write(' -> %s' % value)
+        if 'return' in annotate_tuple[0].attr:
+            if (line_number != self.line_number):
+                self.write("\n" + indent)
+                line_number = self.line_number
+            self.write(' -> ')
+            # value, string = annotate_args['return']
+            # if string:
+            #     self.write(' -> "%s"' % value)
+            # else:
+            #     self.write(' -> %s' % value)
+            self.preorder(node[annotate_last-1])
 
         self.println(":")
 
     if (len(code.co_consts) > 0 and
         code.co_consts[0] is not None and not isLambda): # ugly
         # docstring exists, dump it
-        self.print_docstring(indent, code.co_consts[0])
+        print_docstring(self, self.indent, code.co_consts[0])
 
     code._tokens = None # save memory
     assert ast == 'stmts'
@@ -353,7 +375,7 @@ def make_function2(self, node, isLambda, nested=1, codeNode=None):
 
     if len(code.co_consts) > 0 and code.co_consts[0] is not None and not isLambda: # ugly
         # docstring exists, dump it
-        self.print_docstring(indent, code.co_consts[0])
+        print_docstring(self, indent, code.co_consts[0])
 
     code._tokens = None # save memory
     assert ast == 'stmts'
@@ -542,7 +564,7 @@ def make_function3(self, node, isLambda, nested=1, codeNode=None):
 
     if len(code.co_consts) > 0 and code.co_consts[0] is not None and not isLambda: # ugly
         # docstring exists, dump it
-        self.print_docstring(self.indent, code.co_consts[0])
+        print_docstring(self, self.indent, code.co_consts[0])
 
     code._tokens = None # save memory
     assert ast == 'stmts'