3.x funciton and annotation bug fixes

next_offset, op_size, has_argument

.gitignore

@@ -16,3 +16,5 @@
 /unpyc
 __pycache__
 build
+/.venv*
+/.idea

.travis.yml

@@ -9,9 +9,10 @@ python:
   - '3.3'
   - '3.4'
   - '3.2'
+  - '3.6'
 
 install:
-- pip install -r requirements.txt
+- pip install -e .
 - pip install -r requirements-dev.txt
 
 script:

ChangeLog

@@ -1,6 +1,792 @@
+2017-06-25  rocky <rb@dustyfeet.com>
+
+	* uncompyle6/version.py: Get ready for release 2.11.1
+
+2017-06-24  rocky <rb@dustyfeet.com>
+
+	* __pkginfo__.py, uncompyle6/scanner.py,
+	uncompyle6/scanners/scanner2.py, uncompyle6/scanners/scanner3.py,
+	uncompyle6/scanners/scanner30.py, uncompyle6/semantics/pysource.py: 
+	Use xdis' instruction offset calculation fns..  next_offset, op_size, has_argument
+
+2017-06-19  rocky <rb@dustyfeet.com>
+
+	* uncompyle6/semantics/pysource.py: Python 2 sometimes need
+	str->uncode in writing?
+
+2017-06-19  rocky <rb@dustyfeet.com>
+
+	* uncompyle6/semantics/pysource.py: Allow deparsed out to be str as
+	well as unicode
+
+2017-06-18  rocky <rb@dustyfeet.com>
+
+	* ChangeLog, NEWS, uncompyle6/version.py: Get ready for release
+	2.11.0
+
+2017-06-13  rocky <rb@dustyfeet.com>
+
+	* uncompyle6/semantics/fragments.py: Adjust nodeInfo if it is a
+	Token
+
+2017-06-13  rocky <rb@dustyfeet.com>
+
+	* uncompyle6/semantics/fragments.py: Add nonterminal node in
+	extractInfo
+
+2017-06-10  rocky <rb@dustyfeet.com>
+
+	* uncompyle6/semantics/fragments.py,
+	uncompyle6/semantics/make_function.py: Fragment tag more expressions Revise make_function3 comment wrt args and kwargs
+
+2017-06-10  rocky <rb@dustyfeet.com>
+
+	* uncompyle6/semantics/fragments.py: Fragment tag array subscripts
+
+2017-06-10  R. Bernstein <rocky@users.noreply.github.com>
+
+	* README.rst: Create README.rst
+
+2017-06-10  R. Bernstein <rocky@users.noreply.github.com>
+
+	* README.rst: Create README.rst
+
+2017-06-10  rocky <rb@dustyfeet.com>
+
+	* uncompyle6/semantics/fragments.py: Set YIELD_VALUE offset in a
+	<yield> expr
+
+2017-06-10  rocky <rb@dustyfeet.com>
+
+	* uncompyle6/semantics/make_function.py: Python 3.2 MAKE_FUNCTION
+	again..  Was handling bug32/01_named_and_kwargs.py wrong again
+
+2017-06-09  R. Bernstein <rocky@users.noreply.github.com>
+
+	* : Merge pull request #119 from rocky/scan-longconstant Simplify access to L65536 ...
+
+2017-06-09  rocky <rb@dustyfeet.com>
+
+	* uncompyle6/semantics/make_function.py: Attempt to document the
+	MAKE_FUNCTION/MAKE_LAMBDA mess...  in Python 3.0+
+
+2017-06-08  rocky <rb@dustyfeet.com>
+
+	* uncompyle6/semantics/make_function.py: Correct make_function3 for
+	Pytohn 3.2
+
+2017-06-08  rocky <rb@dustyfeet.com>
+
+	* uncompyle6/semantics/pysource.py: Disable "continue" removal in
+	pysource.py "continue" could be the only statement and then removing it might
+	lead to a dangling "else".
+
+2017-06-07  rocky <rb@dustyfeet.com>
+
+	* uncompyle6/semantics/fragments.py: Mark "pass" offsets.  Start routine to find previous node.
+
+2017-06-06  rocky <rb@dustyfeet.com>
+
+	* uncompyle6/parsers/parse3.py, uncompyle6/semantics/fragments.py: 
+	Remove hacky fragments try fixup...  hacky call_function code is also not needed or will be reinstated
+	properly. Better grammar structure for Python 3.6 call_function.
+
+2017-06-05  rocky <rb@dustyfeet.com>
+
+	* uncompyle6/parsers/parse3.py, uncompyle6/parsers/parse36.py,
+	uncompyle6/scanners/scanner36.py: BUILD_{MAP,TUPLE}_UNPACK &
+	CALL_FUNCTION_EX_KW...  Bang on these in 3.6. Not totally succesfull right now.  In fact a
+	regression on one of the test cases
+
+2017-06-05  rocky <rb@dustyfeet.com>
+
+	* uncompyle6/semantics/fragments.py: Important fragments bug fix...  start, finish that had been adjusted wasn't getting reflected in
+	final returned deparsed.offsets dictionary. Redo keeping API
+	compatibility, i.e we still use namedtuple NodeInfo.
+
+2017-06-04  rocky <rb@dustyfeet.com>
+
+	* uncompyle6/parsers/parse3.py, uncompyle6/semantics/pysource.py: 
+	Python 3.5 *args with kwargs handling.  3.5 is a snowflake here. Thank you, Python.  Fully fixes Issue 95.  3.6 is broken on this source, but for a *different* reason. Sigh.
+
+2017-06-03  rocky <rb@dustyfeet.com>
+
+	* README.rst, __pkginfo__.py,
+	test/simple_source/bug35/04_CALL_FUNCTION_VAR_KW.py,
+	uncompyle6/semantics/fragments.py: Small changes.  fragment tag EXEC_STMT
+
+2017-06-03  rocky <rb@dustyfeet.com>
+
+	* .travis.yml: Streamline .travis.yml a little bit
+
+2017-06-03  rocky <rb@dustyfeet.com>
+
+	* __pkginfo__.py: We need six
+
+2017-06-03  rocky <rb@dustyfeet.com>
+
+	* README.rst, circle.yml, requirements-dev.txt: Go over
+	administrivia
+
+2017-06-03  rocky <rb@dustyfeet.com>
+
+	* ChangeLog, NEWS, uncompyle6/version.py: Get ready for release
+	2.10.1
+
+2017-06-03  rocky <rb@dustyfeet.com>
+
+	* uncompyle6/semantics/fragments.py,
+	uncompyle6/semantics/pysource.py: Fragment bugs fragment.py: * deparse_code_aorund_offset: was sometimes returning the wrong type * capture function name offset * lint imports pysource.py: use a clearer variable name
+
+2017-06-02  rocky <rb@dustyfeet.com>
+
+	* uncompyle6/semantics/fragments.py: Track changes in ifelstmtr..  in fragments from pysource
+
+2017-05-30  rocky <rb@dustyfeet.com>
+
+	* ChangeLog, NEWS, uncompyle6/version.py: Get ready for release
+	2.10.0
+
+2017-05-30  rocky <rb@dustyfeet.com>
+
+	* uncompyle6/semantics/fragments.py: Python 3.6 makefunction
+	handling for fragments
+
+2017-05-23  rocky <rb@dustyfeet.com>
+
+	* uncompyle6/semantics/pysource.py: Fix up 3.6 unmapexpr
+
+2017-05-23  rocky <rb@dustyfeet.com>
+
+	* uncompyle6/semantics/pysource.py: Fix up retreiving "async"
+	property on 3.6
+
+2017-05-23  rocky <rb@dustyfeet.com>
+
+	* uncompyle6/semantics/pysource.py: Fix bug in a 3.6 class name.
+
+2017-05-23  rocky <rb@dustyfeet.com>
+
+	* uncompyle6/semantics/fragments.py,
+	uncompyle6/semantics/pysource.py: Add fuzzy offset deparse lookup
+
+2017-05-21  rocky <rb@dustyfeet.com>
+
+	* uncompyle6/scanners/scanner3.py: Correct EXTENDED_ARG handling on
+	Python 3.6...  where it can appear several times and xdis may handle it as well.
+	It possibly in other versions bug since EXTENDED_ARG is used so
+	rarely there because it has such a high value 1<<16, it's hard to
+	test and determine that.
+
+2017-05-20  rocky <rb@dustyfeet.com>
+
+	* uncompyle6/semantics/pysource.py: Worse results. Revert some of
+	the last changes
+
+2017-05-20  rocky <rb@dustyfeet.com>
+
+	* uncompyle6/parsers/parse3.py, uncompyle6/semantics/pysource.py: 
+	More explicit about 3.5 UNMAP_PACK Have to reduce 3.5 bytecode testing for now, code is more solid.
+
+2017-05-19  rocky <rb@dustyfeet.com>
+
+	* uncompyle6/parsers/parse3.py, uncompyle6/parsers/parse36.py,
+	uncompyle6/scanners/scanner3.py: Simplify EXTENDED_ARG on 3.x We largely remove them and fold them itno the next op.
+	MAKE_FUNCTION though before 3.6 is an exception as that indicates an
+	annotated function
+
+2017-05-19  rocky <rb@dustyfeet.com>
+
+	* uncompyle6/scanners/scanner26.py: EXTENDED_ARG is implemented in
+	2.6
+
+2017-05-19  rocky <rb@dustyfeet.com>
+
+	* test/simple_source/expression/06_huge_list.py,
+	uncompyle6/parsers/parse3.py, uncompyle6/semantics/pysource.py: Fix
+	EXTENDED_ARG for long lists, sets, maps
+
+2017-05-19  rocky <rb@dustyfeet.com>
+
+	* uncompyle6/scanners/scanner3.py: Another attempt at getting
+	get_target() correct
+
+2017-05-19  rocky <rb@dustyfeet.com>
+
+	* uncompyle6/parsers/parse3.py: Bug in pypy JUMP_IF_NOT_DEBUG
+	handling
+
+2017-05-19  rocky <rb@dustyfeet.com>
+
+	* uncompyle6/parsers/parse36.py, uncompyle6/scanners/scanner3.py: 
+	EXTENDED_ARG handling...  get_target() wasn't taking into account EXTENDED_ARG before opcode.  This is mostly relevant in Python 3.6 where the max size before
+	needing EXTENDED_ARG has been reduced to 256, but theoretically
+	possible in earlier versions.
+
+2017-05-18  rocky <rb@dustyfeet.com>
+
+	* __pkginfo__.py: Enforce using xdis >=3.3.1 ..  to pick up bug fixes to 3.6 in xdis
+
+2017-05-17  rocky <rb@dustyfeet.com>
+
+	* __pkginfo__.py, uncompyle6/parsers/parse36.py,
+	uncompyle6/scanners/scanner3.py: Small changes....  * __pkginfo__.py: Need spark parser 1.6.1 for corrected
+	remove_rules() fn * parser36.py: remove replaced Python3 rules * scanner3.py: corrected comment. Thanks to moagstar here.  *
+
+2017-05-16  rocky <rb@dustyfeet.com>
+
+	* uncompyle6/parsers/parse36.py: Fix broken CI on 3.6...  Another grammar rule replacing SETUP_LOOP with setup_loop
+
+2017-05-16  rocky <rb@dustyfeet.com>
+
+	* uncompyle6/parsers/parse36.py: More EXTENDED_ARGS on 3.6
+
+2017-05-16  rocky <rb@dustyfeet.com>
+
+	* uncompyle6/parsers/parse36.py: extend use of EXTENDED_ARGS in 3.6 switching to a wordcode seems to have made opcode fields smaller so
+	we need EXTENDED_ARG more?
+
+2017-05-16  rocky <rb@dustyfeet.com>
+
+	* uncompyle6/parsers/parse36.py, uncompyle6/semantics/pysource.py: 
+	Allow LOAD_CONST EXTENDED_ARG
+
+2017-05-15  rocky <rb@dustyfeet.com>
+
+	* uncompyle6/parsers/parse3.py: Reinstate 3.6 listcomp rule
+
+2017-05-15  rocky <rb@dustyfeet.com>
+
+	* uncompyle6/parsers/parse3.py: Bang on 3.6 MAKE_FUNCTION some more
+
+2017-05-14  rocky <rb@dustyfeet.com>
+
+	* uncompyle6/semantics/pysource.py: towards fixing a
+	3.5.CALL_FUNCTONI_VAR bug
+
+2017-05-14  rocky <rb@dustyfeet.com>
+
+	* test/simple_source/bug35/04_CALL_FUNCTION_VAR_KW.py,
+	uncompyle6/parsers/parse3.py: Python 3.5 kw arg can be an expr Fixes Issue #95
+
+2017-05-14  R. Bernstein <rocky@users.noreply.github.com>
+
+	* : Merge pull request #117 from rocky/3.6-MAKE_FUNCTION 3.6 make function
+
+2017-05-13  rocky <rb@dustyfeet.com>
+
+	* uncompyle6/scanners/scanner3.py: MAKE_FUNCTION_FLAGS can be a
+	simpler tuple
+
+2017-05-13  rocky <rb@dustyfeet.com>
+
+	* uncompyle6/parsers/parse3.py: Grammar rules for Python 3.6
+	MAKE_FUNCTION
+
+2017-05-13  rocky <rb@dustyfeet.com>
+
+	* README.rst, uncompyle6/parsers/parse3.py,
+	uncompyle6/parsers/parse36.py, uncompyle6/semantics/pysource.py: 
+	Bang on 3.6 MAKE_FUNCTION a bit more parse3.py, parse36.py: adding return_closure rule tags what's going
+	on with this rule pysource.py: start changing semantic rules to support code changed
+	by new make_function semantics README.rst: typo
+
+2017-05-13  rocky <rb@dustyfeet.com>
+
+	* uncompyle6/scanners/scanner3.py: Typo
+
+2017-05-12  rocky <rb@dustyfeet.com>
+
+	* uncompyle6/parsers/parse27.py: Bug in 2.7 decompiling ourself! Troublesome file was uncompyle6.semantics.pysource.engine()
+
+2017-05-11  R. Bernstein <rocky@users.noreply.github.com>
+
+	* : Merge pull request #113 from grkov90/patch-1 Fixed out_base bug
+
+2017-05-11  rocky <rb@dustyfeet.com>
+
+	* uncompyle6/parsers/parse3.py, uncompyle6/scanners/scanner3.py,
+	uncompyle6/semantics/make_function.py: WIP: start 3.6 MAKE_FUNCTION
+	handling
+
+2017-05-11  Daniel Bradburn <moagstar@gmail.com>
+
+	* : Merge pull request #116 from moagstar/function_call_keyword_only Added support for Python 3.6 CALL_FUNCTION_KW
+
+2017-05-10  Daniel Bradburn <moagstar@gmail.com>
+
+	* uncompyle6/semantics/pysource.py: Fixed bug in compiling double
+	star arg only function calls where the closing parenthesis would be
+	missed
+
+2017-05-10  Daniel Bradburn <moagstar@gmail.com>
+
+	* requirements-dev.txt: Adding requirement for pytest >= 3.0 to fix
+	strange INTERNALERROR in combination with hypothesis when using
+	pytest 2.6.4
+
+2017-05-10  Daniel Bradburn <moagstar@gmail.com>
+
+	* pytest/test_CALL_FUNCTION_KW.sh, pytest/test_function_call.py,
+	uncompyle6/parsers/parse3.py, uncompyle6/parsers/parse36.py,
+	uncompyle6/scanners/scanner36.py, uncompyle6/semantics/pysource.py: 
+	Added support for support for Python 3.6 CALL_FUNCTION_KW
+
+2017-05-08  rocky <rb@dustyfeet.com>
+
+	* appveyor.yml, test/test_pyenvlib.py,
+	uncompyle6/semantics/pysource.py: pysource guard and another
+	appveyor test
+
+2017-05-08  rocky <rb@dustyfeet.com>
+
+	* appveyor.yml: appveyor take 2
+
+2017-05-08  rocky <rb@dustyfeet.com>
+
+	* appveyor.yml, appveyor/install.ps1, appveyor/run_with_env.cmd: Try
+	appveyor
+
+2017-05-07  rocky <rb@dustyfeet.com>
+
+	* uncompyle6/semantics/pysource.py: More guarded CONTINUE deletion
+
+2017-05-07  rocky <rb@dustyfeet.com>
+
+	* uncompyle6/scanner.py, uncompyle6/scanners/scanner3.py,
+	uncompyle6/semantics/pysource.py: Reduce spurious "continue"
+	statements
+
+2017-05-07  rocky <rb@dustyfeet.com>
+
+	* test/Makefile: --weak-verify on 3.3 with inclusion of last commit Note that the result is sematically equivalent, so it is is correct.
+
+2017-05-07  rocky <rb@dustyfeet.com>
+
+	* test/simple_source/looping/12_if_while_true_pass.py,
+	uncompyle6/scanners/scanner3.py: Python 3.x control-flow bug...  "pass" statement inside "while True"
+
+2017-05-07  rocky <rb@dustyfeet.com>
+
+	* HOW-TO-REPORT-A-BUG.md: Small typo
+
+2017-05-07  rocky <rb@dustyfeet.com>
+
+	* uncompyle6/scanners/scanner3.py: Fix improper COME_FROM_EXCEPT in
+	Python 3.3+
+
+2017-05-06  rocky <rb@dustyfeet.com>
+
+	* uncompyle6/parsers/parse33.py: python 3.3 while True parsing bug
+
+2017-05-06  rocky <rb@dustyfeet.com>
+
+	* ChangeLog, NEWS, uncompyle6/version.py: Get ready for release
+	2.9.11
+
+2017-05-06  rocky <rb@dustyfeet.com>
+
+	* test/Makefile: fix PYTHON variable setting in test/Makefile
+
+2017-05-06  rocky <rb@dustyfeet.com>
+
+	* test/simple_source/bug32/01_try_except_raise.py,
+	test/simple_source/bug32/03_if.py, uncompyle6/parsers/parse32.py,
+	uncompyle6/parsers/parse33.py: Fix more Python3.2 parser errors
+
+2017-05-05  rocky <rb@dustyfeet.com>
+
+	* uncompyle6/parsers/parse32.py, uncompyle6/scanners/scanner3.py: 
+	Improve Python 3.2 decompilation ...  by removing a lot of the control-flow labels of 3.3+
+
+2017-05-05  rocky <rb@dustyfeet.com>
+
+	* .travis.yml: Try CI testing on Python 3.6
+
+2017-05-03  Gregory <grkov90@gmail.com>
+
+	* uncompyle6/main.py: Some fix
+
+2017-05-03  Gregory <grkov90@gmail.com>
+
+	* uncompyle6/main.py: Fixed out_base bug Variable filename using in for tags uncompyle6 -o haven't worked argument -o haven't worked
+
+2017-05-02  rocky <rb@dustyfeet.com>
+
+	* test/simple_source/bug35/01_map_unpack.py, uncompyle6/parser.py,
+	uncompyle6/parsers/parse3.py, uncompyle6/parsers/parse35.py,
+	uncompyle6/semantics/pysource.py: Bang more on BUIlD_MAP_UNPACK there are still bugs. Note: {**{'x': 1}, **{'y': 2}} and {{'x': 1}, **{'y': 2}} generate the same Python 3.5+ bytecode.
+
+2017-05-02  rocky <rb@dustyfeet.com>
+
+	* test/simple_source/bug35/01_map_unpack.py, uncompyle6/parser.py,
+	uncompyle6/parsers/parse3.py, uncompyle6/parsers/parse35.py: 
+	BUILD_MAP_UNPACK'ing of dictionaries in 3.5
+
+2017-05-01  rocky <rb@dustyfeet.com>
+
+	* uncompyle6/semantics/pysource.py: Remove extra unpack *. Issue #98
+
+2017-04-29  R. Bernstein <rocky@users.noreply.github.com>
+
+	* HISTORY.md: Update HISTORY.md
+
+2017-04-29  rocky <rb@dustyfeet.com>
+
+	* test/simple_source/bug35/01_map_unpack.py,
+	uncompyle6/parsers/parse35.py, uncompyle6/semantics/pysource.py: 
+	Handle BUILD_MAP_UNPACK in a build_list
+
+2017-04-27  rocky <rb@dustyfeet.com>
+
+	* uncompyle6/semantics/pysource.py: A hacky way to get
+	CALL_FUNCTION_EX_KW to work.
+
+2017-04-26  rocky <rb@dustyfeet.com>
+
+	* uncompyle6/semantics/pysource.py: remove debug code
+
+2017-04-25  rocky <rb@dustyfeet.com>
+
+	* test/simple_source/bug36/01_call_function.py,
+	uncompyle6/parsers/parse36.py, uncompyle6/scanners/scanner36.py,
+	uncompyle6/semantics/pysource.py: Python 3.6 CALL_FUNCTION_EX first
+	attempt
+
+2017-04-22  rocky <rb@dustyfeet.com>
+
+	* uncompyle6/parser.py, uncompyle6/parsers/parse34.py: Reduse scope
+	of LOAD_ASSERT as expr to 3.4+
+
+2017-04-22  rocky <rb@dustyfeet.com>
+
+	* uncompyle6/parser.py, uncompyle6/verify.py: LOAD_ASSERT can also
+	be an expr This may have the undesirable property that assert statements might
+	get tagged with equivalant low-level Python code that uses "raise
+	AssertionError", but so be it.  Fixes #103
+
+2017-04-22  R. Bernstein <rocky@users.noreply.github.com>
+
+	* HISTORY.md: Update HISTORY.md
+
+2017-04-22  R. Bernstein <rocky@users.noreply.github.com>
+
+	* HISTORY.md: Update HISTORY.md
+
+2017-04-22  rocky <rb@dustyfeet.com>
+
+	* HISTORY.md: History keeps gettting amended
+
+2017-04-22  rocky <rb@dustyfeet.com>
+
+	* README.rst: Document Python 3.x status
+
+2017-04-22  rocky <rb@dustyfeet.com>
+
+	* test/simple_source/bug35/03_async_await.py,
+	uncompyle6/parsers/parse35.py, uncompyle6/semantics/pysource.py: Add
+	await expr Fixes #111
+
+2017-04-22  rocky <rb@dustyfeet.com>
+
+	* : Update test
+
+2017-04-22  rocky <rb@dustyfeet.com>
+
+	* test/simple_source/bug33/02_pos_args.py,
+	uncompyle6/parsers/parse3.py, uncompyle6/semantics/make_function.py: 
+	3.3+ bug in handling single kwarg after * Towards fixing issue #110
+
+2017-04-20  rocky <rb@dustyfeet.com>
+
+	* test/simple_source/bug35/02_async_for.py,
+	uncompyle6/parsers/parse35.py: Add async for with pass statement Fixes #109
+
+2017-04-19  rocky <rb@dustyfeet.com>
+
+	* test/simple_source/bug35/03_while-if-break.py,
+	uncompyle6/parsers/parse3.py: 3.5 ifelsestmtl grammar bug.  Fixes #108
+
+2017-04-18  rocky <rb@dustyfeet.com>
+
+	* test/simple_source/bug35/03_async_await.py,
+	uncompyle6/parsers/parse35.py: Expand await stmt handling Fixes #107
+
+2017-04-18  rocky <rb@dustyfeet.com>
+
+	* test/simple_source/bug33/01_delete_deref.py,
+	uncompyle6/parsers/parse32.py, uncompyle6/semantics/pysource.py: Add
+	DELETE_DEREF grammar rule Fixes Issue #106
+
+2017-04-17  rocky <rb@dustyfeet.com>
+
+	* test/simple_source/bug36/01_extended_arg.py,
+	test/simple_source/bug36/01_if_file.py: Rename test case to
+	something more appropriate
+
+2017-04-17  rocky <rb@dustyfeet.com>
+
+	* test/simple_source/bug36/01_if_file.py: Fix botched test case Thanks to Zm908 for pointing this out
+
+2017-04-16  rocky <rb@dustyfeet.com>
+
+	* uncompyle6/parsers/parse3.py: Comment on what's up with last
+	change
+
+2017-04-16  rocky <rb@dustyfeet.com>
+
+	* test/simple_source/bug22/03_if1.py,
+	test/simple_source/bug31/02_ifelse_comprehension.py,
+	uncompyle6/parsers/parse3.py: Python 3.x ifelse in comprehension Fixes Issue #91
+
+2017-04-16  rocky <rb@dustyfeet.com>
+
+	* : Add 2.7 complex test
+
+2017-04-15  rocky <rb@dustyfeet.com>
+
+	* test/simple_source/bug35/01_map_unpack.py,
+	uncompyle6/semantics/pysource.py: Correct bug in 3.5+ build_list
+	with UNPACK
+
+2017-04-15  R. Bernstein <rocky@users.noreply.github.com>
+
+	* HOW-TO-REPORT-A-BUG.md: Update HOW-TO-REPORT-A-BUG.md
+
+2017-04-15  R. Bernstein <rocky@users.noreply.github.com>
+
+	* HOW-TO-REPORT-A-BUG.md: Update HOW-TO-REPORT-A-BUG.md
+
+2017-04-15  rocky <rb@dustyfeet.com>
+
+	* test/simple_source/bug36/01_if_file.py,
+	uncompyle6/parsers/parse36.py: 3.6 generates Wonky EXTENDED_ARG in
+	expression Fixes Issue #102
+
+2017-04-15  rocky <rb@dustyfeet.com>
+
+	* HOW-TO-REPORT-A-BUG.md, MANIFEST.in: Add how to report a bug Add test case for ... if 1 else ...
+
+2017-04-14  rocky <rb@dustyfeet.com>
+
+	* test/simple_source/bug35/01_map_unpack.py,
+	uncompyle6/parsers/parse3.py, uncompyle6/parsers/parse35.py,
+	uncompyle6/semantics/pysource.py: Python 3.5+ BUILD_UNMAP_PACK rules Towards addressing Issue #98
+
+2017-04-14  rocky <rb@dustyfeet.com>
+
+	* uncompyle6/scanners/scanner3.py: Reduce adding RETURN_END_IF in
+	3.5+ The whole control flow determination has to be redone in a less
+	haphazard way using real flow-control analysis. Hopefully that's on
+	the way.  In the meantime we have this hack.
+
+2017-04-14  rocky <rb@dustyfeet.com>
+
+	* test/simple_source/bug27+/03_if_1_else.py,
+	test/simple_source/bug27+/03_if_true_else.py: Better names for a
+	test
+
+2017-04-13  rocky <rb@dustyfeet.com>
+
+	* test/simple_source/bug27+/03_if_true_else.py,
+	uncompyle6/parser.py, uncompyle6/parsers/parse3.py,
+	uncompyle6/semantics/consts.py: Add if1else. Fixes #101
+
+2017-04-13  rocky <rb@dustyfeet.com>
+
+	* uncompyle6/parsers/parse3.py: In 3.x come_from should include
+	COME_FROM_EXCEPT
+
+2017-04-13  rocky <rb@dustyfeet.com>
+
+	* uncompyle6/parsers/parse35.py: Towards fixing issue #92
+
+2017-04-13  rocky <rb@dustyfeet.com>
+
+	* uncompyle6/parsers/parse23.py, uncompyle6/semantics/pysource.py: 
+	Add Python 2.3 rule for "if 1: ..." Fully fixes #97 for Python 2.3. Python 2.4 was fixed in a previous
+	commit.
+
+2017-04-12  rocky <rb@dustyfeet.com>
+
+	* uncompyle6/parsers/parse3.py,
+	uncompyle6/semantics/make_function.py: annotate args type need to be
+	expr's not constants
+
+2017-04-12  rocky <rb@dustyfeet.com>
+
+	* uncompyle6/parsers/parse24.py: Handle Python 2.4 "if 1...."
+
+2017-04-11  rocky <rb@dustyfeet.com>
+
+	* test/simple_source/bug31/04_def_annotate.py,
+	uncompyle6/semantics/fragments.py,
+	uncompyle6/semantics/make_function.py: Bang on 3.x annotations
+
+2017-04-11  rocky <rb@dustyfeet.com>
+
+	* test/simple_source/bug31/04_def_annotate.py,
+	uncompyle6/parsers/parse3.py, uncompyle6/semantics/pysource.py: 
+	Towards fixing annotated decorator functions...  and annotate functions
+
+2017-04-10  rocky <rb@dustyfeet.com>
+
+	* uncompyle6/parsers/parse2.py, uncompyle6/scanners/scanner27.py,
+	uncompyle6/semantics/check_ast.py, uncompyle6/semantics/pysource.py: 
+	Misc bugs parse2.py: restore accidently-removed while1stmt rule scanner27.py:
+	grammar typo check_ast: add while1else to list of looping constructs
+	pysource.py: CALL_FUNCTION_VAR_KW_ARGS with positional args rule is
+	different?
+
+2017-04-10  rocky <rb@dustyfeet.com>
+
+	* test/simple_source/stmts/02_while1else.py,
+	uncompyle6/parsers/parse2.py, uncompyle6/parsers/parse3.py,
+	uncompyle6/parsers/parse35.py: Add more while1else grammar rules Towards addressing issue #93
+
+2017-04-09  rocky <rb@dustyfeet.com>
+
+	* : One more FUNCTION_VAR test for 3.3
+
+2017-04-09  rocky <rb@dustyfeet.com>
+
+	* test/simple_source/def/10_kw+pos_args-bug.py,
+	uncompyle6/parsers/parse3.py, uncompyle6/semantics/pysource.py: 
+	Another Python 3.5 FUNCTION_VAR bug Fixes #94
+
+2017-04-09  rocky <rb@dustyfeet.com>
+
+	* : commit 4199bc7f617e387fb03fc06939cd17366dc15c5e Author: rocky
+	<rb@dustyfeet.com> Date:   Sun Apr 9 05:30:45 2017 -0400
+
+2017-04-03  rocky <rb@dustyfeet.com>
+
+	* : commit 6773a66b99d07e48290a77dbbbe3c71cc39c31ba Author: rocky
+	<rb@dustyfeet.com> Date:   Mon Apr 3 06:53:12 2017 -0400
+
+2017-03-27  rocky <rb@dustyfeet.com>
+
+	* : commit a91cd716670be09d3cef34e1bb36a67f96f91712 Author: rocky
+	<rb@dustyfeet.com> Date:   Mon Mar 27 07:08:59 2017 -0400
+
+2017-03-19  rocky <rb@dustyfeet.com>
+
+	* __pkginfo__.py: Use more-recent xdis
+
+2017-03-15  rocky <rb@dustyfeet.com>
+
+	* HISTORY.md, test/simple_source/bug33/01_if_try_except.py: grammar
+	typo and add another test
+
+2017-03-12  rocky <rb@dustyfeet.com>
+
+	* uncompyle6/scanners/scanner3.py: Python 3.0 doesn't have
+	POP_JUMP_IF...
+
+2017-03-12  rocky <rb@dustyfeet.com>
+
+	* README.rst: Note problem in handling pathologically long lists
+
+2017-03-07  rocky <rb@dustyfeet.com>
+
+	* uncompyle6/scanners/scanner3.py: Small cleanup - remove
+	POP_JUMP_TF
+
+2017-03-05  rocky <rb@dustyfeet.com>
+
+	* pytest/test_grammar.py, uncompyle6/parsers/parse3.py,
+	uncompyle6/parsers/parse33.py, uncompyle6/scanners/scanner3.py: More
+	accurate ranges of try blocks in 3.x
+
+2017-03-05  rocky <rb@dustyfeet.com>
+
+	* test/simple_source/bug33/01_try_except.py: More accurate ranges of
+	try blocks in 3.x
+
+2017-03-04  R. Bernstein <rocky@users.noreply.github.com>
+
+	* : Merge pull request #84 from
+	moagstar/property_based_test_function_call Property based test function call
+
+2017-03-04  rocky <rb@dustyfeet.com>
+
+	* README.rst: README updates for 3.5 and 1.5
+
+2017-03-04  rocky <rb@dustyfeet.com>
+
+	* test/simple_source/bug32/01_named_and_kwargs.py,
+	uncompyle6/parsers/parse3.py: Bug found by hypothesis in creating
+	function calls
+
+2017-03-04  Daniel Bradburn <moagstar@gmail.com>
+
+	* pytest/test_function_call.py: marked all function call tests as
+	failing until they pass across all python versions
+
+2017-03-04  Daniel Bradburn <moagstar@gmail.com>
+
+	* pytest/test_function_call.py: added minimal examples for various
+	function call opcodes
+
+2017-03-04  Daniel Bradburn <moagstar@gmail.com>
+
+	* pytest/test_function_call.py: added property based test for
+	verifying uncompylation of function calls. A number of minimal
+	examples for the various function call opcodes have been generated
+	with the majority marked as expected failure until python 3.6 opcode
+	support is complete. I'm hoping this will make it easier to figure
+	out what needs to be done to support the new opcodes and changed
+	semntics for function calls
+
+2017-03-03  Daniel Bradburn <moagstar@gmail.com>
+
+	* pytest/test_function_call.py: reduced errors when generating
+	function call instances
+
+2017-03-03  Daniel Bradburn <moagstar@gmail.com>
+
+	* pytest/test_function_call.py: added test file for function calls
+
+2017-03-03  Daniel Bradburn <moagstar@gmail.com>
+
+	* .gitignore: added .idea to gitignore
+
+2017-03-03  Daniel Bradburn <moagstar@gmail.com>
+
+	* .gitignore: added .venv to gitignore
+
+2017-03-01  rocky <rb@dustyfeet.com>
+
+	* uncompyle6/scanner.py, uncompyle6/scanners/scanner2.py,
+	uncompyle6/scanners/scanner3.py, uncompyle6/verify.py: COME_FROM for
+	3.x POP_EXCEPT, DRY with op_name() ...  Start adding COME_FROMs for POP_EXCEPT in preparation for getting
+	tryelse blocks correct.  Simpler opname access functions:   - self.op_name(op) is self.opc.opname[op]   - self.op_name_from_offset(offset) is
+	  self.opc.opname[self.code[offset]] verify.py: not all offsets are ints
+
+2017-02-28  rocky <rb@dustyfeet.com>
+
+	* README.rst, uncompyle6/parser.py, uncompyle6/parsers/parse26.py: 
+	Python 2.6 a == b or c == d == 3 grammar bug
+
+2017-02-28  rocky <rb@dustyfeet.com>
+
+	* : 2.6 a == b or x == y == z bug
+
+2017-02-28  rocky <rb@dustyfeet.com>
+
+	* test/simple_source/bug26/03_double_equals.py,
+	uncompyle6/semantics/consts.py: Predidence of cmp_list: x == y == z The x, y, z should not have parenthesis around pairs of them (x ==
+	y) or (y == z)
+
+2017-02-28  rocky <rb@dustyfeet.com>
+
+	* uncompyle6/parser.py, uncompyle6/parsers/parse27.py: Python 2.7
+	check jump targets of "and"
+
 2017-02-25  rocky <rb@dustyfeet.com>
 
-	* uncompyle6/version.py: Get ready for release 2.9.10
+	* ChangeLog, NEWS, __pkginfo__.py, uncompyle6/version.py: Get ready
+	for release 2.9.10
 
 2017-02-25  rocky <rb@dustyfeet.com>
 

HISTORY.md

@@ -44,7 +44,8 @@ it appears that Hartmut did most of the work to get this code to
 accept the full Python language. He added precedence to the table
 specifiers, support for multiple versions of Python, the
 pretty-printing of docstrings, lists, and hashes. He also wrote test and verification routines of
-deparsed bytecode, and used this in an extensive set of tests that he also wrote. He could verify against the entire Python library.
+deparsed bytecode, and used this in an extensive set of tests that he also wrote. He says he could verify against the 
+entire Python library. However I have subsequently found small and relatively obscure bugs in the decompilation code. 
 
 decompyle2.2 was packaged for Debian (sarge) by
 [Ben Burton around 2002](https://packages.qa.debian.org/d/decompyle.html). As
@@ -65,10 +66,12 @@ code to handle first Python 2.3 and then 2.4 bytecodes. Because of
 jump optimization introduced in the CPython bytecode compiler at that
 time, various JUMP instructions were classifed as going backwards, and
 COME FROM instructions were reintroduced.  See
-RELEASE-2.4-CHANGELOG.txt for more details here. There wasn't a public
+[RELEASE-2.4-CHANGELOG.txt](https://github.com/rocky/python-uncompyle6/blob/master/DECOMPYLE-2.4-CHANGELOG.txt) 
+for more details here. There wasn't a public
 release of RELEASE-2.4 and bytecodes other than Python 2.4 weren't
 supported. Dan says the Python 2.3 version could verify the entire
-python library.
+Python library. But given subsequent bugs found like simply
+recognizing complex-number constants in bytecode, decompilation wasn't perfect.
 
 Next we get to ["uncompyle" and
 PyPI](https://pypi.python.org/pypi/uncompyle/1.1) and the era of
@@ -95,17 +98,17 @@ so. Then hamled made a few commits earler on, while Eike Siewertsen
 made a few commits later on. But mostly wibiti, and Guenther
 Starnberger got the code to where uncompyle2 was around 2012.
 
-In uncompyle2 decompilation of python bytecode 2.5 & 2.6 is done by
+In `uncompyle`, decompilation of python bytecode 2.5 & 2.6 is done by
 transforming the byte code into a a pseudo 2.7 python bytecode and is
 based on code from Eloi Vanderbeken.
 
-This project, uncompyle6, abandons that approach for various
+This project, `uncompyle6`, abandons that approach for various
 reasons. However the main reason is that we need offsets in fragment
 deparsing to be exactly the same, and the transformation process can
-remove instructions.  Adding instructions with psuedo_offsets is
+remove instructions.  _Adding_ instructions with psuedo offsets is
 however okay.
 
-Uncompyle6, however owes its existence to the fork of uncompyle2 by
+`Uncompyle6` however owes its existence to the fork of `uncompyle2` by
 Myst herie (Mysterie) whose first commit picks up at
 2012. I chose this since it seemed to have been at that time the most
 actively, if briefly, worked on. Also starting around 2012 is Dark
@@ -115,9 +118,12 @@ I started working on this late 2015, mostly to add fragment support.
 In that, I decided to make this runnable on Python 3.2+ and Python 2.6+
 while, handling Python bytecodes from Python versions 2.5+ and
 3.2+. In doing so, it has been expedient to separate this into three
-projects: load loading and disassembly (xdis), parsing and tree
-building (spark_parser), and grammar and semantic actions for
-decompiling (uncompyle6).
+projects:
+
+* bytecode loading and disassembly ([xdis](https://pypi.python.org/pypi/xdis)),
+* parsing and tree building ([spark_parser](https://pypi.python.org/pypi/spark_parser)),
+* this project - grammar and semantic actions for decompiling
+  ([uncompyle6](https://pypi.python.org/pypi/spark_parser)).
 
 
 Over the many years, code styles and Python features have
@@ -142,16 +148,19 @@ if the grammar is LR or left recursive.
 
 Another approach that doesn't use grammars is to do something like
 simulate execution symbolically and build expression trees off of
-stack results. The two important projects that work this way are
-[unpyc3](https://code.google.com/p/unpyc3/) and most especially
-[pycdc](https://github.com/zrax/pycdc) The latter project is largely
-by Michael Hansen and Darryl Pogue. If they supported getting
-source-code fragments and I could call it from Python, I'd probably
-ditch this and use that. From what I've seen, the code runs blindingly
-fast and spans all versions of Python.
+stack results. Control flow in that apprproach still needs to be
+handled somewhat ad hoc.  The two important projects that work this
+way are [unpyc3](https://code.google.com/p/unpyc3/) and most
+especially [pycdc](https://github.com/zrax/pycdc) The latter project
+is largely by Michael Hansen and Darryl Pogue. If they supported
+getting source-code fragments, did a better job in supporting Python
+more fully, and had a way I could call it from Python, I'd probably
+would have ditched this and used that. The code runs blindingly fast
+and spans all versions of Python, although more recently Python 3
+support has been lagging.
 
 Tests for the project have been, or are being, culled from all of the
 projects mentioned.
 
-NB. If you find mistakes, want corrections, or want your name added (or removed),
-please contact me.
+NB. If you find mistakes, want corrections, or want your name added
+(or removed), please contact me.

HOW-TO-REPORT-A-BUG.md

@@ -0,0 +1,68 @@
+# How to report a Bug
+
+## The difficulty of the problem
+
+There is no Python decompiler yet, that I know about that will
+decompyle everything. This one probably does the
+best job of *any* Python decompiler. But it is a constant work in progress: Python keeps changing, and so does its code generation.
+
+I have found bugs in *every* Python decompiler I have tried. Even
+those where authors/maintainers claim that they have used it on
+the entire Python standard library. And I don't mean that
+the program doesn't come out with the same Python source instructions,
+but that the program is *semantically* not equivalent.
+
+So it is likely you'll find a mistranslation in decompiling.
+
+## What to send (minimum requirements)
+
+The basic requirement is pretty simple:
+
+* Python bytecode
+* Source text
+
+## What to send (additional helpful information)
+
+Some kind folks also give the invocation they used and the output
+which usually includes an error message produced. This is helpful. I
+can figure out what OS you are running this on and what version of
+*uncomplye6* was used. Therefore, if you don't provide the input
+command and the output from that, please give:
+
+* _uncompyle6_ version used
+* OS that you used this on
+* Python interpreter version used
+
+
+### But I don't *have* the source code!
+
+Sure, I get it. No problem. There is Python assembly code on parse
+errors, so simply by hand decompile that. To get a full disassembly,
+use pydisasm from the [xdis](https://pypi.python.org/pypi/xdis)
+package. Opcodes are described in the documentation for
+the [dis](https://docs.python.org/3.6/library/dis.html) module.
+
+### But I don't *have* the source code and am incapable of figuring how how to do a hand disassembly!
+
+Well, you could learn. No one is born into this world knowing how to
+disassemble Python bytecode. And as Richard Feynman once said, "What
+one fool can learn, so can another."
+
+## Narrowing the problem
+
+I don't need the entire source code base for which one file or module
+can't be decompiled. I just need that one file or module only. If
+there are several files, file a bug report for each file.
+
+Python modules can get quite large, and usually decompilation problems
+occur in a single function or maybe the main-line code but not any of
+the functions or classes. So please chop down the source code by
+removing those parts that do to decompile properly.
+
+By doing this, you'll probably have a better sense of what exactly is
+the problem. Perhaps you can find the boundary of what decompiles, and
+what doesn't. That is useful. Or maybe the same file will decompile
+properly on a neighboring version of Python. That is helpful too.
+
+In sum, the more you can isolate or narrow the problem, the more
+likley the problem will be fixed and fixed sooner.

MANIFEST.in

@@ -1,6 +1,7 @@
 include README.rst
 include ChangeLog
 include HISTORY.md
+include HOW-TO-REPORT-A-BUG.md
 include LICENSE
 include Makefile
 include requirements.txt

NEWS

@@ -1,3 +1,52 @@
+uncompyle6 2.11.1 2016-06-22
+
+- Python 3.x annotatoin and function signature fixes
+- Bump xdis version
+- Small pysource bug fixes
+
+uncompyle6 2.11.0 2016-06-18 Fleetwood
+- Major improvements in fragment tracking
+  * Add nonterminal node in extractInfo
+  * tag more offsets in expressions
+  * tag array subscripts
+  * set YIELD value offset in a <yield> expr
+  * fix a long-standing bug in not adjusting final AST when melding other deparse ASTs
+- Fixes yet again for make_function node handling; document what's up here
+- Fix bug in snowflake Python 3.5 *args kwargs
+
+uncompyle6 2.10.1 2016-06-3 Marylin Frankel
+
+- fix some fragments parsing bugs
+  - was returning the wrong type sometimes in deparse_code_around_offset()
+  - capture function name in offsets
+  - track changes to ifelstrmtr node from pysource into fragments
+
+uncompyle6 2.10.0 2016-05-30 Elaine Gordon
+
+- Add fuzzy offset deparse lookup
+- 3.6 bugfixes
+   - fix EXTENDED_ARGS handling (and in 2.6 and others)
+   - semantic routine make_function fragments.py
+   - MAKE_FUNCTION handling
+   - CALL_FUNCTION_EX handling
+   - async property on defs
+   - support for CALL_FUNCTION_KW (moagstar)
+- 3.5+ UNMAP_PACK and BUILD_UNMAP_PACK handling
+- 3.5 FUNCTION_VAR bug
+- 3.x pass statement insdie while True
+- Improve 3.2 decompilation
+- Fixed -o argument processing (Gregrory)
+- Reduce scope of LOAD_ASSERT as expr to 3.4+
+- "await" statement fixes
+- 2.3, 2.4 "if 1 .." fixes
+- 3.x annotation fixes
+
+uncompyle6 2.9.11 2016-04-06
+
+- Better support for Python 3.5+ BUILD_MAP_UNPACK
+- Start 3.6 CALL_FUNCTION_EX support
+- Many decompilation bug fixes. (Many more remain). See ChangeLog
+
 uncompyle6 2.9.10 2016-02-25
 
 - Python grammar rule fixes

README.rst

@@ -11,8 +11,8 @@ Introduction
 ------------
 
 *uncompyle6* translates Python bytecode back into equivalent Python
-source code. It accepts bytecodes from Python version 2.1 to 3.6 or
-so, including PyPy bytecode and Dropbox's Python 2.5 bytecode.
+source code. It accepts bytecodes from Python version 1.5, and 2.1 to
+3.6 or so, including PyPy bytecode and Dropbox's Python 2.5 bytecode.
 
 Why this?
 ---------
@@ -21,7 +21,8 @@ There were a number of decompyle, uncompile, uncompyle2, uncompyle3
 forks around. All of them came basically from the same code base, and
 almost all of them no were no longer actively maintained. Only one
 handled Python 3, and even there, only 3.2 or 3.3 depending on which
-code is used. This code pulls these together and moves forward. It
+code is used. This code pulls these together and moves forward. This
+project has the most complete support for Python 3.3 and above. It
 also addresses a number of open issues in the previous forks.
 
 What makes this different from other CPython bytecode decompilers?: its
@@ -46,7 +47,7 @@ Requirements
 This project requires Python 2.6 or later, PyPy 3-2.4, or PyPy-5.0.1.
 Python versions 2.4-2.7 are supported in the python-2.4 branch.
 The bytecode files it can read has been tested on Python bytecodes from
-versions 1.5, 2.1-2.7, and 3.2-3.6 and the above-mentioned PyPy versions.
+versions 1.5, 2.1-2.7, and 3.0-3.6 and the above-mentioned PyPy versions.
 
 Installation
 ------------
@@ -55,7 +56,7 @@ This uses setup.py, so it follows the standard Python routine:
 
 ::
 
-    pip install -r requirements.txt
+    pip install -e .
     pip install -r requirements-dev.txt
     python setup.py install # may need sudo
     # or if you have pyenv:
@@ -112,7 +113,7 @@ with handling control flow. All of the Python decompilers I have looked
 at have the same problem. In some cases we can detect an erroneous
 decompilation and report that.
 
-About 90% of the decompilation of Python standard library packages in
+Over 98% of the decompilation of Python standard library packages in
 Python 2.7.12 verifies correctly. Over 99% of Python 2.7 and 3.3-3.5
 "weakly" verify. Python 2.6 drops down to 96% weakly verifying.
 Other versions drop off in quality too.
@@ -140,11 +141,10 @@ and 2.0.
 
 In the Python 3 series, Python support is is strongest around 3.4 or
 3.3 and drops off as you move further away from those versions. Python
-3.5 largely works, but still has some bugs in it and is missing some
-opcodes.  Python 3.6 changes things drastically by using word codes
-rather than byte codes. That has been addressed, but then it also
-changes function call opcodes and its semantics and has more problems
-with control flow than 3.5 has.
+3.6 changes things drastically by using word codes rather than byte
+codes. That has been addressed, but then it also changes function call
+opcodes and its semantics and has more problems with control flow than
+3.5 has.
 
 Currently not all Python magic numbers are supported. Specifically in
 some versions of Python, notably Python 3.6, the magic number has
@@ -158,17 +158,20 @@ We also don't handle PJOrion_ obfuscated code. For that try: PJOrion
 Deobfuscator_ to unscramble the bytecode to get valid bytecode before
 trying this tool.
 
+Handling pathologically long lists of expressions or statements is
+slow.
+
 
 There is lots to do, so please dig in and help.
 
 See Also
 --------
 
-* https://github.com/zrax/pycdc : supports all versions of Python and is written in C++
-* https://code.google.com/archive/p/unpyc3/ : supports Python 3.2 only. The above projects use a different decompiling technique what is used here.
+* https://github.com/zrax/pycdc : supports all versions of Python and is written in C++. Support for later Python 3 versions is a bit lacking though.
+* https://code.google.com/archive/p/unpyc3/ : supports Python 3.2 only. The above projects use a different decompiling technique than what is used here.
 * https://github.com/figment/unpyc3/ : fork of above, but supports Python 3.3 only. Include some fixes like supporting function annotations
 * The HISTORY_ file.
-
+* `How to report a bug <https://github.com/rocky/python-uncompyle6/blob/master/HISTORY.md>`_
 .. |downloads| image:: https://img.shields.io/pypi/dd/uncompyle6.svg
 .. _trepan: https://pypi.python.org/pypi/trepan
 .. _HISTORY: https://github.com/rocky/python-uncompyle6/blob/master/HISTORY.md

__pkginfo__.py

@@ -33,14 +33,14 @@ classifiers =  ['Development Status :: 5 - Production/Stable',
 # The rest in alphabetic order
 author             = "Rocky Bernstein, Hartmut Goebel, John Aycock, and others"
 author_email       = "rb@dustyfeet.com"
-entry_points={
+entry_points       = {
     'console_scripts': [
         'uncompyle6=uncompyle6.bin.uncompile:main_bin',
         'pydisassemble=uncompyle6.bin.pydisassemble:main',
     ]}
 ftp_url            = None
-install_requires   = ['spark-parser >= 1.6.0, < 1.7.0',
-                      'xdis >= 3.2.4, < 3.3.0']
+install_requires   = ['spark-parser >= 1.6.1, < 1.7.0',
+                      'xdis >= 3.4.0, < 3.5.0', 'six']
 license            = 'MIT'
 mailing_list       = 'python-debugger@googlegroups.com'
 modname            = 'uncompyle6'

appveyor.yml

@@ -0,0 +1,78 @@
+environment:
+  global:
+    # SDK v7.0 MSVC Express 2008's SetEnv.cmd script will fail if the
+    # /E:ON and /V:ON options are not enabled in the batch script intepreter
+    # See: http://stackoverflow.com/a/13751649/163740
+    CMD_IN_ENV: "cmd /E:ON /V:ON /C .\\appveyor\\run_with_env.cmd"
+
+  matrix:
+
+    # Pre-installed Python versions, which Appveyor may upgrade to
+    # a later point release.
+    # See: http://www.appveyor.com/docs/installed-software#python
+
+    # - PYTHON: "C:\\Python27"
+    #   PYTHON_VERSION: "2.7.x"
+    #   PYTHON_ARCH: "32"
+
+    - PYTHON: "C:\\Python27-x64"
+      PYTHON_VERSION: "2.7.x"
+      PYTHON_ARCH: "64"
+
+    # - PYTHON: "C:\\Python26"
+    #   PYTHON_VERSION: "2.6.x"
+    #   PYTHON_ARCH: "32"
+
+    # - PYTHON: "C:\\Python26-x64"
+    #   PYTHON_VERSION: "2.6.x"
+    #   PYTHON_ARCH: "64"
+
+install:
+  # We need wheel installed to build wheels
+  - "%PYTHON%\\python.exe -m pip install wheel"
+
+  # Install Python (from the official .msi of http://python.org) and pip when
+  # not already installed.
+  - ps: if (-not(Test-Path($env:PYTHON))) { & appveyor\install.ps1 }
+
+  # Prepend newly installed Python to the PATH of this build (this cannot be
+  # done from inside the powershell script as it would require to restart
+  # the parent CMD process).
+  - "SET PATH=%PYTHON%;%PYTHON%\\Scripts;%PATH%"
+  - "SET HOME=."
+
+  # Check that we have the expected version and architecture for Python
+  - "python --version"
+  - "python -c \"import struct; print(struct.calcsize('P') * 8)\""
+
+  # Upgrade to the latest version of pip to avoid it displaying warnings
+  # about it being out of date.
+  - "pip install --disable-pip-version-check --user --upgrade pip"
+
+  # Install the build dependencies of the project. If some dependencies contain
+  # compiled extensions and are not provided as pre-built wheel packages,
+  # pip will build them from source using the MSVC compiler matching the
+  # target Python version and architecture
+  - "%CMD_IN_ENV% pip install -r requirements.txt"
+
+build_script:
+  # Build the compiled extension
+  - "%CMD_IN_ENV% python setup.py build"
+
+test_script:
+  # Run the project tests
+  - "%CMD_IN_ENV% python test/test_pyenvlib.py --native --weak-verify"
+
+after_test:
+  # If tests are successful, create binary packages for the project.
+  - "%CMD_IN_ENV% python setup.py bdist_wininst"
+  - "%CMD_IN_ENV% python setup.py bdist_msi"
+  - ps: "ls dist"
+
+artifacts:
+  # Archive the generated packages in the ci.appveyor.com build report.
+  - path: dist\*
+
+#on_success:
+#  - TODO: upload the content of dist/*.whl to a public wheelhouse
+#

appveyor/install.ps1

@@ -0,0 +1,229 @@
+# Sample script to install Python and pip under Windows
+# Authors: Olivier Grisel, Jonathan Helmus, Kyle Kastner, and Alex Willmer
+# License: CC0 1.0 Universal: http://creativecommons.org/publicdomain/zero/1.0/
+
+$MINICONDA_URL = "http://repo.continuum.io/miniconda/"
+$BASE_URL = "https://www.python.org/ftp/python/"
+$GET_PIP_URL = "https://bootstrap.pypa.io/get-pip.py"
+$GET_PIP_PATH = "C:\get-pip.py"
+
+$PYTHON_PRERELEASE_REGEX = @"
+(?x)
+(?<major>\d+)
+\.
+(?<minor>\d+)
+\.
+(?<micro>\d+)
+(?<prerelease>[a-z]{1,2}\d+)
+"@
+
+
+function Download ($filename, $url) {
+    $webclient = New-Object System.Net.WebClient
+
+    $basedir = $pwd.Path + "\"
+    $filepath = $basedir + $filename
+    if (Test-Path $filename) {
+        Write-Host "Reusing" $filepath
+        return $filepath
+    }
+
+    # Download and retry up to 3 times in case of network transient errors.
+    Write-Host "Downloading" $filename "from" $url
+    $retry_attempts = 2
+    for ($i = 0; $i -lt $retry_attempts; $i++) {
+        try {
+            $webclient.DownloadFile($url, $filepath)
+            break
+        }
+        Catch [Exception]{
+            Start-Sleep 1
+        }
+    }
+    if (Test-Path $filepath) {
+        Write-Host "File saved at" $filepath
+    } else {
+        # Retry once to get the error message if any at the last try
+        $webclient.DownloadFile($url, $filepath)
+    }
+    return $filepath
+}
+
+
+function ParsePythonVersion ($python_version) {
+    if ($python_version -match $PYTHON_PRERELEASE_REGEX) {
+        return ([int]$matches.major, [int]$matches.minor, [int]$matches.micro,
+                $matches.prerelease)
+    }
+    $version_obj = [version]$python_version
+    return ($version_obj.major, $version_obj.minor, $version_obj.build, "")
+}
+
+
+function DownloadPython ($python_version, $platform_suffix) {
+    $major, $minor, $micro, $prerelease = ParsePythonVersion $python_version
+
+    if (($major -le 2 -and $micro -eq 0) `
+        -or ($major -eq 3 -and $minor -le 2 -and $micro -eq 0) `
+        ) {
+        $dir = "$major.$minor"
+        $python_version = "$major.$minor$prerelease"
+    } else {
+        $dir = "$major.$minor.$micro"
+    }
+
+    if ($prerelease) {
+        if (($major -le 2) `
+            -or ($major -eq 3 -and $minor -eq 1) `
+            -or ($major -eq 3 -and $minor -eq 2) `
+            -or ($major -eq 3 -and $minor -eq 3) `
+            ) {
+            $dir = "$dir/prev"
+        }
+    }
+
+    if (($major -le 2) -or ($major -le 3 -and $minor -le 4)) {
+        $ext = "msi"
+        if ($platform_suffix) {
+            $platform_suffix = ".$platform_suffix"
+        }
+    } else {
+        $ext = "exe"
+        if ($platform_suffix) {
+            $platform_suffix = "-$platform_suffix"
+        }
+    }
+
+    $filename = "python-$python_version$platform_suffix.$ext"
+    $url = "$BASE_URL$dir/$filename"
+    $filepath = Download $filename $url
+    return $filepath
+}
+
+
+function InstallPython ($python_version, $architecture, $python_home) {
+    Write-Host "Installing Python" $python_version "for" $architecture "bit architecture to" $python_home
+    if (Test-Path $python_home) {
+        Write-Host $python_home "already exists, skipping."
+        return $false
+    }
+    if ($architecture -eq "32") {
+        $platform_suffix = ""
+    } else {
+        $platform_suffix = "amd64"
+    }
+    $installer_path = DownloadPython $python_version $platform_suffix
+    $installer_ext = [System.IO.Path]::GetExtension($installer_path)
+    Write-Host "Installing $installer_path to $python_home"
+    $install_log = $python_home + ".log"
+    if ($installer_ext -eq '.msi') {
+        InstallPythonMSI $installer_path $python_home $install_log
+    } else {
+        InstallPythonEXE $installer_path $python_home $install_log
+    }
+    if (Test-Path $python_home) {
+        Write-Host "Python $python_version ($architecture) installation complete"
+    } else {
+        Write-Host "Failed to install Python in $python_home"
+        Get-Content -Path $install_log
+        Exit 1
+    }
+}
+
+
+function InstallPythonEXE ($exepath, $python_home, $install_log) {
+    $install_args = "/quiet InstallAllUsers=1 TargetDir=$python_home"
+    RunCommand $exepath $install_args
+}
+
+
+function InstallPythonMSI ($msipath, $python_home, $install_log) {
+    $install_args = "/qn /log $install_log /i $msipath TARGETDIR=$python_home"
+    $uninstall_args = "/qn /x $msipath"
+    RunCommand "msiexec.exe" $install_args
+    if (-not(Test-Path $python_home)) {
+        Write-Host "Python seems to be installed else-where, reinstalling."
+        RunCommand "msiexec.exe" $uninstall_args
+        RunCommand "msiexec.exe" $install_args
+    }
+}
+
+function RunCommand ($command, $command_args) {
+    Write-Host $command $command_args
+    Start-Process -FilePath $command -ArgumentList $command_args -Wait -Passthru
+}
+
+
+function InstallPip ($python_home) {
+    $pip_path = $python_home + "\Scripts\pip.exe"
+    $python_path = $python_home + "\python.exe"
+    if (-not(Test-Path $pip_path)) {
+        Write-Host "Installing pip..."
+        $webclient = New-Object System.Net.WebClient
+        $webclient.DownloadFile($GET_PIP_URL, $GET_PIP_PATH)
+        Write-Host "Executing:" $python_path $GET_PIP_PATH
+        & $python_path $GET_PIP_PATH
+    } else {
+        Write-Host "pip already installed."
+    }
+}
+
+
+function DownloadMiniconda ($python_version, $platform_suffix) {
+    if ($python_version -eq "3.4") {
+        $filename = "Miniconda3-3.5.5-Windows-" + $platform_suffix + ".exe"
+    } else {
+        $filename = "Miniconda-3.5.5-Windows-" + $platform_suffix + ".exe"
+    }
+    $url = $MINICONDA_URL + $filename
+    $filepath = Download $filename $url
+    return $filepath
+}
+
+
+function InstallMiniconda ($python_version, $architecture, $python_home) {
+    Write-Host "Installing Python" $python_version "for" $architecture "bit architecture to" $python_home
+    if (Test-Path $python_home) {
+        Write-Host $python_home "already exists, skipping."
+        return $false
+    }
+    if ($architecture -eq "32") {
+        $platform_suffix = "x86"
+    } else {
+        $platform_suffix = "x86_64"
+    }
+    $filepath = DownloadMiniconda $python_version $platform_suffix
+    Write-Host "Installing" $filepath "to" $python_home
+    $install_log = $python_home + ".log"
+    $args = "/S /D=$python_home"
+    Write-Host $filepath $args
+    Start-Process -FilePath $filepath -ArgumentList $args -Wait -Passthru
+    if (Test-Path $python_home) {
+        Write-Host "Python $python_version ($architecture) installation complete"
+    } else {
+        Write-Host "Failed to install Python in $python_home"
+        Get-Content -Path $install_log
+        Exit 1
+    }
+}
+
+
+function InstallMinicondaPip ($python_home) {
+    $pip_path = $python_home + "\Scripts\pip.exe"
+    $conda_path = $python_home + "\Scripts\conda.exe"
+    if (-not(Test-Path $pip_path)) {
+        Write-Host "Installing pip..."
+        $args = "install --yes pip"
+        Write-Host $conda_path $args
+        Start-Process -FilePath "$conda_path" -ArgumentList $args -Wait -Passthru
+    } else {
+        Write-Host "pip already installed."
+    }
+}
+
+function main () {
+    InstallPython $env:PYTHON_VERSION $env:PYTHON_ARCH $env:PYTHON
+    InstallPip $env:PYTHON
+}
+
+main

appveyor/run_with_env.cmd

@@ -0,0 +1,87 @@
+:: To build extensions for 64 bit Python 3, we need to configure environment
+:: variables to use the MSVC 2010 C++ compilers from GRMSDKX_EN_DVD.iso of:
+:: MS Windows SDK for Windows 7 and .NET Framework 4 (SDK v7.1)
+::
+:: To build extensions for 64 bit Python 2, we need to configure environment
+:: variables to use the MSVC 2008 C++ compilers from GRMSDKX_EN_DVD.iso of:
+:: MS Windows SDK for Windows 7 and .NET Framework 3.5 (SDK v7.0)
+::
+:: 32 bit builds, and 64-bit builds for 3.5 and beyond, do not require specific
+:: environment configurations.
+::
+:: Note: this script needs to be run with the /E:ON and /V:ON flags for the
+:: cmd interpreter, at least for (SDK v7.0)
+::
+:: More details at:
+:: https://github.com/cython/cython/wiki/64BitCythonExtensionsOnWindows
+:: http://stackoverflow.com/a/13751649/163740
+::
+:: Author: Olivier Grisel
+:: License: CC0 1.0 Universal: http://creativecommons.org/publicdomain/zero/1.0/
+::
+:: Notes about batch files for Python people:
+::
+:: Quotes in values are literally part of the values:
+::      SET FOO="bar"
+:: FOO is now five characters long: " b a r "
+:: If you don't want quotes, don't include them on the right-hand side.
+::
+:: The CALL lines at the end of this file look redundant, but if you move them
+:: outside of the IF clauses, they do not run properly in the SET_SDK_64==Y
+:: case, I don't know why.
+@ECHO OFF
+SET COMMAND_TO_RUN=%*
+SET WIN_SDK_ROOT=C:\Program Files\Microsoft SDKs\Windows
+SET WIN_WDK=c:\Program Files (x86)\Windows Kits\10\Include\wdf
+
+:: Extract the major and minor versions, and allow for the minor version to be
+:: more than 9.  This requires the version number to have two dots in it.
+SET MAJOR_PYTHON_VERSION=%PYTHON_VERSION:~0,1%
+IF "%PYTHON_VERSION:~3,1%" == "." (
+    SET MINOR_PYTHON_VERSION=%PYTHON_VERSION:~2,1%
+) ELSE (
+    SET MINOR_PYTHON_VERSION=%PYTHON_VERSION:~2,2%
+)
+
+:: Based on the Python version, determine what SDK version to use, and whether
+:: to set the SDK for 64-bit.
+IF %MAJOR_PYTHON_VERSION% == 2 (
+    SET WINDOWS_SDK_VERSION="v7.0"
+    SET SET_SDK_64=Y
+) ELSE (
+    IF %MAJOR_PYTHON_VERSION% == 3 (
+        SET WINDOWS_SDK_VERSION="v7.1"
+        IF %MINOR_PYTHON_VERSION% LEQ 4 (
+            SET SET_SDK_64=Y
+        ) ELSE (
+            SET SET_SDK_64=N
+            IF EXIST "%WIN_WDK%" (
+                :: See: https://connect.microsoft.com/VisualStudio/feedback/details/1610302/
+                REN "%WIN_WDK%" 0wdf
+            )
+        )
+    ) ELSE (
+        ECHO Unsupported Python version: "%MAJOR_PYTHON_VERSION%"
+        EXIT 1
+    )
+)
+
+IF %PYTHON_ARCH% == 64 (
+    IF %SET_SDK_64% == Y (
+        ECHO Configuring Windows SDK %WINDOWS_SDK_VERSION% for Python %MAJOR_PYTHON_VERSION% on a 64 bit architecture
+        SET DISTUTILS_USE_SDK=1
+        SET MSSdk=1
+        "%WIN_SDK_ROOT%\%WINDOWS_SDK_VERSION%\Setup\WindowsSdkVer.exe" -q -version:%WINDOWS_SDK_VERSION%
+        "%WIN_SDK_ROOT%\%WINDOWS_SDK_VERSION%\Bin\SetEnv.cmd" /x64 /release
+        ECHO Executing: %COMMAND_TO_RUN%
+        call %COMMAND_TO_RUN% || EXIT 1
+    ) ELSE (
+        ECHO Using default MSVC build environment for 64 bit architecture
+        ECHO Executing: %COMMAND_TO_RUN%
+        call %COMMAND_TO_RUN% || EXIT 1
+    )
+) ELSE (
+    ECHO Using default MSVC build environment for 32 bit architecture
+    ECHO Executing: %COMMAND_TO_RUN%
+    call %COMMAND_TO_RUN% || EXIT 1
+)

circle.yml

@@ -6,7 +6,7 @@ machine:
 
 dependencies:
     override:
-        - pip install -r requirements.txt
+        - pip install -e .
         - pip install -r requirements-dev.txt
 test:
     override:

pytest/test_CALL_FUNCTION_KW.sh

@@ -0,0 +1,6 @@
+source ../.venv.3.6/bin/activate
+py.test -k test_CALL_FUNCTION_KW
+source ../.venv.3.5/bin/activate
+py.test -k test_CALL_FUNCTION_KW
+source ../.venv.2.7/bin/activate
+py.test -k test_CALL_FUNCTION_KW

pytest/test_function_call.py

@@ -0,0 +1,175 @@
+# std
+import string
+# 3rd party
+from hypothesis import given, assume, example, settings, strategies as st
+import pytest
+# uncompyle
+from validate import validate_uncompyle
+from test_fstring import expressions
+
+
+alpha = st.sampled_from(string.ascii_lowercase)
+numbers = st.sampled_from(string.digits)
+alphanum = st.sampled_from(string.ascii_lowercase + string.digits)
+
+
+@st.composite
+def function_calls(draw,
+                   min_keyword_args=0, max_keyword_args=5,
+                   min_positional_args=0, max_positional_args=5,
+                   min_star_args=0, max_star_args=1,
+                   min_double_star_args=0, max_double_star_args=1):
+    """
+    Strategy factory for generating function calls.
+
+    :param draw: Callable which draws examples from other strategies.
+
+    :return: The function call text.
+    """
+    st_positional_args = st.lists(
+        alpha,
+        min_size=min_positional_args,
+        max_size=max_positional_args
+    )
+    st_keyword_args = st.lists(
+        alpha,
+        min_size=min_keyword_args,
+        max_size=max_keyword_args
+    )
+    st_star_args = st.lists(
+        alpha,
+        min_size=min_star_args,
+        max_size=max_star_args
+    )
+    st_double_star_args = st.lists(
+        alpha,
+        min_size=min_double_star_args,
+        max_size=max_double_star_args
+    )
+
+    positional_args = draw(st_positional_args)
+    keyword_args = draw(st_keyword_args)
+    st_values = st.lists(
+        expressions(),
+        min_size=len(keyword_args),
+        max_size=len(keyword_args)
+    )
+    keyword_args = [
+        x + '=' + e
+        for x, e in
+        zip(keyword_args, draw(st_values))
+    ]
+    star_args = ['*' + x for x in draw(st_star_args)]
+    double_star_args = ['**' + x for x in draw(st_double_star_args)]
+
+    arguments = positional_args + keyword_args + star_args + double_star_args
+    draw(st.randoms()).shuffle(arguments)
+    arguments = ','.join(arguments)
+
+    function_call = 'fn({arguments})'.format(arguments=arguments)
+    try:
+        # TODO: Figure out the exact rules for ordering of positional, keyword,
+        # star args, double star args and in which versions the various
+        # types of arguments are supported so we don't need to check that the
+        # expression compiles like this.
+        compile(function_call, '<string>', 'single')
+    except:
+        assume(False)
+    return function_call
+
+
+def test_function_no_args():
+    validate_uncompyle("fn()")
+
+
+def isolated_function_calls(which):
+    """
+    Returns a strategy for generating function calls, but isolated to 
+    particular types of arguments, for example only positional arguments.
+
+    This can help reason about debugging errors in specific types of function
+    calls.
+
+    :param which: One of 'keyword', 'positional', 'star', 'double_star'
+
+    :return: Strategy for generating an function call isolated to specific
+             argument types.
+    """
+    kwargs = dict(
+        max_keyword_args=0,
+        max_positional_args=0,
+        max_star_args=0,
+        max_double_star_args=0,
+    )
+    kwargs['_'.join(('min', which, 'args'))] = 1
+    kwargs['_'.join(('max', which, 'args'))] = 5 if 'star' not in which else 1
+    return function_calls(**kwargs)
+
+
+with settings(max_examples=25):
+
+    @given(isolated_function_calls('positional'))
+    @example("fn(0)")
+    def test_function_positional_only(expr):
+        validate_uncompyle(expr)
+
+    @given(isolated_function_calls('keyword'))
+    @example("fn(a=0)")
+    def test_function_call_keyword_only(expr):
+        validate_uncompyle(expr)
+
+    @given(isolated_function_calls('star'))
+    @example("fn(*items)")
+    def test_function_call_star_only(expr):
+        validate_uncompyle(expr)
+
+    @given(isolated_function_calls('double_star'))
+    @example("fn(**{})")
+    def test_function_call_double_star_only(expr):
+        validate_uncompyle(expr)
+
+
+@pytest.mark.xfail()
+def test_BUILD_CONST_KEY_MAP_BUILD_MAP_UNPACK_WITH_CALL_BUILD_TUPLE_CALL_FUNCTION_EX():
+    validate_uncompyle("fn(w=0,m=0,**v)")
+
+
+@pytest.mark.xfail()
+def test_BUILD_MAP_BUILD_MAP_UNPACK_WITH_CALL_BUILD_TUPLE_CALL_FUNCTION_EX():
+    validate_uncompyle("fn(a=0,**g)")
+
+
+@pytest.mark.xfail()
+def test_CALL_FUNCTION_EX():
+    validate_uncompyle("fn(*g,**j)")
+
+
+@pytest.mark.xfail()
+def test_BUILD_MAP_CALL_FUNCTION_EX():
+    validate_uncompyle("fn(*z,u=0)")
+
+
+@pytest.mark.xfail()
+def test_BUILD_TUPLE_CALL_FUNCTION_EX():
+    validate_uncompyle("fn(**a)")
+
+
+@pytest.mark.xfail()
+def test_BUILD_MAP_BUILD_TUPLE_BUILD_TUPLE_UNPACK_WITH_CALL_CALL_FUNCTION_EX():
+    validate_uncompyle("fn(b,b,b=0,*a)")
+
+
+@pytest.mark.xfail()
+def test_BUILD_TUPLE_BUILD_TUPLE_UNPACK_WITH_CALL_CALL_FUNCTION_EX():
+    validate_uncompyle("fn(*c,v)")
+
+
+@pytest.mark.xfail()
+def test_BUILD_CONST_KEY_MAP_CALL_FUNCTION_EX():
+    validate_uncompyle("fn(i=0,y=0,*p)")
+
+
+@pytest.mark.skip(reason='skipping property based test until all individual tests are passing')
+@given(function_calls())
+def test_function_call(function_call):
+    validate_uncompyle(function_call)

pytest/test_grammar.py

@@ -40,7 +40,9 @@ def test_grammar():
     ignore_set = set(
             """
             JUMP_BACK CONTINUE RETURN_END_IF
-            COME_FROM COME_FROM_EXCEPT COME_FROM_LOOP COME_FROM_WITH
+            COME_FROM COME_FROM_EXCEPT
+            COME_FROM_EXCEPT_CLAUSE
+            COME_FROM_LOOP COME_FROM_WITH
             COME_FROM_FINALLY ELSE
             LOAD_GENEXPR LOAD_ASSERT LOAD_SETCOMP LOAD_DICTCOMP
             LAMBDA_MARKER RETURN_LAST

pytest/validate.py

@@ -2,18 +2,23 @@
 from __future__ import print_function
 # std
 import os
-import dis
 import difflib
 import subprocess
 import tempfile
+import functools
 # compatability
 import six
 # uncompyle6 / xdis
-from uncompyle6 import PYTHON_VERSION, deparse_code
+from uncompyle6 import PYTHON_VERSION, IS_PYPY, deparse_code
+# TODO : I think we can get xdis to support the dis api (python 3 version) by doing something like this there
+from xdis.bytecode import Bytecode
+from xdis.main import get_opcode
+opc = get_opcode(PYTHON_VERSION, IS_PYPY)
+Bytecode = functools.partial(Bytecode, opc=opc)
 
 
 def _dis_to_text(co):
-    return dis.Bytecode(co).dis()
+    return Bytecode(co).dis()
 
 
 def print_diff(original, uncompyled):
@@ -99,9 +104,8 @@ def are_code_objects_equal(co1, co2):
 
     :return: True if the two code objects are approximately equal, otherwise False.
     """
-    # TODO : Use xdis for python2 compatability
-    instructions1 = dis.Bytecode(co1)
-    instructions2 = dis.Bytecode(co2)
+    instructions1 = Bytecode(co1)
+    instructions2 = Bytecode(co2)
     for opcode1, opcode2 in zip(instructions1, instructions2):
         if not are_instructions_equal(opcode1, opcode2):
             return False

requirements-dev.txt

@@ -1,4 +1,3 @@
-pytest
+pytest>=3.0.0
 flake8
 hypothesis
-six

test/Makefile

@@ -3,7 +3,7 @@ PHONY=check clean dist distclean test test-unit test-functional rmChangeLog clea
 GIT2CL ?= git2cl
 PYTHON ?= python
 
-PYTHON_VERSION = $(shell $(PYTHON) -V | cut -d ' ' -f 2 | cut -d'.' -f1,2)
+PYTHON_VERSION = $(shell $(PYTHON) -V 2>&1 | cut -d ' ' -f 2 | cut -d'.' -f1,2)
 NATIVE_CHECK = check-$(PYTHON_VERSION)
 
 # Set COMPILE='--compile' to force compilation before check
@@ -16,11 +16,10 @@ check-short:
 
 # Run all tests
 check:
-	@$(PYTHON) -V && PYTHON_VERSION=`$(PYTHON) -V 2>&1 | cut -d ' ' -f 2 | cut -d'.' -f1,2`; \
-	$(MAKE) check-$$PYTHON_VERSION
+	$(MAKE) check-$(PYTHON_VERSION)
 
 #: Run working tests from Python 2.6 or 2.7
-check-2.6 check-2.7: check-bytecode-2 check-bytecode-3 check-bytecode-1  check-2.7-ok
+check-2.6 check-2.7: check-bytecode-2 check-bytecode-3 check-bytecode-1 check-native-short
 
 #: Run working tests from Python 3.0
 check-3.0: check-bytecode
@@ -36,7 +35,7 @@ check-3.2: check-bytecode
 
 #: Run working tests from Python 3.3
 check-3.3: check-bytecode
-	$(PYTHON) test_pythonlib.py --bytecode-3.3 --verify $(COMPILE)
+	$(PYTHON) test_pythonlib.py --bytecode-3.3 --weak-verify $(COMPILE)
 
 #: Run working tests from Python 3.4
 check-3.4: check-bytecode check-3.4-ok check-2.7-ok

test/simple_source/bug22/03_if1.py

@@ -0,0 +1,7 @@
+# From https://github.com/ToontownInfinite /src/otp/avatar/LocalAvatar.py#L364
+if 1:
+    def jumpLandAnimFix(self, jumpTime):
+        return 5
+
+    def jumpLand(self):
+        return 6

test/simple_source/bug26/03_double_equals.py

@@ -0,0 +1,6 @@
+# From Python 2.7 parse_starttag HTMLParser.pyc
+attrvalue = [1,2]
+while attrvalue:
+    if attrvalue[:1] == 5 or \
+       attrvalue[:1] == 2 == attrvalue[-1:]:
+        attrvalue = 10

test/simple_source/bug27+/03_if_1_else.py

@@ -0,0 +1 @@
+1 if 1 else __file__

test/simple_source/bug31/02_ifelse_comprehension.py

@@ -0,0 +1,12 @@
+# Python 2.7 sqlalchemy-1.013/sql/crud.py
+def _extend_values_for_multiparams(compiler, stmt, c):
+    c(
+        [
+            (
+                (compiler() if compiler()
+                    else compiler())
+                if c in stmt else compiler(),
+            )
+        ]
+        for i in enumerate(stmt)
+    )

test/simple_source/bug31/04_def_annotate.py

@@ -9,7 +9,7 @@ def open(file, mode = "r", buffering = None,
          newline = None, closefd = True) -> "IOBase":
     return text
 
-def foo(x: 'an argument that defaults to 5' = 5):
+def foo1(x: 'an argument that defaults to 5' = 5):
     print(x)
 
 def div(a: dict(type=float, help='the dividend'),
@@ -17,3 +17,14 @@ def div(a: dict(type=float, help='the dividend'),
     ) -> dict(type=float, help='the result of dividing a by b'):
     """Divide a by b"""
     return a / b
+
+class TestSignatureObject(unittest.TestCase):
+    def test_signature_on_wkwonly(self):
+        def test(*, a:float, b:str) -> int:
+            pass
+
+class SupportsInt(_Protocol):
+
+    @abstractmethod
+    def __int__(self) -> int:
+        pass

test/simple_source/bug32/01_named_and_kwargs.py

@@ -18,3 +18,12 @@ def __init__(self, defaults=None, dict_type=_default_dict,
              default_section=DEFAULTSECT,
              interpolation=_UNSET):
     pass
+
+# Bug found by hypothesis in creating function calls
+# thanks to moagstar
+def fn(a, b, d):
+    return (a, b, d)
+
+b = {'b': 1,
+     'd': 2}
+fn(a=0, **b)

test/simple_source/bug32/01_try_except_raise.py

@@ -0,0 +1,9 @@
+# From 3.2 _abcoll.py
+def pop(self):
+    it = iter(self)
+    try:
+        value = next(it)
+    except StopIteration:
+        raise KeyError
+    self.discard(value)
+    return value

test/simple_source/bug32/03_if.py

@@ -0,0 +1,11 @@
+# From 3.2 shlex.py
+def _samefile(os, src, dst):
+    if hasattr(os.path, 'samefile'):
+        try:
+            return os.path.samefile(src, dst)
+        except OSError:
+            return False
+
+    # All other platforms: check for same pathname.
+    return (os.path.normcase(os.path.abspath(src)) ==
+            os.path.normcase(os.path.abspath(dst)))

test/simple_source/bug33/01_delete_deref.py

@@ -0,0 +1,4 @@
+def a():
+    del y
+    def b():
+        return y

test/simple_source/bug33/01_if_try_except.py

@@ -0,0 +1,16 @@
+# From 3.3.5 _osx_support.py
+def _get_system_version():
+    if __file__ is None:
+        try:
+            m = 5
+        except IOError:
+            pass
+        else:
+            try:
+                m = 10
+            finally:
+                m = 15
+            if m is not None:
+                m = 20
+
+    return m

test/simple_source/bug33/01_try_except.py

@@ -0,0 +1,16 @@
+# From 3.3.5 _osx_support.py
+def _get_system_version():
+    if __file__ is None:
+        try:
+            m = 5
+        except IOError:
+            pass
+        else:
+            try:
+                m = 10
+            finally:
+                m = 15
+            if m is not None:
+                m = 20
+
+    return m

test/simple_source/bug33/02_pos_args.py

@@ -1,3 +1,6 @@
 # From Python 3.3.6 hmac.py
 # Problem was getting wrong placement of positional args
 digest_cons = lambda d=b'': 5
+
+# Handle single kwarg
+lambda *, d=0: None

test/simple_source/bug35/01_map_unpack.py

@@ -0,0 +1,9 @@
+# Python 3.5+ PEP 448 - Additional Unpacking Generalizations for dictionaries
+{**{}}
+{**{'a': 1, 'b': 2}}
+## {**{'x': 1}, **{'y': 2}}
+# {'c': 1, {'d': 2}, **{'e': 3}}
+[*[]]
+{**{0:0 for a in b}}
+## {**{}, **{}}
+## {**{}, **{}, **{}}

test/simple_source/bug35/02_async_for.py

@@ -0,0 +1,3 @@
+async def a(b, c):
+    async for b in c:
+        pass

test/simple_source/bug35/03_async_await.py

@@ -24,3 +24,9 @@ async def awith_test():
 async def awith_as_test():
     async with 1 as i:
         print(i)
+
+async def f(z):
+    await z
+
+async def g(z):
+    return await z

test/simple_source/bug35/03_while-if-break.py

@@ -5,3 +5,10 @@ def display_date(loop):
         if loop.time():
             break
         x = 5
+
+    # Another loop to test 3.5 ifelsestmtl grammar rule
+    while loop:
+        if x:
+            True
+        else:
+            True

test/simple_source/bug35/04_CALL_FUNCTION_VAR_KW.py

@@ -0,0 +1,8 @@
+# sql/schema.py
+# Note that kwargs comes before "positional" args
+def tometadata(self, metadata, schema, Table, args, name=None):
+    table = Table(
+        name, metadata, schema=schema,
+        *args, **self.kwargs
+    )
+    return table

test/simple_source/bug36/01_call_function.py

@@ -0,0 +1,5 @@
+# Python 3.6's changes for calling functions.
+# See https://github.com/rocky/python-uncompyle6/issues/58
+# CALL_FUNCTION_EX takes 2 to 3 arguments on the stack: the function, the tuple of positional arguments,
+# and optionally the dict of keyword arguments if bit 0 of oparg is 1.
+a(*[])

test/simple_source/bug36/01_extended_arg.py

@@ -0,0 +1,2 @@
+if __file__:
+    0+0+0+0+0+0+0+0+0+0+0+0+0+0+0+0+0+0+0+0+0+0+0+0+0+0+0+0+0+0+0+0+0+0+0+0+0+0+0+0+0+0+0+0+0+0+0+0+0+0+0+0+0+0+0+0+0+0+0+0+0+0+0

test/simple_source/def/10_kw+pos_args-bug.py

@@ -8,3 +8,20 @@ def __init__(self, defaults=None, dict_type=_default_dict,
              default_section=DEFAULTSECT,
              interpolation=_UNSET):
     pass
+
+# From 3.5 sqlalchemy/orm/__init__.py
+# Python 3.5 changes the stack position of where * args are (furthest down the stack)
+# Python 3.6+ replaces CALL_FUNCTION_VAR_KW with CALL_FUNCTION_EX
+def deferred(*columns, **kw):
+    return ColumnProperty(deferred=True, *columns, **kw)
+
+
+# From sqlalchemy/sql/selectable.py
+class GenerativeSelect():
+    def __init__(self,
+                 ClauseList,
+                 util,
+                 order_by=None):
+        self._order_by_clause = ClauseList(
+            *util.to_list(order_by),
+            _literal_as_text=5)

test/simple_source/looping/12_if_while_true_pass.py

@@ -0,0 +1,14 @@
+# Python 3.3 pyclbr.py
+# Note that Python 3 adds a lot of unecessary "continues"
+# and puts that in for "pass"
+def _readmodule(g, token, path):
+    for tokentype in g:
+        if g:
+            while True:
+                if token:
+                    token = 1
+                elif token:
+                    pass
+                elif tokentype:
+                    token = 7
+            token = 10

test/simple_source/stmts/02_while1else.py

@@ -0,0 +1,8 @@
+# From Python-3.5.2/Lib/multiprocessing/connection.py
+
+def PipeClient(address):
+    while 1:
+        z = 2
+    else:
+        raise
+    return

test/test_pyenvlib.py

@@ -22,7 +22,7 @@ Step 2: Run the test:
 from __future__ import print_function
 
 from uncompyle6 import main, PYTHON3
-import os, time, shutil
+import os, time, shutil, sys
 from fnmatch import fnmatch
 
 #----- configure this for your needs
@@ -33,7 +33,7 @@ TEST_VERSIONS=('2.3.7', '2.4.6', '2.5.6', '2.6.9',
                '2.7.10', '2.7.11', '2.7.12', '2.7.13',
                '3.0.1', '3.1.5', '3.2.6',
                '3.3.5', '3.3.6',
-               '3.4.2', '3.5.1', '3.6.0')
+               '3.4.2', '3.5.1', '3.6.0', 'native')
 
 target_base = '/tmp/py-dis/'
 lib_prefix = os.path.join(os.environ['HOME'], '.pyenv/versions')
@@ -54,6 +54,11 @@ for vers in TEST_VERSIONS:
         short_vers = vers[0:-2]
         test_options[vers] = (os.path.join(lib_prefix, vers, 'lib_pypy'),
                               PYC, 'python-lib'+short_vers)
+    if vers == 'native':
+        short_vers = os.path.basename(sys.path[-1])
+        test_options[vers] = (sys.path[-1],
+                              PYC, short_vers)
+
     else:
         short_vers = vers[:3]
         test_options[vers] = (os.path.join(lib_prefix, vers, 'lib', 'python'+short_vers),

uncompyle6/main.py

@@ -106,6 +106,7 @@ def main(in_base, out_base, files, codes, outfile=None,
         return open(outfile, 'w')
 
     tot_files = okay_files = failed_files = verify_failed_files = 0
+    current_outfile = outfile
 
     for filename in files:
         infile = os.path.join(in_base, filename)
@@ -134,11 +135,11 @@ def main(in_base, out_base, files, codes, outfile=None,
                 os.dup2(tee.stdin.fileno(), sys.stderr.fileno())
         else:
             if filename.endswith('.pyc'):
-                outfile = os.path.join(out_base, filename[0:-1])
+                current_outfile = os.path.join(out_base, filename[0:-1])
             else:
-                outfile = os.path.join(out_base, filename) + '_dis'
-            outstream = _get_outstream(outfile)
-        # print(outfile, file=sys.stderr)
+                current_outfile = os.path.join(out_base, filename) + '_dis'
+            outstream = _get_outstream(current_outfile)
+        # print(current_outfile, file=sys.stderr)
 
         # Try to uncompile the input file
         try:
@@ -157,16 +158,16 @@ def main(in_base, out_base, files, codes, outfile=None,
             raise
         # except:
         #     failed_files += 1
-        #     if outfile:
+        #     if current_outfile:
         #         outstream.close()
-        #         os.rename(outfile, outfile + '_failed')
+        #         os.rename(current_outfile, current_outfile + '_failed')
         #     else:
         #         sys.stderr.write("\n# %s" % sys.exc_info()[1])
         #         sys.stderr.write("\n# Can't uncompile %s\n" % infile)
         else: # uncompile successful
-            if outfile:
+            if current_outfile:
                 if do_linemaps:
-                    mapping = line_number_mapping(infile, outfile)
+                    mapping = line_number_mapping(infile, current_outfile)
                     outstream.write("\n\n## Line number correspondences\n")
                     import pprint
                     s = pprint.pformat(mapping, indent=2, width=80)
@@ -177,8 +178,8 @@ def main(in_base, out_base, files, codes, outfile=None,
                 if do_verify:
                     weak_verify = do_verify == 'weak'
                     try:
-                        msg = verify.compare_code_with_srcfile(infile, outfile, weak_verify=weak_verify)
-                        if not outfile:
+                        msg = verify.compare_code_with_srcfile(infile, current_outfile, weak_verify=weak_verify)
+                        if not current_outfile:
                             if not msg:
                                 print('\n# okay decompiling %s' % infile)
                                 okay_files += 1
@@ -187,7 +188,7 @@ def main(in_base, out_base, files, codes, outfile=None,
                     except verify.VerifyCmpError as e:
                         print(e)
                         verify_failed_files += 1
-                        os.rename(outfile, outfile + '_unverified')
+                        os.rename(current_outfile, current_outfile + '_unverified')
                         sys.stderr.write("### Error Verifying %s\n" % filename)
                         sys.stderr.write(str(e) + "\n")
                         if not outfile:
@@ -201,15 +202,15 @@ def main(in_base, out_base, files, codes, outfile=None,
                 pass
             else:
                 okay_files += 1
-                if not outfile:
+                if not current_outfile:
                     mess = '\n# okay decompiling'
                     # mem_usage = __memUsage()
                     print(mess, infile)
-        if outfile:
+        if current_outfile:
             sys.stdout.write("%s\r" %
                              status_msg(do_verify, tot_files, okay_files, failed_files, verify_failed_files))
             sys.stdout.flush()
-    if outfile:
+    if current_outfile:
         sys.stdout.write("\n")
         sys.stdout.flush()
     return (tot_files, okay_files, failed_files, verify_failed_files)

uncompyle6/parser.py

@@ -31,13 +31,19 @@ class PythonParser(GenericASTBuilder):
     def __init__(self, AST, start, debug):
         super(PythonParser, self).__init__(AST, start, debug)
         self.collect = frozenset(
-            ['stmts', 'except_stmts', '_stmts',
-             'exprlist', 'kvlist', 'kwargs', 'come_froms',
+            ['stmts', 'except_stmts', '_stmts', 'load_attrs',
+             'exprlist', 'kvlist', 'kwargs', 'come_froms', '_come_from',
               # Python < 3
              'print_items',
              # PyPy:
              'kvlist_n'])
 
+    def ast_first_offset(self, ast):
+        if hasattr(ast, 'offset'):
+            return ast.offset
+        else:
+            return self.ast_first_offset(ast[0])
+
     def add_unique_rule(self, rule, opname, count, customize):
         """Add rule to grammar, but only if it hasn't been added previously
            opname and count are used in the customize() semantic the actions
@@ -395,8 +401,7 @@ class PythonParser(GenericASTBuilder):
         import_cont ::= LOAD_CONST LOAD_CONST import_as_cont
         import_as_cont ::= IMPORT_FROM designator
 
-        load_attrs ::= LOAD_ATTR
-        load_attrs ::= load_attrs LOAD_ATTR
+        load_attrs ::= LOAD_ATTR+
         """
 
     def p_list_comprehension(self, args):
@@ -503,8 +508,12 @@ class PythonParser(GenericASTBuilder):
         expr ::= conditional
         conditional ::= expr jmp_false expr JUMP_FORWARD expr COME_FROM
         conditional ::= expr jmp_false expr JUMP_ABSOLUTE expr
+
         expr ::= conditionalnot
-        conditionalnot ::= expr jmp_true expr _jump expr COME_FROM
+        conditionalnot  ::= expr jmp_true expr _jump expr COME_FROM
+
+        expr            ::= conditionalTrue
+        conditionalTrue ::= expr JUMP_FORWARD expr COME_FROM
 
         ret_expr ::= expr
         ret_expr ::= ret_and

uncompyle6/parsers/parse2.py

@@ -44,7 +44,8 @@ class Python2Parser(PythonParser):
         while1stmt     ::= SETUP_LOOP l_stmts     JUMP_BACK COME_FROM
         while1stmt     ::= SETUP_LOOP l_stmts     JUMP_BACK POP_BLOCK COME_FROM
 
-        while1elsestmt ::= SETUP_LOOP l_stmts JUMP_BACK else_suite COME_FROM
+        while1elsestmt ::= SETUP_LOOP l_stmts JUMP_BACK POP_BLOCK else_suite COME_FROM
+        while1elsestmt ::= SETUP_LOOP l_stmts JUMP_BACK           else_suite COME_FROM
 
         exec_stmt ::= expr exprlist DUP_TOP EXEC_STMT
         exec_stmt ::= expr exprlist EXEC_STMT

uncompyle6/parsers/parse23.py

@@ -1,4 +1,4 @@
-#  Copyright (c) 2016 Rocky Bernstein
+#  Copyright (c) 2016-2017 Rocky Bernstein
 #  Copyright (c) 2000-2002 by hartmut Goebel <hartmut@goebel.noris.de>
 #  Copyright (c) 1999 John Aycock
 
@@ -14,6 +14,17 @@ class Python23Parser(Python24Parser):
 
     def p_misc23(self, args):
         '''
+        # Python 2.4 only adds something like the below for if 1:
+        # However we will just treat it as a noop (which of course messes up
+        # simple verify of bytecode.
+        # See also below in reduce_is_invalid where we check that the JUMP_FORWARD
+        # target matches the COME_FROM target
+        stmt     ::= if1_stmt
+        if1_stmt ::= JUMP_FORWARD JUMP_IF_FALSE THEN POP_TOP COME_FROM
+                     stmts
+                     JUMP_FORWARD COME_FROM POP_TOP COME_FROM
+
+
         # Used to keep semantic positions the same across later versions
         # of Python
         _while1test ::= SETUP_LOOP JUMP_FORWARD JUMP_IF_FALSE POP_TOP COME_FROM
@@ -33,6 +44,23 @@ class Python23Parser(Python24Parser):
         lc_body ::= LOAD_FAST expr LIST_APPEND
         '''
 
+    def add_custom_rules(self, tokens, customize):
+        super(Python23Parser, self).add_custom_rules(tokens, customize)
+
+    def reduce_is_invalid(self, rule, ast, tokens, first, last):
+        invalid = super(Python24Parser,
+                        self).reduce_is_invalid(rule, ast,
+                                                tokens, first, last)
+        if invalid:
+            return invalid
+
+        # FiXME: this code never gets called...
+        lhs = rule[0]
+        if lhs == 'nop_stmt':
+            return not int(tokens[first].pattr) == tokens[last].offset
+
+        return False
+
 class Python23ParserSingle(Python23Parser, PythonParserSingle):
     pass
 

uncompyle6/parsers/parse24.py

@@ -14,6 +14,15 @@ class Python24Parser(Python25Parser):
 
     def p_misc24(self, args):
         '''
+        # Python 2.4 only adds something like the below for if 1:
+        # However we will just treat it as a noop (which of course messes up
+        # simple verify of bytecode.
+        # See also below in reduce_is_invalid where we check that the JUMP_FORWARD
+        # target matches the COME_FROM target
+        stmt     ::= nop_stmt
+        nop_stmt ::= JUMP_FORWARD POP_TOP COME_FROM
+
+
         # 2.5+ has two LOAD_CONSTs, one for the number '.'s in a relative import
         # keep positions similar to simplify semantic actions
 
@@ -37,6 +46,25 @@ class Python24Parser(Python25Parser):
         gen_comp_body ::= expr YIELD_VALUE
         '''
 
+    def add_custom_rules(self, tokens, customize):
+        super(Python24Parser, self).add_custom_rules(tokens, customize)
+        if self.version == 2.4:
+            self.check_reduce['nop_stmt'] = 'tokens'
+
+    def reduce_is_invalid(self, rule, ast, tokens, first, last):
+        invalid = super(Python24Parser,
+                        self).reduce_is_invalid(rule, ast,
+                                                tokens, first, last)
+        if invalid:
+            return invalid
+
+        # FiXME: this code never gets called...
+        lhs = rule[0]
+        if lhs == 'nop_stmt':
+            return not int(tokens[first].pattr) == tokens[last].offset
+
+        return False
+
 class Python24ParserSingle(Python24Parser, PythonParserSingle):
     pass
 

uncompyle6/parsers/parse26.py

@@ -157,7 +157,6 @@ class Python26Parser(Python2Parser):
         # Semantic actions want the else to be at position 3
         ifelsestmt     ::= testexpr      c_stmts_opt jf_cf_pop else_suite come_froms
         ifelsestmt     ::= testexpr_then c_stmts_opt jf_cf_pop else_suite come_froms
-        ifelsestmt     ::= testexpr      c_stmts_opt filler else_suitel come_froms POP_TOP
         ifelsestmt     ::= testexpr_then c_stmts_opt filler else_suitel come_froms POP_TOP
 
         # Semantic actions want else_suitel to be at index 3

uncompyle6/parsers/parse27.py

@@ -1,4 +1,4 @@
-#  Copyright (c) 2016 Rocky Bernstein
+#  Copyright (c) 2016-2017 Rocky Bernstein
 #  Copyright (c) 2005 by Dan Pascu <dan@windowmaker.org>
 #  Copyright (c) 2000-2002 by hartmut Goebel <hartmut@goebel.noris.de>
 
@@ -52,6 +52,8 @@ class Python27Parser(Python2Parser):
         come_froms ::= come_froms COME_FROM
         come_froms ::= COME_FROM
 
+        iflaststmtl ::= testexpr c_stmts_opt
+
         _ifstmts_jump ::= c_stmts_opt JUMP_FORWARD come_froms
         bp_come_from    ::= POP_BLOCK COME_FROM
 
@@ -96,6 +98,27 @@ class Python27Parser(Python2Parser):
         while1stmt ::= SETUP_LOOP return_stmts COME_FROM
         """
 
+    def add_custom_rules(self, tokens, customize):
+        super(Python27Parser, self).add_custom_rules(tokens, customize)
+        self.check_reduce['and'] = 'AST'
+        return
+
+    def reduce_is_invalid(self, rule, ast, tokens, first, last):
+        invalid = super(Python27Parser,
+                        self).reduce_is_invalid(rule, ast,
+                                                tokens, first, last)
+        if invalid:
+            return invalid
+        if rule == ('and', ('expr', 'jmp_false', 'expr', '\\e_come_from_opt')):
+            # Test that jmp_false jumps to the end of "and"
+            # or that it jumps to the same place as the end of "and"
+            jmp_false = ast[1][0]
+            jmp_target = jmp_false.offset + jmp_false.attr + 3
+            return not (jmp_target == tokens[last].offset or
+                        tokens[last].pattr == jmp_false.pattr)
+        return False
+
+
 class Python27ParserSingle(Python27Parser, PythonParserSingle):
     pass
 

uncompyle6/parsers/parse3.py

@@ -1,4 +1,4 @@
-#  Copyright (c) 2015, 2016 Rocky Bernstein
+#  Copyright (c) 2015-2017 Rocky Bernstein
 #  Copyright (c) 2005 by Dan Pascu <dan@windowmaker.org>
 #  Copyright (c) 2000-2002 by hartmut Goebel <h.goebel@crazy-compilers.com>
 #  Copyright (c) 1999 John Aycock
@@ -149,23 +149,25 @@ class Python3Parser(PythonParser):
         iflaststmtl ::= testexpr c_stmts_opt JUMP_BACK COME_FROM_LOOP
 
         # These are used to keep AST indices the same
-        jf_else    ::= JUMP_FORWARD ELSE
-        ja_else    ::= JUMP_ABSOLUTE ELSE
+        jump_forward_else  ::= JUMP_FORWARD ELSE
+        jump_absolute_else ::= JUMP_ABSOLUTE ELSE
 
         # Note: in if/else kinds of statements, we err on the side
         # of missing "else" clauses. Therefore we include grammar
         # rules with and without ELSE.
 
-        ifelsestmt ::= testexpr c_stmts_opt JUMP_FORWARD else_suite COME_FROM
-        ifelsestmt ::= testexpr c_stmts_opt jf_else else_suite _come_from
+        ifelsestmt ::= testexpr c_stmts_opt JUMP_FORWARD else_suite opt_come_from_except
+        ifelsestmt ::= testexpr c_stmts_opt jump_forward_else else_suite _come_from
 
         ifelsestmtc ::= testexpr c_stmts_opt JUMP_ABSOLUTE else_suitec
-        ifelsestmtc ::= testexpr c_stmts_opt ja_else else_suitec
+        ifelsestmtc ::= testexpr c_stmts_opt jump_absolute_else else_suitec
 
         ifelsestmtr ::= testexpr return_if_stmts return_stmts
 
         ifelsestmtl ::= testexpr c_stmts_opt JUMP_BACK else_suitel
-        ifelsestmtl ::= testexpr c_stmts_opt COME_FROM JUMP_BACK else_suitel
+        ifelsestmtl ::= testexpr c_stmts_opt cf_jump_back else_suitel
+
+        cf_jump_back ::= COME_FROM JUMP_BACK
 
         # FIXME: this feels like a hack. Is it just 1 or two
         # COME_FROMs?  the parsed tree for this and even with just the
@@ -180,14 +182,17 @@ class Python3Parser(PythonParser):
                            POP_BLOCK LOAD_CONST
                            come_from_or_finally suite_stmts_opt END_FINALLY
 
+        tryelsestmt    ::= SETUP_EXCEPT suite_stmts_opt POP_BLOCK
+                           try_middle else_suite come_from_except_clauses
+
         tryelsestmt    ::= SETUP_EXCEPT suite_stmts_opt POP_BLOCK
                            try_middle else_suite come_froms
 
         tryelsestmtc ::= SETUP_EXCEPT suite_stmts_opt POP_BLOCK
-                         try_middle else_suitec COME_FROM
+                         try_middle else_suitec come_from_except_clauses
 
         tryelsestmtl ::= SETUP_EXCEPT suite_stmts_opt POP_BLOCK
-                         try_middle else_suitel COME_FROM
+                         try_middle else_suitel come_from_except_clauses
 
         try_middle ::= jmp_abs COME_FROM except_stmts
                        END_FINALLY
@@ -254,7 +259,10 @@ class Python3Parser(PythonParser):
 
     def p_misc3(self, args):
         """
-        try_middle ::= JUMP_FORWARD COME_FROM_EXCEPT except_stmts END_FINALLY COME_FROM
+        try_middle ::= JUMP_FORWARD COME_FROM_EXCEPT except_stmts
+                       END_FINALLY COME_FROM
+        try_middle ::= JUMP_FORWARD COME_FROM_EXCEPT except_stmts
+                       END_FINALLY COME_FROM_EXCEPT_CLAUSE
 
         for_block ::= l_stmts_opt opt_come_from_loop JUMP_BACK
         for_block ::= l_stmts
@@ -269,12 +277,14 @@ class Python3Parser(PythonParser):
         stmt ::= funcdef_annotate
         funcdef_annotate ::= mkfunc_annotate designator
 
+        mkfuncdeco0 ::= mkfunc_annotate
+
         # This has the annotation value.
         # LOAD_NAME is used in an annotation type like
         # int, float, str
         annotate_arg    ::= LOAD_NAME
         # LOAD_CONST is used in an annotation string
-        annotate_arg    ::= LOAD_CONST
+        annotate_arg    ::= expr
 
         # This stores the tuple of parameter names
         # that have been annotated
@@ -285,10 +295,13 @@ class Python3Parser(PythonParser):
         """
         opt_come_from_except ::= COME_FROM_EXCEPT
         opt_come_from_except ::= come_froms
+        opt_come_from_except ::= come_from_except_clauses
 
-        come_froms ::= come_froms COME_FROM
-        come_froms ::=
+        come_froms ::= COME_FROM*
 
+        come_from_except_clauses ::= COME_FROM_EXCEPT_CLAUSE+
+
+        opt_come_from_loop ::= opt_come_from_loop COME_FROM_LOOP
         opt_come_from_loop ::= opt_come_from_loop COME_FROM_LOOP
         opt_come_from_loop ::=
 
@@ -321,7 +334,9 @@ class Python3Parser(PythonParser):
 
     def p_stmt3(self, args):
         """
-        stmt ::= LOAD_CLOSURE RETURN_VALUE RETURN_LAST
+        stmt ::= return_closure
+        return_closure ::= LOAD_CLOSURE RETURN_VALUE RETURN_LAST
+
         stmt ::= whileTruestmt
         ifelsestmt ::= testexpr c_stmts_opt JUMP_FORWARD else_suite _come_from
         """
@@ -372,6 +387,8 @@ class Python3Parser(PythonParser):
 
         while1stmt        ::= SETUP_LOOP l_stmts COME_FROM JUMP_BACK COME_FROM_LOOP
 
+        while1elsestmt    ::= SETUP_LOOP l_stmts JUMP_BACK
+                              else_suite COME_FROM_LOOP
 
         # FIXME: investigate - can code really produce a NOP?
         whileTruestmt     ::= SETUP_LOOP l_stmts_opt JUMP_BACK NOP
@@ -393,9 +410,12 @@ class Python3Parser(PythonParser):
 
     def p_expr3(self, args):
         """
-        conditional    ::= expr jmp_false expr jf_else expr COME_FROM
-        conditionalnot ::= expr jmp_true  expr jf_else expr COME_FROM
+        conditional    ::= expr jmp_false expr jump_forward_else expr COME_FROM
+        conditionalnot ::= expr jmp_true  expr jump_forward_else expr COME_FROM
 
+        # a JUMP_FORWARD to another JUMP_FORWARD can get turned into
+        # a JUMP_ABSOLUTE with no COME_FROM
+        conditional    ::= expr jmp_false expr jump_absolute_else expr
 
         expr ::= LOAD_CLASSNAME
 
@@ -453,9 +473,9 @@ class Python3Parser(PythonParser):
     def custom_classfunc_rule(self, opname, token, customize):
         """
         call_function ::= expr {expr}^n CALL_FUNCTION_n
-        call_function ::= expr {expr}^n CALL_FUNCTION_VAR_n POP_TOP
-        call_function ::= expr {expr}^n CALL_FUNCTION_VAR_KW_n POP_TOP
-        call_function ::= expr {expr}^n CALL_FUNCTION_KW_n POP_TOP
+        call_function ::= expr {expr}^n CALL_FUNCTION_VAR_n
+        call_function ::= expr {expr}^n CALL_FUNCTION_VAR_KW_n
+        call_function ::= expr {expr}^n CALL_FUNCTION_KW_n
 
         classdefdeco2 ::= LOAD_BUILD_CLASS mkfunc {expr}^n-1 CALL_FUNCTION_n
         """
@@ -463,25 +483,55 @@ class Python3Parser(PythonParser):
         # high byte number of positional parameters
         args_pos = token.attr & 0xff
         args_kw = (token.attr >> 8) & 0xff
+        args_kw = (token.attr >> 8) & 0xff
+        # args_ann = (token.attr >> 16) & 0x7FFF
+
+        # Additional exprs for * and ** args:
+        #  0 if neither
+        #  1 for CALL_FUNCTION_VAR or CALL_FUNCTION_KW
+        #  2 for * and ** args (CALL_FUNCTION_VAR_KW).
+        # Yes, this computation based on instruction name is a little bit hoaky.
         nak = ( len(opname)-len('CALL_FUNCTION') ) // 3
+
         token.type = self.call_fn_name(token)
-        rule = ('call_function ::= expr ' +
-                ('pos_arg ' * args_pos) +
-                ('kwarg ' * args_kw) +
-                'expr ' * nak + token.type)
-        self.add_unique_rule(rule, token.type, args_pos, customize)
+        uniq_param = args_kw + args_pos
+        if self.version == 3.5 and opname.startswith('CALL_FUNCTION_VAR'):
+            # Python 3.5 changes the stack position of *args. KW args come
+            # after *args.
+            # Python 3.6+ replaces CALL_FUNCTION_VAR_KW with CALL_FUNCTION_EX
+            if opname.endswith('KW'):
+                kw = 'expr '
+            else:
+                kw = ''
+            rule = ('call_function ::= expr expr ' +
+                    ('pos_arg ' * args_pos) +
+                    ('kwarg ' * args_kw) + kw + token.type)
+            self.add_unique_rule(rule, token.type, uniq_param, customize)
+        if self.version >= 3.6 and opname == 'CALL_FUNCTION_EX_KW':
+            rule = ('call_function36 ::= '
+                    'expr build_tuple_unpack_with_call build_map_unpack_with_call '
+                    'CALL_FUNCTION_EX_KW_1')
+            self.add_unique_rule(rule, token.type, uniq_param, customize)
+            rule = 'call_function ::= call_function36'
+        else:
+            rule = ('call_function ::= expr ' +
+                    ('pos_arg ' * args_pos) +
+                    ('kwarg ' * args_kw) +
+                    'expr ' * nak + token.type)
+
+        self.add_unique_rule(rule, token.type, uniq_param, customize)
         if self.version >= 3.5:
             rule = ('async_call_function ::= expr ' +
                     ('pos_arg ' * args_pos) +
                     ('kwarg ' * args_kw) +
                     'expr ' * nak + token.type +
                     ' GET_AWAITABLE LOAD_CONST YIELD_FROM')
-            self.add_unique_rule(rule, token.type, args_pos, customize)
-            self.add_unique_rule('expr ::= async_call_function', token.type, args_pos, customize)
+            self.add_unique_rule(rule, token.type, uniq_param, customize)
+            self.add_unique_rule('expr ::= async_call_function', token.type, uniq_param, customize)
 
         rule = ('classdefdeco2 ::= LOAD_BUILD_CLASS mkfunc %s%s_%d'
                 %  (('expr ' * (args_pos-1)), opname, args_pos))
-        self.add_unique_rule(rule, token.type, args_pos, customize)
+        self.add_unique_rule(rule, token.type, uniq_param, customize)
 
     def add_make_function_rule(self, rule, opname, attr, customize):
         """Python 3.3 added a an addtional LOAD_CONST before MAKE_FUNCTION and
@@ -566,8 +616,9 @@ class Python3Parser(PythonParser):
                     assign2_pypy ::= expr expr designator designator
                 """, nop_func)
                 continue
-            elif opname in ('CALL_FUNCTION', 'CALL_FUNCTION_VAR',
-                            'CALL_FUNCTION_VAR_KW', 'CALL_FUNCTION_KW'):
+            elif (opname in ('CALL_FUNCTION', 'CALL_FUNCTION_VAR',
+                             'CALL_FUNCTION_VAR_KW', 'CALL_FUNCTION_EX_KW')
+                  or opname.startswith('CALL_FUNCTION_KW')):
                 self.custom_classfunc_rule(opname, token, customize)
             elif opname == 'LOAD_DICTCOMP':
                 rule_pat = ("dictcomp ::= LOAD_DICTCOMP %sMAKE_FUNCTION_0 expr "
@@ -580,6 +631,26 @@ class Python3Parser(PythonParser):
                 self.add_make_function_rule(rule_pat, opname, token.attr, customize)
             elif opname == 'LOAD_BUILD_CLASS':
                 self.custom_build_class_rule(opname, i, token, tokens, customize)
+            elif opname.startswith('BUILD_LIST_UNPACK'):
+                v = token.attr
+                rule = ('build_list_unpack ::= ' + 'expr1024 ' * int(v//1024) +
+                        'expr32 ' * int((v//32) % 32) +
+                        'expr ' * (v % 32) + opname)
+                self.add_unique_rule(rule, opname, token.attr, customize)
+                rule = 'expr ::= build_list_unpack'
+                self.add_unique_rule(rule, opname, token.attr, customize)
+            elif opname.startswith('BUILD_TUPLE_UNPACK_WITH_CALL'):
+                v = token.attr
+                rule = ('build_tuple_unpack_with_call ::= ' + 'expr1024 ' * int(v//1024) +
+                        'expr32 ' * int((v//32) % 32) +
+                        'expr ' * (v % 32) + opname)
+                self.add_unique_rule(rule, opname, token.attr, customize)
+            elif opname.startswith('BUILD_MAP_UNPACK_WITH_CALL'):
+                v = token.attr
+                rule = ('build_map_unpack_with_call ::= ' + 'expr1024 ' * int(v//1024) +
+                        'expr32 ' * int((v//32) % 32) +
+                        'expr ' * (v % 32) + opname)
+                self.add_unique_rule(rule, opname, token.attr, customize)
             elif opname_base in ('BUILD_LIST', 'BUILD_TUPLE', 'BUILD_SET'):
                 v = token.attr
                 rule = ('build_list ::= ' + 'expr1024 ' * int(v//1024) +
@@ -589,13 +660,17 @@ class Python3Parser(PythonParser):
                 if opname_base == 'BUILD_TUPLE':
                     rule = ('load_closure ::= %s%s' % (('LOAD_CLOSURE ' * v), opname))
                     self.add_unique_rule(rule, opname, token.attr, customize)
-
+                    rule = ('build_tuple ::= ' + 'expr1024 ' * int(v//1024) +
+                        'expr32 ' * int((v//32) % 32) +
+                        'expr ' * (v % 32) + opname)
+                    self.add_unique_rule(rule, opname, token.attr, customize)
             elif opname == 'LOOKUP_METHOD':
                 # A PyPy speciality - DRY with parse2
                 self.add_unique_rule("load_attr ::= expr LOOKUP_METHOD",
                                      opname, token.attr, customize)
                 continue
             elif opname == 'JUMP_IF_NOT_DEBUG':
+                v = token.attr
                 self.add_unique_rule(
                     "stmt ::= assert_pypy", opname, v, customize)
                 self.add_unique_rule(
@@ -625,10 +700,20 @@ class Python3Parser(PythonParser):
                     rule = "mapexpr ::=  BUILD_MAP_n kvlist_n"
                 elif self.version >= 3.5:
                     if opname != 'BUILD_MAP_WITH_CALL':
-                        rule = kvlist_n + ' ::= ' + 'expr ' * (token.attr*2)
-                        self.add_unique_rule(rule, opname, token.attr, customize)
-                        rule = "mapexpr ::=  %s %s" % (kvlist_n, opname)
-
+                        if opname == 'BUILD_MAP_UNPACK':
+                            rule = kvlist_n + ' ::= ' + 'expr ' * (token.attr*2)
+                            self.add_unique_rule(rule, opname, token.attr, customize)
+                            rule = 'dict ::= ' + 'expr ' * (token.attr*2)
+                            self.add_unique_rule(rule, opname, token.attr, customize)
+                            rule = 'mapexpr ::= ' + 'dict ' * token.attr
+                            self.add_unique_rule(rule, opname, token.attr, customize)
+                            rule = ('unmap_dict ::= ' +
+                                    ('mapexpr ' * token.attr) +
+                                    'BUILD_MAP_UNPACK')
+                        else:
+                            rule = kvlist_n + ' ::= ' + 'expr ' * (token.attr*2)
+                            self.add_unique_rule(rule, opname, token.attr, customize)
+                            rule = "mapexpr ::=  %s %s" % (kvlist_n, opname)
                 else:
                     rule = kvlist_n + ' ::= ' + 'expr expr STORE_MAP ' * token.attr
                     self.add_unique_rule(rule, opname, token.attr, customize)
@@ -650,12 +735,38 @@ class Python3Parser(PythonParser):
                 rule = 'unpack_list ::= ' + opname + ' designator' * token.attr
             elif opname_base.startswith('MAKE_FUNCTION'):
                 # DRY with MAKE_CLOSURE
-                args_pos, args_kw, annotate_args  = token.attr
+                if self.version >= 3.6:
+                    # The semantics of MAKE_FUNCTION in 3.6 are totally different from
+                    # before.
+                    args_pos, args_kw, annotate_args, closure  = token.attr
+                    stack_count = args_pos + args_kw + annotate_args
+                    rule = ('mkfunc ::= %s%s%s%s' %
+                                ('expr ' * stack_count,
+                                 'load_closure ' * closure,
+                                 'LOAD_CONST ' * 2,
+                                 opname))
+                    self.add_unique_rule(rule, opname, token.attr, customize)
+                    rule_pat = ('mklambda ::= %s%sLOAD_LAMBDA %%s%s' %
+                                (('pos_arg '* args_pos),
+                                 ('kwarg '* args_kw),
+                                 opname))
+                    self.add_make_function_rule(rule_pat, opname, token.attr, customize)
+                    rule_pat  = ("listcomp ::= %sLOAD_LISTCOMP %%s%s expr "
+                                 "GET_ITER CALL_FUNCTION_1" % ('expr ' * args_pos, opname))
+                    self.add_make_function_rule(rule_pat, opname, token.attr, customize)
+                    continue
+                if self.version < 3.6:
+                    args_pos, args_kw, annotate_args  = token.attr
+                else:
+                    args_pos, args_kw, annotate_args, closure  = token.attr
 
                 rule_pat = ("genexpr ::= %sload_genexpr %%s%s expr "
                             "GET_ITER CALL_FUNCTION_1" % ('pos_arg '* args_pos, opname))
                 self.add_make_function_rule(rule_pat, opname, token.attr, customize)
-                rule_pat = ('mklambda ::= %sLOAD_LAMBDA %%s%s' % ('pos_arg '* args_pos, opname))
+                rule_pat = ('mklambda ::= %s%sLOAD_LAMBDA %%s%s' %
+                            (('pos_arg '* args_pos),
+                            ('kwarg '* args_kw),
+                            opname))
                 self.add_make_function_rule(rule_pat, opname, token.attr, customize)
                 rule_pat  = ("listcomp ::= %sLOAD_LISTCOMP %%s%s expr "
                              "GET_ITER CALL_FUNCTION_1" % ('expr ' * args_pos, opname))
@@ -676,10 +787,18 @@ class Python3Parser(PythonParser):
                             ('pos_arg ' * args_pos, opname))
                 self.add_unique_rule(rule, opname, token.attr, customize)
                 if opname.startswith('MAKE_FUNCTION_A'):
-                    # rule = ('mkfunc2 ::= %s%sEXTENDED_ARG %s' %
-                    #         ('pos_arg ' * (args_pos), 'kwargs ' * (annotate_args-1), opname))
-                    self.add_unique_rule(rule, opname, token.attr, customize)
+                    if self.version >= 3.6:
+                        rule = ('mkfunc_annotate ::= %s%sannotate_tuple LOAD_CONST LOAD_CONST %s' %
+                                (('pos_arg ' * (args_pos)),
+                                 ('call_function ' * (annotate_args-1)), opname))
+                        self.add_unique_rule(rule, opname, token.attr, customize)
+                        rule = ('mkfunc_annotate ::= %s%sannotate_tuple LOAD_CONST LOAD_CONST %s' %
+                                (('pos_arg ' * (args_pos)),
+                                 ('annotate_arg ' * (annotate_args-1)), opname))
                     if self.version >= 3.3:
+                        # Normally we remove EXTENDED_ARG from the opcodes, but in the case of
+                        # annotated functions can use the EXTENDED_ARG tuple to signal we have an annotated function.
+                        # Yes this is a little hacky
                         rule = ('mkfunc_annotate ::= %s%sannotate_tuple LOAD_CONST LOAD_CONST EXTENDED_ARG %s' %
                                 (('pos_arg ' * (args_pos)),
                                  ('call_function ' * (annotate_args-1)), opname))
@@ -688,6 +807,7 @@ class Python3Parser(PythonParser):
                                 (('pos_arg ' * (args_pos)),
                                  ('annotate_arg ' * (annotate_args-1)), opname))
                     else:
+                        # See above comment about use of EXTENDED_ARG
                         rule = ('mkfunc_annotate ::= %s%sannotate_tuple LOAD_CONST EXTENDED_ARG %s' %
                                 (('pos_arg ' * (args_pos)),
                                  ('annotate_arg ' * (annotate_args-1)), opname))

uncompyle6/parsers/parse32.py

@@ -20,10 +20,17 @@ class Python32Parser(Python3Parser):
         whileTruestmt     ::= SETUP_LOOP return_stmts
                               COME_FROM_LOOP
 
+        try_middle ::= JUMP_FORWARD COME_FROM_EXCEPT except_stmts
+                       END_FINALLY
+
         # Python 3.2+ has more loop optimization that removes
         # JUMP_FORWARD in some cases, and hence we also don't
         # see COME_FROM
         _ifstmts_jump ::= c_stmts_opt
+        _ifstmts_jump ::= c_stmts_opt JUMP_FORWARD _come_from
+
+        stmt           ::= del_deref_stmt
+        del_deref_stmt ::= DELETE_DEREF
         """
     pass
 

uncompyle6/parsers/parse33.py

@@ -20,7 +20,22 @@ class Python33Parser(Python32Parser):
 
         iflaststmt    ::= testexpr c_stmts_opt33
         c_stmts_opt33 ::= JUMP_BACK JUMP_ABSOLUTE c_stmts_opt
-        _ifstmts_jump ::= c_stmts_opt JUMP_FORWARD _come_from
+
+        whileTruestmt ::= SETUP_LOOP l_stmts JUMP_ABSOLUTE
+                          JUMP_BACK COME_FROM_LOOP
+
+        # Python 3.5+ has jump optimization to remove the redundant
+        # jump_excepts. But in 3.3 we need them added
+
+        tryelsestmt ::= SETUP_EXCEPT suite_stmts_opt POP_BLOCK
+                        try_middle else_suite
+                        jump_excepts come_from_except_clauses
+
+        trystmt     ::= SETUP_EXCEPT suite_stmts_opt POP_BLOCK
+                        try_middle
+                        jump_excepts come_from_except_clauses
+
+        jump_excepts ::= jump_except+
         """
 
 class Python33ParserSingle(Python33Parser, PythonParserSingle):

uncompyle6/parsers/parse34.py

@@ -15,6 +15,8 @@ class Python34Parser(Python33Parser):
 
     def p_misc34(self, args):
         """
+        expr ::= LOAD_ASSERT
+
         # Python 3.4+ optimizes the trailing two JUMPS away
 
         # Is this 3.4 only?

uncompyle6/parsers/parse35.py

@@ -20,12 +20,27 @@ class Python35Parser(Python34Parser):
         # I'm sure by the time Python 4 comes around these will be turned
         # into special opcodes
 
-        while1stmt  ::= SETUP_LOOP l_stmts COME_FROM JUMP_BACK
-                        POP_BLOCK COME_FROM_LOOP
+        while1stmt     ::= SETUP_LOOP l_stmts COME_FROM JUMP_BACK
+                           POP_BLOCK COME_FROM_LOOP
+        while1stmt     ::= SETUP_LOOP l_stmts POP_BLOCK COME_FROM_LOOP
+        while1elsestmt ::= SETUP_LOOP l_stmts JUMP_BACK
+                           POP_BLOCK else_suite COME_FROM_LOOP
 
         # Python 3.5+ Await statement
-        stmt ::= await_stmt
-        await_stmt ::= call_function GET_AWAITABLE LOAD_CONST YIELD_FROM POP_TOP
+        expr       ::= await_expr
+        await_expr ::= expr GET_AWAITABLE LOAD_CONST YIELD_FROM
+
+        stmt       ::= await_stmt
+        await_stmt ::= await_expr POP_TOP
+
+        expr       ::= unmap_dict
+        expr       ::= unmapexpr
+
+        unmap_dict ::= dictcomp BUILD_MAP_UNPACK
+
+        unmap_dict ::= kv_lists BUILD_MAP_UNPACK
+        kv_lists   ::= kv_list kv_lists
+        kv_lists   ::= kv_list
 
         # Python 3.5+ has WITH_CLEANUP_START/FINISH
 
@@ -65,20 +80,31 @@ class Python35Parser(Python34Parser):
                                GET_AWAITABLE LOAD_CONST YIELD_FROM
                                WITH_CLEANUP_FINISH END_FINALLY
 
-
         stmt               ::= async_for_stmt
         async_for_stmt     ::= SETUP_LOOP expr
                                GET_AITER
                                LOAD_CONST YIELD_FROM SETUP_EXCEPT GET_ANEXT LOAD_CONST
                                YIELD_FROM
                                designator
-                               POP_BLOCK JUMP_FORWARD COME_FROM_EXCEPT DUP_TOP
+                               POP_BLOCK jump_except COME_FROM_EXCEPT DUP_TOP
                                LOAD_GLOBAL COMPARE_OP POP_JUMP_IF_FALSE
                                POP_TOP POP_TOP POP_TOP POP_EXCEPT POP_BLOCK
                                JUMP_ABSOLUTE END_FINALLY COME_FROM
                                for_block POP_BLOCK JUMP_ABSOLUTE
                                opt_come_from_loop
 
+        async_for_stmt     ::= SETUP_LOOP expr
+                               GET_AITER
+                               LOAD_CONST YIELD_FROM SETUP_EXCEPT GET_ANEXT LOAD_CONST
+                               YIELD_FROM
+                               designator
+                               POP_BLOCK jump_except COME_FROM_EXCEPT DUP_TOP
+                               LOAD_GLOBAL COMPARE_OP POP_JUMP_IF_FALSE
+                               POP_TOP POP_TOP POP_TOP POP_EXCEPT POP_BLOCK
+                               JUMP_ABSOLUTE END_FINALLY JUMP_BACK
+                               passstmt POP_BLOCK JUMP_ABSOLUTE
+                               opt_come_from_loop
+
         stmt               ::= async_forelse_stmt
         async_forelse_stmt ::= SETUP_LOOP expr
                                GET_AITER
@@ -112,7 +138,6 @@ class Python35Parser(Python34Parser):
         # differently than 3.3, 3.4
 
         yield_from ::= expr GET_YIELD_FROM_ITER LOAD_CONST YIELD_FROM
-
         """
 
     def add_custom_rules(self, tokens, customize):

uncompyle6/parsers/parse36.py

@@ -14,22 +14,31 @@ class Python36Parser(Python35Parser):
         super(Python36Parser, self).__init__(debug_parser)
         self.customized = {}
 
+
     def p_36misc(self, args):
         """
+        # 3.6 redoes how return_closure works
+        return_closure ::= LOAD_CLOSURE DUP_TOP STORE_NAME RETURN_VALUE RETURN_LAST
+
         fstring_multi ::= fstring_expr_or_strs BUILD_STRING
         fstring_expr_or_strs ::= fstring_expr_or_str+
 
         func_args36   ::= expr BUILD_TUPLE_0
         call_function ::= func_args36 unmapexpr CALL_FUNCTION_EX
+        call_function ::= func_args36 build_map_unpack_with_call CALL_FUNCTION_EX_KW_1
 
         withstmt ::= expr SETUP_WITH POP_TOP suite_stmts_opt POP_BLOCK LOAD_CONST
                      WITH_CLEANUP_START WITH_CLEANUP_FINISH END_FINALLY
+
+        call_function ::= expr expr CALL_FUNCTION_EX
+        call_function ::= expr expr expr CALL_FUNCTION_EX_KW_1
         """
 
     def add_custom_rules(self, tokens, customize):
         super(Python36Parser, self).add_custom_rules(tokens, customize)
         for i, token in enumerate(tokens):
             opname = token.type
+
             if opname == 'FORMAT_VALUE':
                 rules_str = """
                     expr ::= fstring_single
@@ -52,6 +61,17 @@ class Python36Parser(Python35Parser):
                 """ % (fstring_expr_or_str_n, fstring_expr_or_str_n, "fstring_expr_or_str " * v)
                 self.add_unique_doc_rules(rules_str, customize)
 
+    def custom_classfunc_rule(self, opname, token, customize):
+
+        if opname.startswith('CALL_FUNCTION_KW'):
+            values = 'expr ' * token.attr
+            rule = 'call_function ::= expr kwargs_only_36 {token.type}'.format(**locals())
+            self.add_unique_rule(rule, token.type, token.attr, customize)
+            rule = 'kwargs_only_36 ::= {values} LOAD_CONST'.format(**locals())
+            self.add_unique_rule(rule, token.type, token.attr, customize)
+        else:
+            super(Python36Parser, self).custom_classfunc_rule(opname, token, customize)
+
 
 class Python36ParserSingle(Python36Parser, PythonParserSingle):
     pass

uncompyle6/scanner.py

@@ -16,6 +16,7 @@ import sys
 
 from uncompyle6 import PYTHON3, IS_PYPY
 from uncompyle6.scanners.tok import Token
+from xdis.bytecode import op_size
 
 # The byte code versions we support
 PYTHON_VERSIONS = (1.5,
@@ -66,6 +67,12 @@ class Scanner(object):
         # FIXME: This weird Python2 behavior is not Python3
         self.resetTokenClass()
 
+    def opname_for_offset(self, offset):
+        return self.opc.opname[self.code[offset]]
+
+    def op_name(self, op):
+        return self.opc.opname[op]
+
     def is_jump_forward(self, offset):
         """
         Return True if the code at offset is some sort of jump forward.
@@ -208,9 +215,6 @@ class Scanner(object):
                         result.append(offset)
         return result
 
-    def op_hasArgument(self, op):
-        return self.op_size(op) > 1
-
     def op_range(self, start, end):
         """
         Iterate through positions of opcodes, skipping
@@ -218,17 +222,7 @@ class Scanner(object):
         """
         while start < end:
             yield start
-            start += self.op_size(self.code[start])
-
-    def op_size(self, op):
-        """
-        Return size of operator with its arguments
-        for given opcode <op>.
-        """
-        if op < self.opc.HAVE_ARGUMENT:
-            return 2 if self.version >= 3.6 else 1
-        else:
-            return 2 if self.version >= 3.6 else 3
+            start += op_size(self.code[start], self.opc)
 
     def remove_mid_line_ifs(self, ifs):
         """
@@ -260,9 +254,6 @@ class Scanner(object):
         self.Token = tokenClass
         return self.Token
 
-def op_has_argument(op, opc):
-    return op >= opc.HAVE_ARGUMENT
-
 def parse_fn_counts(argc):
     return ((argc & 0xFF), (argc >> 8) & 0xFF, (argc >> 16) & 0x7FFF)
 

uncompyle6/scanners/scanner2.py

@@ -1,4 +1,4 @@
-# Copyright (c) 2015, 2016 by Rocky Bernstein
+# Copyright (c) 2015-2017 by Rocky Bernstein
 # Copyright (c) 2005 by Dan Pascu <dan@windowmaker.org>
 # Copyright (c) 2000-2002 by hartmut Goebel <h.goebel@crazy-compilers.com>
 """
@@ -25,14 +25,15 @@ from __future__ import print_function
 from collections import namedtuple
 from array import array
 
-from uncompyle6.scanner import op_has_argument
+from uncompyle6.scanner import L65536
 from xdis.code import iscode
+from xdis.bytecode import op_has_argument, op_size
 
-import uncompyle6.scanner as scan
+from uncompyle6.scanner import Scanner
 
-class Scanner2(scan.Scanner):
+class Scanner2(Scanner):
     def __init__(self, version, show_asm=None, is_pypy=False):
-        scan.Scanner.__init__(self, version, show_asm, is_pypy)
+        Scanner.__init__(self, version, show_asm, is_pypy)
         self.pop_jump_if = frozenset([self.opc.PJIF, self.opc.PJIT])
         self.jump_forward = frozenset([self.opc.JUMP_ABSOLUTE, self.opc.JUMP_FORWARD])
         # This is the 2.5+ default
@@ -164,7 +165,7 @@ class Scanner2(scan.Scanner):
                     #     continue
                     # last_offset = jump_offset
                     come_from_name = 'COME_FROM'
-                    op_name = self.opc.opname[self.code[jump_offset]]
+                    op_name = self.opname_for_offset(jump_offset)
                     if op_name.startswith('SETUP_') and self.version == 2.7:
                         come_from_type = op_name[len('SETUP_'):]
                         if come_from_type not in ('LOOP', 'EXCEPT'):
@@ -178,7 +179,7 @@ class Scanner2(scan.Scanner):
                     pass
 
             op = self.code[offset]
-            op_name = self.opc.opname[op]
+            op_name = self.op_name(op)
 
             oparg = None; pattr = None
             has_arg = op_has_argument(op, self.opc)
@@ -186,7 +187,7 @@ class Scanner2(scan.Scanner):
                 oparg = self.get_argument(offset) + extended_arg
                 extended_arg = 0
                 if op == self.opc.EXTENDED_ARG:
-                    extended_arg = oparg * scan.L65536
+                    extended_arg = oparg * L65536
                     continue
                 if op in self.opc.hasconst:
                     const = co.co_consts[oparg]
@@ -327,7 +328,7 @@ class Scanner2(scan.Scanner):
         for i in self.op_range(0, n):
             op = self.code[i]
             self.prev.append(i)
-            if self.op_hasArgument(op):
+            if op_has_argument(op, self.opc):
                 self.prev.append(i)
                 self.prev.append(i)
                 pass
@@ -380,7 +381,7 @@ class Scanner2(scan.Scanner):
                     if elem != code[i]:
                         match = False
                         break
-                    i += self.op_size(code[i])
+                    i += op_size(code[i], self.opc)
 
                 if match:
                     i = self.prev[i]
@@ -406,7 +407,7 @@ class Scanner2(scan.Scanner):
                 while code[j] == self.opc.JUMP_ABSOLUTE:
                     j = self.prev[j]
                 if (self.version >= 2.3 and
-                    self.opc.opname[code[j]] == 'LIST_APPEND'): # list comprehension
+                    self.opname_for_offset(j) == 'LIST_APPEND'): # list comprehension
                     stmts.remove(s)
                     continue
             elif code[s] == self.opc.POP_TOP:
@@ -617,7 +618,7 @@ class Scanner2(scan.Scanner):
                                        'start': jump_back+3,
                                        'end':   end})
         elif op == self.opc.SETUP_EXCEPT:
-            start  = offset + self.op_size(op)
+            start  = offset + op_size(op, self.opc)
             target = self.get_target(offset, op)
             end    = self.restrict_to_parent(target, parent)
             if target != end:
@@ -641,7 +642,7 @@ class Scanner2(scan.Scanner):
                         setup_except_nest -= 1
                 elif self.code[end_finally_offset] == self.opc.SETUP_EXCEPT:
                     setup_except_nest += 1
-                end_finally_offset += self.op_size(code[end_finally_offset])
+                end_finally_offset += op_size(code[end_finally_offset], self.opc)
                 pass
 
             # Add the except blocks
@@ -842,9 +843,9 @@ class Scanner2(scan.Scanner):
                     else:
                         # We still have the case in 2.7 that the next instruction
                         # is a jump to a SETUP_LOOP target.
-                        next_offset = target + self.op_size(self.code[target])
+                        next_offset = target + op_size(self.code[target], self.opc)
                         next_op = self.code[next_offset]
-                        if self.opc.opname[next_op] == 'JUMP_FORWARD':
+                        if self.op_name(next_op) == 'JUMP_FORWARD':
                             jump_target = self.get_target(next_offset, next_op)
                             if jump_target in self.setup_loops:
                                 self.structs.append({'type':  'while-loop',
@@ -882,12 +883,12 @@ class Scanner2(scan.Scanner):
                     # 39_0  COME_FROM 3
                     # 40 ...
 
-                    if self.opc.opname[code[jump_if_offset]].startswith('JUMP_IF'):
+                    if self.opname_for_offset(jump_if_offset).startswith('JUMP_IF'):
                         jump_if_target = code[jump_if_offset+1]
-                        if self.opc.opname[code[jump_if_target + jump_if_offset + 3]] == 'POP_TOP':
+                        if self.opname_for_offset(jump_if_target + jump_if_offset + 3) == 'POP_TOP':
                             jump_inst = jump_if_target + jump_if_offset
                             jump_offset = code[jump_inst+1]
-                            jump_op = self.opc.opname[code[jump_inst]]
+                            jump_op = self.opname_for_offset(jump_inst)
                             if (jump_op == 'JUMP_FORWARD' and jump_offset == 1):
                                 self.structs.append({'type':  'if-then',
                                                      'start': start-3,
@@ -922,7 +923,7 @@ class Scanner2(scan.Scanner):
                     # 256
                     if if_then_maybe and jump_op == 'JUMP_ABSOLUTE':
                         jump_target = self.get_target(jump_inst, code[jump_inst])
-                        if self.opc.opname[code[end]] == 'JUMP_FORWARD':
+                        if self.opname_for_offset(end) == 'JUMP_FORWARD':
                             end_target = self.get_target(end, code[end])
                             if jump_target == end_target:
                                 self.structs.append(if_then_maybe)
@@ -991,7 +992,7 @@ class Scanner2(scan.Scanner):
                 oparg = self.get_argument(offset)
 
                 if label is None:
-                    if op in self.opc.hasjrel and self.opc.opname[op] != 'FOR_ITER':
+                    if op in self.opc.hasjrel and self.op_name(op) != 'FOR_ITER':
                         # if (op in self.opc.hasjrel and
                         #     (self.version < 2.0 or op != self.opc.FOR_ITER)):
                         label = offset + 3 + oparg

uncompyle6/scanners/scanner26.py

@@ -1,4 +1,4 @@
-#  Copyright (c) 2015, 2016 by Rocky Bernstein
+#  Copyright (c) 2015-2017 by Rocky Bernstein
 #  Copyright (c) 2005 by Dan Pascu <dan@windowmaker.org>
 #  Copyright (c) 2000-2002 by hartmut Goebel <h.goebel@crazy-compilers.com>
 """
@@ -15,6 +15,7 @@ if PYTHON3:
     intern = sys.intern
 
 import uncompyle6.scanners.scanner2 as scan
+from uncompyle6.scanner import L65536
 
 # bytecode verification, verify(), uses JUMP_OPs from here
 from xdis.opcodes import opcode_26
@@ -178,8 +179,7 @@ class Scanner26(scan.Scanner2):
                 oparg = self.get_argument(offset) + extended_arg
                 extended_arg = 0
                 if op == self.opc.EXTENDED_ARG:
-                    raise NotImplementedError
-                    extended_arg = oparg * scan.L65536
+                    extended_arg = oparg * L65536
                     continue
                 if op in self.opc.hasconst:
                     const = co.co_consts[oparg]

uncompyle6/scanners/scanner27.py

@@ -1,4 +1,4 @@
-# Copyright (c) 2015, 2016 by Rocky Bernstein
+# Copyright (c) 2015-2017 by Rocky Bernstein
 """
 Python 2.7 bytecode ingester.
 
@@ -48,7 +48,7 @@ class Scanner27(Scanner2):
             self.opc.DELETE_SLICE_2,   self.opc.DELETE_SLICE_3,
         ])
 
-        # opcodes with expect a variable number pushed values whose
+        # opcodes which expect a variable number pushed values and whose
         # count is in the opcode. For parsing we generally change the
         # opcode name to include that number.
         varargs_ops = set([

uncompyle6/scanners/scanner3.py

@@ -1,4 +1,4 @@
-#  Copyright (c) 2015, 2016 by Rocky Bernstein
+#  Copyright (c) 2015-2017 by Rocky Bernstein
 #  Copyright (c) 2005 by Dan Pascu <dan@windowmaker.org>
 #  Copyright (c) 2000-2002 by hartmut Goebel <h.goebel@crazy-compilers.com>
 """
@@ -25,10 +25,11 @@ from __future__ import print_function
 from collections import namedtuple
 from array import array
 
-from uncompyle6.scanner import Scanner, op_has_argument
+from uncompyle6.scanner import Scanner
 from xdis.code import iscode
-from xdis.bytecode import Bytecode
+from xdis.bytecode import Bytecode, op_has_argument, op_size
 from uncompyle6.scanner import Token, parse_fn_counts
+import xdis
 
 # Get all the opcodes into globals
 import xdis.opcodes.opcode_33 as op3
@@ -40,9 +41,6 @@ if PYTHON3:
 
 globals().update(op3.opmap)
 
-# POP_JUMP_IF is used by verify
-POP_JUMP_TF = (POP_JUMP_IF_TRUE, POP_JUMP_IF_FALSE)
-
 class Scanner3(Scanner):
 
     def __init__(self, version, show_asm=None, is_pypy=False):
@@ -61,6 +59,11 @@ class Scanner3(Scanner):
             setup_ops.append(self.opc.SETUP_WITH)
         self.setup_ops = frozenset(setup_ops)
 
+        if self.version == 3.0:
+            self.pop_jump_tf = frozenset([self.opc.JUMP_IF_FALSE, self.opc.JUMP_IF_TRUE])
+        else:
+            self.pop_jump_tf = frozenset([self.opc.PJIF, self.opc.PJIT])
+
         self.setup_ops_no_loop = frozenset(setup_ops) - frozenset([self.opc.SETUP_LOOP])
 
         # Opcodes that can start a statement.
@@ -128,13 +131,20 @@ class Scanner3(Scanner):
             varargs_ops.add(self.opc.CALL_METHOD)
         if self.version >= 3.6:
             varargs_ops.add(self.opc.BUILD_CONST_KEY_MAP)
+            # Below is in bit order, "default = bit 0, closure = bit 3
+            self.MAKE_FUNCTION_FLAGS = tuple("""
+             default keyword-only annotation closure""".split())
 
         self.varargs_ops = frozenset(varargs_ops)
         # FIXME: remove the above in favor of:
         # self.varargs_ops = frozenset(self.opc.hasvargs)
 
-    def opName(self, offset):
-        return self.opc.opname[self.code[offset]]
+    def extended_arg_val(self, val):
+        if self.version < 3.6:
+            return val * (1<<16)
+        else:
+            return val * (1<<8)
+
 
     def ingest(self, co, classname=None, code_objects={}, show_asm=None):
         """
@@ -206,9 +216,29 @@ class Scanner3(Scanner):
         jump_targets = self.find_jump_targets(show_asm)
         last_op_was_break = False
 
-        for inst in bytecode:
+        extended_arg = 0
+        for i, inst in enumerate(bytecode):
 
             argval = inst.argval
+            op     = inst.opcode
+            has_arg = op_has_argument(op, self.opc)
+            if has_arg:
+                if op == self.opc.EXTENDED_ARG:
+                    extended_arg += self.extended_arg_val(argval)
+
+                    # Normally we remove EXTENDED_ARG from the
+                    # opcodes, but in the case of annotated functions
+                    # can use the EXTENDED_ARG tuple to signal we have
+                    # an annotated function.
+                    if not bs[i+1].opname.startswith("MAKE_FUNCTION"):
+                        continue
+
+            if isinstance(argval, int) and extended_arg:
+                min_extended= self.extended_arg_val(1)
+                if argval < min_extended:
+                    argval += extended_arg
+            extended_arg = 0
+
             if inst.offset in jump_targets:
                 jump_idx = 0
                 # We want to process COME_FROMs to the same offset to be in *descending*
@@ -219,11 +249,16 @@ class Scanner3(Scanner):
                 # "loop" tag last so the grammar rule matches that properly.
                 for jump_offset in sorted(jump_targets[inst.offset], reverse=True):
                     come_from_name = 'COME_FROM'
-                    opname = self.opName(jump_offset)
+                    opname = self.opname_for_offset(jump_offset)
                     if opname.startswith('SETUP_'):
                         come_from_type = opname[len('SETUP_'):]
                         come_from_name = 'COME_FROM_%s' % come_from_type
                         pass
+                    elif inst.offset in self.except_targets:
+                        come_from_name = 'COME_FROM_EXCEPT_CLAUSE'
+                        if self.version <= 3.2:
+                            continue
+                        pass
                     tokens.append(Token(come_from_name,
                                         None, repr(jump_offset),
                                         offset='%s_%s' % (inst.offset, jump_idx),
@@ -240,12 +275,11 @@ class Scanner3(Scanner):
 
                 pass
 
-            pattr =  inst.argrepr
+            pattr  = inst.argrepr
             opname = inst.opname
-            op = inst.opcode
 
             if opname in ['LOAD_CONST']:
-                const = inst.argval
+                const = argval
                 if iscode(const):
                     if const.co_name == '<lambda>':
                         opname = 'LOAD_LAMBDA'
@@ -267,20 +301,37 @@ class Scanner3(Scanner):
                     pattr = const
                     pass
             elif opname in ('MAKE_FUNCTION', 'MAKE_CLOSURE'):
-                pos_args, name_pair_args, annotate_args = parse_fn_counts(inst.argval)
-                if name_pair_args > 0:
-                    opname = '%s_N%d' % (opname, name_pair_args)
-                    pass
-                if annotate_args > 0:
-                    opname = '%s_A_%d' % (opname, annotate_args)
-                    pass
-                opname = '%s_%d' % (opname, pos_args)
-                pattr = ("%d positional, %d keyword pair, %d annotated" %
-                             (pos_args, name_pair_args, annotate_args))
+                if self.version >= 3.6:
+                    # 3.6+ doesn't have MAKE_CLOSURE, so opname == 'MAKE_FUNCTION'
+                    flags = argval
+                    opname = 'MAKE_FUNCTION_%d' % (flags)
+                    attr = []
+                    for flag in self.MAKE_FUNCTION_FLAGS:
+                        bit = flags & 1
+                        if bit:
+                            if pattr:
+                                pattr += ", " + flag
+                            else:
+                                pattr += flag
+                        attr.append(bit)
+                        flags >>= 1
+                    attr = attr[:4] # remove last value: attr[5] == False
+                else:
+                    pos_args, name_pair_args, annotate_args = parse_fn_counts(inst.argval)
+                    pattr = ("%d positional, %d keyword pair, %d annotated" %
+                                 (pos_args, name_pair_args, annotate_args))
+                    if name_pair_args > 0:
+                        opname = '%s_N%d' % (opname, name_pair_args)
+                        pass
+                    if annotate_args > 0:
+                        opname = '%s_A_%d' % (opname, annotate_args)
+                        pass
+                    opname = '%s_%d' % (opname, pos_args)
+                    attr = (pos_args, name_pair_args, annotate_args)
                 tokens.append(
                     Token(
                         type_ = opname,
-                        attr = (pos_args, name_pair_args, annotate_args),
+                        attr = attr,
                         pattr = pattr,
                         offset = inst.offset,
                         linestart = inst.starts_line,
@@ -291,7 +342,7 @@ class Scanner3(Scanner):
                 )
                 continue
             elif op in self.varargs_ops:
-                pos_args = inst.argval
+                pos_args = argval
                 if self.is_pypy and not pos_args and opname == 'BUILD_MAP':
                     opname = 'BUILD_MAP_n'
                 else:
@@ -303,9 +354,9 @@ class Scanner3(Scanner):
                 customize[opname] = 0
             elif opname == 'UNPACK_EX':
                 # FIXME: try with scanner and parser by
-                # changing inst.argval
-                before_args = inst.argval & 0xFF
-                after_args = (inst.argval >> 8) & 0xff
+                # changing argval
+                before_args = argval & 0xFF
+                after_args = (argval >> 8) & 0xff
                 pattr = "%d before vararg, %d after" % (before_args, after_args)
                 argval = (before_args, after_args)
                 opname = '%s_%d+%d' % (opname, before_args, after_args)
@@ -322,7 +373,7 @@ class Scanner3(Scanner):
                 # comprehensions we might sometimes classify JUMP_BACK
                 # as CONTINUE, but that's okay since we add a grammar
                 # rule for that.
-                pattr = inst.argval
+                pattr = argval
                 target = self.get_target(inst.offset)
                 if target <= inst.offset:
                     next_opname = self.opname[self.code[inst.offset+3]]
@@ -331,11 +382,7 @@ class Scanner3(Scanner):
                         (next_opname not in ('END_FINALLY', 'POP_BLOCK',
                                             # Python 3.0 only uses POP_TOP
                                             'POP_TOP'))):
-                        if (self.version >= 3.4 or
-                            (inst.offset not in self.not_continue) or
-                            (tokens[-1].type == 'RETURN_VALUE')):
-                            opname = 'CONTINUE'
-                            pass
+                        opname = 'CONTINUE'
                     else:
                         opname = 'JUMP_BACK'
                         # FIXME: this is a hack to catch stuff like:
@@ -422,7 +469,7 @@ class Scanner3(Scanner):
         self.prev = self.prev_op = [0]
         for offset in self.op_range(0, codelen):
             op = code[offset]
-            for _ in range(self.op_size(op)):
+            for _ in range(op_size(op, self.opc)):
                 self.prev_op.append(offset)
 
     def find_jump_targets(self, debug):
@@ -446,6 +493,7 @@ class Scanner3(Scanner):
 
         # Map fixed jumps to their real destination
         self.fixed_jumps = {}
+        self.except_targets = {}
         self.ignore_if = set()
         self.build_statement_indices()
         self.else_start = {}
@@ -471,7 +519,7 @@ class Scanner3(Scanner):
                     oparg = code[offset+1]
                 else:
                     oparg = code[offset+1] + code[offset+2] * 256
-                next_offset = offset + self.op_size(op)
+                next_offset = xdis.next_offset(op, self.opc, offset)
 
                 if label is None:
                     if op in op3.hasjrel and op != self.opc.FOR_ITER:
@@ -517,7 +565,7 @@ class Scanner3(Scanner):
                     if elem != code[i]:
                         match = False
                         break
-                    i += self.op_size(code[i])
+                    i += op_size(code[i], self.opc)
 
                 if match is True:
                     i = self.prev_op[i]
@@ -582,14 +630,18 @@ class Scanner3(Scanner):
         Get target offset for op located at given <offset>.
         """
         op = self.code[offset]
+        rel_offset = 0
         if self.version  >= 3.6:
             target = self.code[offset+1]
             if op in self.opc.hasjrel:
-                target += offset + 2
+                rel_offset = offset + 2
         else:
             target = self.code[offset+1] + self.code[offset+2] * 256
             if op in self.opc.hasjrel:
-                target += offset + 3
+                rel_offset = offset + 3
+                pass
+            pass
+        target += rel_offset
 
         return target
 
@@ -660,7 +712,7 @@ class Scanner3(Scanner):
 
                 jump_back += 2
                 if_offset = None
-                if code[self.prev_op[next_line_byte]] not in POP_JUMP_TF:
+                if code[self.prev_op[next_line_byte]] not in self.pop_jump_tf:
                     if_offset = self.prev[next_line_byte]
                 if if_offset:
                     loop_type = 'while'
@@ -705,8 +757,8 @@ class Scanner3(Scanner):
                 self.structs.append({'type': loop_type + '-else',
                                        'start': jump_back+3,
                                        'end':   end})
-        elif op in POP_JUMP_TF:
-            start = offset + self.op_size(op)
+        elif op in self.pop_jump_tf:
+            start = offset + op_size(op, self.opc)
             target = self.get_target(offset)
             rtarget = self.restrict_to_parent(target, parent)
             prev_op = self.prev_op
@@ -734,7 +786,7 @@ class Scanner3(Scanner):
             pre_rtarget = prev_op[rtarget]
 
             # Is it an "and" inside an "if" or "while" block
-            if op == self.opc.POP_JUMP_IF_FALSE:
+            if op == self.opc.POP_JUMP_IF_FALSE and self.version < 3.6:
 
                 # Search for another POP_JUMP_IF_FALSE targetting the same op,
                 # in current statement, starting from current offset, and filter
@@ -752,12 +804,12 @@ class Scanner3(Scanner):
                             target == self.get_target(prev_op[pre_rtarget]) and
                             (prev_op[pre_rtarget] not in self.stmts or
                              self.get_target(prev_op[pre_rtarget]) > prev_op[pre_rtarget]) and
-                            1 == len(self.remove_mid_line_ifs(self.rem_or(start, prev_op[pre_rtarget], POP_JUMP_TF, target)))):
+                            1 == len(self.remove_mid_line_ifs(self.rem_or(start, prev_op[pre_rtarget], self.pop_jump_tf, target)))):
                             pass
                         elif (code[prev_op[pre_rtarget]] == self.opc.RETURN_VALUE
                               and self.remove_mid_line_ifs([offset]) and
                               1 == (len(set(self.remove_mid_line_ifs(self.rem_or(start, prev_op[pre_rtarget],
-                                                                                 POP_JUMP_TF, target))) |
+                                                                                 self.pop_jump_tf, target))) |
                                     set(self.remove_mid_line_ifs(self.rem_or(start, prev_op[pre_rtarget],
                                                                              (self.opc.POP_JUMP_IF_FALSE,
                                                                               self.opc.POP_JUMP_IF_TRUE,
@@ -840,6 +892,9 @@ class Scanner3(Scanner):
                 self.structs.append({'type': 'if-then',
                                      'start': start,
                                      'end': pre_rtarget})
+
+                # FIXME: add this
+                # self.fixed_jumps[offset] = rtarget
                 self.not_continue.add(pre_rtarget)
 
                 if rtarget < end and (
@@ -878,9 +933,9 @@ class Scanner3(Scanner):
                     # not from SETUP_EXCEPT
                     next_op = rtarget
                     if code[next_op] == self.opc.POP_BLOCK:
-                        next_op += self.op_size(self.code[next_op])
+                        next_op += op_size(self.code[next_op], self.opc)
                     if code[next_op] == self.opc.JUMP_ABSOLUTE:
-                        next_op += self.op_size(self.code[next_op])
+                        next_op += op_size(self.code[next_op], self.opc)
                     if next_op in targets:
                         for try_op in targets[next_op]:
                             come_from_op = code[try_op]
@@ -888,7 +943,7 @@ class Scanner3(Scanner):
                                 return
                             pass
                     pass
-                if code[pre_rtarget] == self.opc.RETURN_VALUE:
+                if code[pre_rtarget] == self.opc.RETURN_VALUE and self.version < 3.5:
                     self.return_end_ifs.add(pre_rtarget)
                 else:
                     self.fixed_jumps[offset] = rtarget
@@ -902,6 +957,16 @@ class Scanner3(Scanner):
             target = self.get_target(offset)
             end    = self.restrict_to_parent(target, parent)
             self.fixed_jumps[offset] = end
+        elif op == self.opc.POP_EXCEPT:
+            next_offset = xdis.next_offset(op, self.opc, offset)
+            target = self.get_target(next_offset)
+            if target > next_offset:
+                next_op = code[next_offset]
+                if (self.opc.JUMP_ABSOLUTE == next_op and
+                    END_FINALLY != code[xdis.next_offset(next_op, self.opc, next_offset)]):
+                    self.fixed_jumps[next_offset] = target
+                    self.except_targets[target] = next_offset
+
         elif op == self.opc.SETUP_FINALLY:
             target = self.get_target(offset)
             end    = self.restrict_to_parent(target, parent)