Merge branch 'master' into python-2.4

and some parser list nonterminal cleanup

.travis.yml

@@ -3,12 +3,7 @@ language: python
 sudo: false
 
 python:
-  - '3.5'
-  - '2.7.12'
-  - '2.6'
-  - '3.3'
-  - '3.4'
-  - '3.2'
+  - '2.7' # this is a cheat here because travis doesn't do 2.4-2.6
 
 install:
 - pip install -r requirements.txt

HISTORY.md

@@ -30,7 +30,7 @@ another clever idea: using table-driven semantics routines, using
 format specifiers.
 
 The last mention of a release of SPARK from John is around 2002. As
-released, although the Early Algorithm parser was in good shape, this
+released, although the Earley Algorithm parser was in good shape, this
 code was woefully lacking as serious Python deparser.
 
 In the fall of 2000, Hartmut Goebel
@@ -135,9 +135,9 @@ Hartmut a decade an a half ago:
     NB. This is not a masterpiece of software, but became more like a hack.
     Probably a complete rewrite would be sensefull. hG/2000-12-27
 
-This project deparses using an Early-algorithm parse with lots of
+This project deparses using an Earley-algorithm parse with lots of
 massaging of tokens and the grammar in the scanner
-phase. Early-algorithm parsers are context free and tend to be linear
+phase. Earley-algorithm parsers are context free and tend to be linear
 if the grammar is LR or left recursive.
 
 Another approach that doesn't use grammars is to do something like

Makefile

@@ -37,7 +37,7 @@ check-3.0 check-3.1 check-3.2 check-3.5 check-3.6:
 	$(MAKE) -C test $@
 
 #:Tests for Python 2.6 (doesn't have pytest)
-check-2.6:
+check-2.4 check-2.5 check-2.6:
 	$(MAKE) -C test $@
 
 #:PyPy 2.6.1 or PyPy 5.0.1
@@ -59,7 +59,7 @@ clean: clean_pyc
 
 #: Create source (tarball) and wheel distribution
 dist:
-	$(PYTHON) ./setup.py sdist bdist_wheel
+	$(PYTHON) ./setup.py sdist bdist_egg
 
 #: Remove .pyc files
 clean_pyc:

NEWS

@@ -1,3 +1,32 @@
+uncompyle6 2.9.10 2016-02-25
+
+- Python grammar rule fixes
+- Add ability to get grammar coverage on runs
+- Handle Python 3.6 opcode BUILD_CONST_KEYMAP
+
+uncompyle6 2.9.9 2016-12-16
+
+- Remaining Python 3.5 ops handled
+  (this also means more Python 3.6 ops are handled)
+- Python 3.5 and 3.6 async and await handled
+- Python 3.0 decompilation improved
+- Python 3 annotations fixed
+- Better control-flow detection
+- Code cleanups and misc bug fixes
+
+uncompyle6 2.9.8 2016-12-16
+
+- Better control-flow detection
+- pseudo instruction THEN in 2.x
+  to disambiguate if from and
+- fix bug in --verify option
+- DRY (a little) control-flow detection
+- fix syntax in tuples with one element
+- if AST rule inheritence in Python 2.5
+- NAME_MODULE removal for Python <= 2.4
+- verifycall fixes for Python <= 2.4
+- more Python lint
+
 uncompyle6 2.9.7 2016-12-16
 
 - Start to handle 3.5/3.6 build_map_unpack_with_call

README.rst

@@ -43,10 +43,10 @@ information.
 Requirements
 ------------
 
-This project requires Python 2.6 or later, PyPy 3-2.4, or PyPy-5.0.1. 
-Python versions 2.3-2.7 are supported in the python-2.4 branch.
+This project requires Python 2.6 or later, PyPy 3-2.4, or PyPy-5.0.1.
+Python versions 2.4-2.7 are supported in the python-2.4 branch.
 The bytecode files it can read has been tested on Python bytecodes from
-versions 2.1-2.7, and 3.2-3.6 and the above-mentioned PyPy versions.
+versions 1.5, 2.1-2.7, and 3.2-3.6 and the above-mentioned PyPy versions.
 
 Installation
 ------------
@@ -93,6 +93,16 @@ For usage help:
 
    $ uncompyle6 -h
 
+If you want strong verification of the correctness of the
+decompilation process, add the `--verify` option. But there are
+situations where this will indicate a failure, although the generated
+program is semantically equivalent. Using option `--weak-verify` will
+tell you if there is something definitely wrong. Generally, large
+swaths of code are decompiled correctly, if not the entire program.
+
+You can also cross compare the results with pycdc_ . Since they work
+differently, bugs here often aren't in that, and vice versa.
+
 
 Known Bugs/Restrictions
 -----------------------
@@ -128,10 +138,13 @@ on the lower end Python versions which is more difficult for us to
 handle since we don't have a Python interpreter for versions 1.5, 1.6,
 and 2.0.
 
-Python 3.0 support is weak; Python 3.5 largely works, but still has
-some bugs in it.  Python 3.6 changes things drastically by using word
-codes rather than byte codes. That has been addressed, but then it also
-changes function call opcodes and its semantics.
+In the Python 3 series, Python support is is strongest around 3.4 or
+3.3 and drops off as you move further away from those versions. Python
+3.5 largely works, but still has some bugs in it and is missing some
+opcodes.  Python 3.6 changes things drastically by using word codes
+rather than byte codes. That has been addressed, but then it also
+changes function call opcodes and its semantics and has more problems
+with control flow than 3.5 has.
 
 Currently not all Python magic numbers are supported. Specifically in
 some versions of Python, notably Python 3.6, the magic number has

__pkginfo__.py

@@ -9,7 +9,7 @@
 
 # Things that change more often go here.
 copyright   = """
-Copyright (C) 2015, 2016 Rocky Bernstein <rb@dustyfeet.com>.
+Copyright (C) 2015-2017 Rocky Bernstein <rb@dustyfeet.com>.
 """
 
 classifiers =  ['Development Status :: 5 - Production/Stable',
@@ -39,7 +39,7 @@ entry_points={
         'pydisassemble=uncompyle6.bin.pydisassemble:main',
     ]}
 ftp_url            = None
-install_requires   = ['spark-parser >= 1.5.1, < 1.6.0',
+install_requires   = ['spark-parser >= 1.6.0, < 1.7.0',
                       'xdis >= 3.2.4, < 3.3.0']
 license            = 'MIT'
 mailing_list       = 'python-debugger@googlegroups.com'

circle.yml

@@ -10,4 +10,4 @@ dependencies:
         - pip install -r requirements-dev.txt
 test:
     override:
-        - python ./setup.py develop && make check-2.7
+        - python ./setup.py develop && make check-2.6

pytest/.gitignore

@@ -1 +1,2 @@
+/.hypothesis
 /__pycache__

pytest/test_build_const_key_map.py

@@ -0,0 +1,21 @@
+import pytest
+# uncompyle6
+from uncompyle6 import PYTHON_VERSION
+from validate import validate_uncompyle
+
+
+@pytest.mark.skipif(PYTHON_VERSION < 3.6, reason='need at least python 3.6')
+@pytest.mark.parametrize('text', (
+    "{0.: 'a', -1: 'b'}",   # BUILD_MAP
+    "{'a':'b'}",            # BUILD_MAP
+    "{0: 1}",               # BUILD_MAP
+    "{b'0':1, b'2':3}",     # BUILD_CONST_KEY_MAP
+    "{0: 1, 2: 3}",         # BUILD_CONST_KEY_MAP
+    "{'a':'b','c':'d'}",    # BUILD_CONST_KEY_MAP
+    "{0: 1, 2: 3}",         # BUILD_CONST_KEY_MAP
+    "{'a': 1, 'b': 2}",     # BUILD_CONST_KEY_MAP
+    "{'a':'b','c':'d'}",    # BUILD_CONST_KEY_MAP
+    "{0.0:'b',0.1:'d'}",    # BUILD_CONST_KEY_MAP
+))
+def test_build_const_key_map(text):
+    validate_uncompyle(text)

pytest/test_fstring.py

@@ -1,150 +0,0 @@
-# std
-import os
-# test
-import pytest
-import hypothesis
-from hypothesis import strategies as st
-# uncompyle6
-from uncompyle6 import PYTHON_VERSION, deparse_code
-
-
-@st.composite
-def expressions(draw):
-    # todo : would be nice to generate expressions using hypothesis however
-    # this is pretty involved so for now just use a corpus of expressions
-    # from which to select.
-    return draw(st.sampled_from((
-        'abc',
-        'len(items)',
-        'x + 1',
-        'lineno',
-        'container',
-        'self.attribute',
-        'self.method()',
-        # These expressions are failing, I think these are control
-        # flow problems rather than problems with FORMAT_VALUE,
-        # however I need to confirm this...
-        #'sorted(items, key=lambda x: x.name)',
-        #'func(*args, **kwargs)',
-        #'text or default',
-        #'43 if life_the_universe and everything else None'
-    )))
-
-
-@st.composite
-def format_specifiers(draw):
-    """
-    Generate a valid format specifier using the rules:
-
-    format_spec ::=  [[fill]align][sign][#][0][width][,][.precision][type]
-    fill        ::=  <any character>
-    align       ::=  "<" | ">" | "=" | "^"
-    sign        ::=  "+" | "-" | " "
-    width       ::=  integer
-    precision   ::=  integer
-    type        ::=  "b" | "c" | "d" | "e" | "E" | "f" | "F" | "g" | "G" | "n" | "o" | "s" | "x" | "X" | "%"
-
-    See https://docs.python.org/2/library/string.html
-
-    :param draw: Let hypothesis draw from other strategies.
-
-    :return: An example format_specifier.
-    """
-    alphabet_strategy = st.characters(min_codepoint=ord('a'), max_codepoint=ord('z'))
-    fill = draw(st.one_of(alphabet_strategy, st.none()))
-    align = draw(st.sampled_from(list('<>=^')))
-    fill_align = (fill + align or '') if fill else ''
-
-    type_ = draw(st.sampled_from('bcdeEfFgGnosxX%'))
-    can_have_sign = type_ in 'deEfFgGnoxX%'
-    can_have_comma = type_ in 'deEfFgG%'
-    can_have_precision = type_ in 'fFgG'
-    can_have_pound = type_ in 'boxX%'
-    can_have_zero = type_ in 'oxX'
-
-    sign = draw(st.sampled_from(list('+- ') + [''])) if can_have_sign else ''
-    pound = draw(st.sampled_from(('#', '',))) if can_have_pound else ''
-    zero = draw(st.sampled_from(('0', '',))) if can_have_zero else ''
-
-    int_strategy = st.integers(min_value=1, max_value=1000)
-
-    width = draw(st.one_of(int_strategy, st.none()))
-    width = str(width) if width is not None else ''
-
-    comma = draw(st.sampled_from((',', '',))) if can_have_comma else ''
-    if can_have_precision:
-        precision = draw(st.one_of(int_strategy, st.none()))
-        precision = '.' + str(precision) if precision else ''
-    else:
-        precision = ''
-
-    return ''.join((fill_align, sign, pound, zero, width, comma, precision, type_,))
-
-
-@st.composite
-def fstrings(draw):
-    """
-    Generate a valid f-string.
-    See https://www.python.org/dev/peps/pep-0498/#specification
-
-    :param draw: Let hypothsis draw from other strategies.
-
-    :return: A valid f-string.
-    """
-    character_strategy = st.characters(
-        blacklist_characters='\r\n\'\\s{}',
-        min_codepoint=1,
-        max_codepoint=1000,
-    )
-    is_raw = draw(st.booleans())
-    integer_strategy = st.integers(min_value=0, max_value=3)
-    expression_count = draw(integer_strategy)
-    content = []
-    for _ in range(expression_count):
-        expression = draw(expressions())
-        conversion = draw(st.sampled_from(('', '!s', '!r', '!a',)))
-        has_specifier = draw(st.booleans())
-        specifier = ':' + draw(format_specifiers()) if has_specifier else ''
-        content.append('{{{}{}}}'.format(expression, conversion, specifier))
-        content.append(draw(st.text(character_strategy)))
-    content = ''.join(content)
-    return "f{}'{}'".format('r' if is_raw else '', content)
-
-
-@pytest.mark.skipif(PYTHON_VERSION < 3.6, reason='need at least python 3.6')
-@hypothesis.given(format_specifiers())
-def test_format_specifiers(format_specifier):
-    """Verify that format_specifiers generates valid specifiers"""
-    try:
-        exec('"{:' + format_specifier + '}".format(0)')
-    except ValueError as e:
-        if 'Unknown format code' not in str(e):
-            raise
-
-
-def run_test(text):
-    hypothesis.assume(len(text))
-    hypothesis.assume("f'{" in text)
-    expr = text + '\n'
-    code = compile(expr, '<string>', 'single')
-    deparsed = deparse_code(PYTHON_VERSION, code, compile_mode='single')
-    recompiled = compile(deparsed.text, '<string>', 'single')
-    if recompiled != code:
-        assert 'dis(' + deparsed.text.strip('\n') + ')' == 'dis(' + expr.strip('\n') + ')'
-
-
-@pytest.mark.skipif(PYTHON_VERSION < 3.6, reason='need at least python 3.6')
-@hypothesis.given(fstrings())
-def test_uncompyle_fstring(fstring):
-    """Verify uncompyling fstring bytecode"""
-    run_test(fstring)
-
-
-@pytest.mark.skipif(PYTHON_VERSION < 3.6, reason='need at least python 3.6')
-@pytest.mark.parametrize('fstring', [
-    "f'{abc}{abc!s}'",
-    "f'{abc}0'",
-])
-def test_uncompyle_direct(fstring):
-    """useful for debugging"""
-    run_test(fstring)

pytest/validate.py

@@ -0,0 +1,143 @@
+# future
+from __future__ import print_function
+# std
+import os
+import dis
+import difflib
+import subprocess
+import tempfile
+# compatability
+import six
+# uncompyle6 / xdis
+from uncompyle6 import PYTHON_VERSION, deparse_code
+
+
+def _dis_to_text(co):
+    return dis.Bytecode(co).dis()
+
+
+def print_diff(original, uncompyled):
+    """
+    Try and display a pretty html line difference between the original and
+    uncompyled code and bytecode if elinks and BeautifulSoup are installed
+    otherwise just show the diff.
+
+    :param original: Text describing the original code object.
+    :param uncompyled: Text describing the uncompyled code object.
+    """
+    original_lines = original.split('\n')
+    uncompyled_lines = uncompyled.split('\n')
+    args = original_lines, uncompyled_lines, 'original', 'uncompyled'
+    try:
+        from bs4 import BeautifulSoup
+        diff = difflib.HtmlDiff().make_file(*args)
+        diff = BeautifulSoup(diff, "html.parser")
+        diff.select_one('table[summary="Legends"]').extract()
+    except ImportError:
+        print('\nTo display diff highlighting run:\n    pip install BeautifulSoup4')
+        diff = difflib.HtmlDiff().make_table(*args)
+
+    with tempfile.NamedTemporaryFile(delete=False) as f:
+        f.write(str(diff).encode('utf-8'))
+
+    try:
+        print()
+        html = subprocess.check_output([
+            'elinks',
+            '-dump',
+            '-no-references',
+            '-dump-color-mode',
+            '1',
+            f.name,
+        ]).decode('utf-8')
+        print(html)
+    except:
+        print('\nFor side by side diff install elinks')
+        diff = difflib.Differ().compare(original_lines, uncompyled_lines)
+        print('\n'.join(diff))
+    finally:
+        os.unlink(f.name)
+
+
+def are_instructions_equal(i1, i2):
+    """
+    Determine if two instructions are approximately equal,
+    ignoring certain fields which we allow to differ, namely:
+
+    * code objects are ignore (should probaby be checked) due to address
+    * line numbers
+
+    :param i1: left instruction to compare
+    :param i2: right instruction to compare
+
+    :return: True if the two instructions are approximately equal, otherwise False.
+    """
+    result = (1==1
+        and i1.opname == i2.opname
+        and i1.opcode == i2.opcode
+        and i1.arg == i2.arg
+        # ignore differences due to code objects
+        # TODO : Better way of ignoring address
+        and (i1.argval == i2.argval or '<code object' in str(i1.argval))
+        # TODO : Should probably recurse to check code objects
+        and (i1.argrepr == i2.argrepr or '<code object' in i1.argrepr)
+        and i1.offset == i2.offset
+        # ignore differences in line numbers
+        #and i1.starts_line
+        and i1.is_jump_target == i2.is_jump_target
+    )
+    return result
+
+
+def are_code_objects_equal(co1, co2):
+    """
+    Determine if two code objects are approximately equal,
+    see are_instructions_equal for more information.
+
+    :param i1: left code object to compare
+    :param i2: right code object to compare
+
+    :return: True if the two code objects are approximately equal, otherwise False.
+    """
+    # TODO : Use xdis for python2 compatability
+    instructions1 = dis.Bytecode(co1)
+    instructions2 = dis.Bytecode(co2)
+    for opcode1, opcode2 in zip(instructions1, instructions2):
+        if not are_instructions_equal(opcode1, opcode2):
+            return False
+    return True
+
+
+def validate_uncompyle(text, mode='exec'):
+    """
+    Validate decompilation of the given source code.
+
+    :param text: Source to validate decompilation of.
+    """
+    original_code = compile(text, '<string>', mode)
+    original_dis = _dis_to_text(original_code)
+    original_text = text
+
+    deparsed = deparse_code(PYTHON_VERSION, original_code,
+                            compile_mode=mode, out=six.StringIO())
+    uncompyled_text = deparsed.text
+    uncompyled_code = compile(uncompyled_text, '<string>', 'exec')
+
+    if not are_code_objects_equal(uncompyled_code, original_code):
+
+        uncompyled_dis = _dis_to_text(uncompyled_text)
+
+        def output(text, dis):
+            width = 60
+            return '\n\n'.join([
+                ' SOURCE CODE '.center(width, '#'),
+                text.strip(),
+                ' BYTECODE '.center(width, '#'),
+                dis
+            ])
+
+        original = output(original_text, original_dis)
+        uncompyled = output(uncompyled_text, uncompyled_dis)
+        print_diff(original, uncompyled)
+
+        assert 'original' == 'uncompyled'

requirements-dev.txt

@@ -1,3 +1,4 @@
 pytest
 flake8
-hypothesis
+hypothesis
+six

test/Makefile

@@ -20,7 +20,7 @@ check:
 	$(MAKE) check-$$PYTHON_VERSION
 
 #: Run working tests from Python 2.6 or 2.7
-check-2.6 check-2.7: check-bytecode-2 check-bytecode-3 check-bytecode-1  check-2.7-ok
+check-2.4 check-2.5 check-2.6 check-2.7: check-bytecode-2 check-bytecode-3 check-bytecode-1  check-2.7-ok
 
 #: Run working tests from Python 3.0
 check-3.0: check-bytecode
@@ -98,14 +98,6 @@ check-bytecode-2.4:
 check-bytecode-2.5:
 	$(PYTHON) test_pythonlib.py --bytecode-2.5
 
-#: Check deparsing Python 2.6
-check-bytecode-2.6:
-	$(PYTHON) test_pythonlib.py --bytecode-2.6
-
-#: Check deparsing Python 2.7
-check-bytecode-2.7:
-	$(PYTHON) test_pythonlib.py --bytecode-2.7 --verify
-
 #: Check deparsing Python 3.0
 check-bytecode-3.0:
 	$(PYTHON) test_pythonlib.py --bytecode-3.0
@@ -134,6 +126,26 @@ check-bytecode-3.5:
 check-bytecode-3.6:
 	$(PYTHON) test_pythonlib.py --bytecode-3.6
 
+#: Get grammar coverage for Python 2.4
+grammar-coverage-2.4:
+	SPARK_PARSER_COVERAGE=/tmp/spark-grammar-24.cover $(PYTHON) test_pythonlib.py --bytecode-2.4
+	SPARK_PARSER_COVERAGE=/tmp/spark-grammar-24.cover $(PYTHON) test_pyenvlib.py --2.4.6
+
+#: Get grammar coverage for Python 2.5
+grammar-coverage-2.5:
+	SPARK_PARSER_COVERAGE=/tmp/spark-grammar-25.cover $(PYTHON) test_pythonlib.py --bytecode-2.5
+	SPARK_PARSER_COVERAGE=/tmp/spark-grammar-25.cover $(PYTHON) test_pyenvlib.py --2.5.6
+
+#: Get grammar coverage for Python 2.6
+grammar-coverage-2.6:
+	SPARK_PARSER_COVERAGE=/tmp/spark-grammar-26.cover $(PYTHON) test_pythonlib.py --bytecode-2.6
+	SPARK_PARSER_COVERAGE=/tmp/spark-grammar-26.cover $(PYTHON) test_pyenvlib.py --2.6.9
+
+#: Get grammar coverage for Python 2.7
+grammar-coverage-2.7:
+	SPARK_PARSER_COVERAGE=/tmp/spark-grammar-27.cover $(PYTHON) test_pythonlib.py --bytecode-2.7
+	SPARK_PARSER_COVERAGE=/tmp/spark-grammar-27.cover $(PYTHON) test_pyenvlib.py --2.7.13
+
 #: short tests for bytecodes only for this version of Python
 check-native-short:
 	$(PYTHON) test_pythonlib.py --bytecode-$(PYTHON_VERSION) --verify $(COMPILE)

test/simple_source/bug22/01_ops.py

@@ -0,0 +1,18 @@
+# Statements to beef up grammar coverage rules
+# Force "inplace" ops
+y = +10  # UNARY_POSITIVE
+y /= 1   # INPLACE_DIVIDE
+y %= 4   # INPLACE_MODULO
+y **= 1  # INPLACE POWER
+y >>= 2  # INPLACE_RSHIFT
+y <<= 2  # INPLACE_LSHIFT
+y //= 1  # INPLACE_TRUE_DIVIDE
+y &= 1   # INPLACE_AND
+y ^= 1   # INPLACE_XOR
+
+`y`      # UNARY_CONVERT  - No in Python 3.x
+
+# Beef up augassign and STORE_SLICE+3
+x = [1,2,3,4,5]
+x[0:1] = 1
+x[0:3] += 1, 2, 3

test/simple_source/bug25/02_try_else.py

@@ -0,0 +1,27 @@
+# Python 2.5 bug
+# Was turning into tryelse when there in fact is no else.
+def options(self, section):
+    try:
+        opts = self._sections[section].copy()
+    except KeyError:
+        raise NoSectionError(section)
+    opts.update(self._defaults)
+    if '__name__' in opts:
+        del opts['__name__']
+    return opts.keys()
+
+# From python2.5/distutils/command/register.py
+def post_to_server(self, urllib2):
+    try:
+        result = 5
+    except urllib2.HTTPError, e:
+        result = e.code, e.msg
+    except urllib2.URLError, e:
+        result = 500
+    else:
+        if self.show_response:
+            result = 10
+        result = 200
+    if self.show_response:
+        result = 8
+    return result

test/simple_source/bug25/03_if_for.py

@@ -0,0 +1,7 @@
+# From Python 2.6. distutils/sysconfig.py
+def get_config_vars(_config_vars, args):
+    if _config_vars:
+        if args == 1:
+            if args < 8:
+                for key in ('LDFLAGS', 'BASECFLAGS'):
+                    _config_vars[key] = 4

test/simple_source/bug31/04_def_annotate.py

@@ -8,3 +8,12 @@ def open(file, mode = "r", buffering = None,
          encoding = None, errors = None,
          newline = None, closefd = True) -> "IOBase":
     return text
+
+def foo(x: 'an argument that defaults to 5' = 5):
+    print(x)
+
+def div(a: dict(type=float, help='the dividend'),
+        b: dict(type=float, help='the divisor (must be different than 0)')
+    ) -> dict(type=float, help='the result of dividing a by b'):
+    """Divide a by b"""
+    return a / b

test/simple_source/bug33/02_while1.py

@@ -0,0 +1,14 @@
+# From Python 3.4 mailcap
+def readmailcapfile(caps):
+    while 1:
+        line = 'abc'
+        if line[0] == '#' or line == '':
+            continue
+        key, fields = (1,2)
+        if not (key and fields):
+            continue
+        if key in caps:
+            caps[key].append(fields)
+        else:
+            caps[key] = [fields]
+    return caps

test/simple_source/bug35/03_async_await.py

@@ -0,0 +1,26 @@
+# Python 3.5+ async and await
+async def await_test(asyncio):
+    reader, writer = await asyncio.open_connection(80)
+    await bar()
+
+async def afor_test():
+
+    async for i in [1,2,3]:
+        x = i
+
+
+async def afor_else_test():
+
+    async for i in [1,2,3]:
+        x = i
+    else:
+        z = 4
+
+
+async def awith_test():
+    async with i:
+        print(i)
+
+async def awith_as_test():
+    async with 1 as i:
+        print(i)

test/simple_source/bug35/03_double_star_unpack.py

@@ -0,0 +1,9 @@
+# Bug in Python 3.5 is getting the two star'd arguments right.
+def sum(a,b,c,d):
+    return a + b + c + d
+
+args=(1,2)
+sum(*args, *args)
+
+# FIXME: this is handled incorrectly
+# (*c,) = (3,4)

test/simple_source/bug35/03_while-if-break.py

@@ -0,0 +1,7 @@
+# Python 3.5 and 3.6 break inside a
+# while True and if / break
+def display_date(loop):
+    while True:
+        if loop.time():
+            break
+        x = 5

test/simple_source/exception/02_try_finally.py

@@ -0,0 +1,11 @@
+# From 2.6.9 cmd.py
+try:
+    if __file__:
+        x = 2
+    x = 3
+finally:
+    if x and __file__:
+        try:
+            x = 1
+        except:
+            pass

test/simple_source/expression/03_tuple_assign.py

@@ -5,3 +5,8 @@ def some_function():
 def some_other_function():
     some_variable, = some_function()
     print(some_variable)
+
+empty_tup = ()
+one_item_tup = ("item1", )
+one_item_tup_without_parentheses = "item", 
+many_items_tup = ("item1", "item2", "item3")

test/test_pyenvlib.py

@@ -19,8 +19,6 @@ Step 2: Run the test:
 	  test_pyenvlib --mylib --verify # decompile verify 'mylib'
 """
 
-from __future__ import print_function
-
 from uncompyle6 import main, PYTHON3
 import os, time, shutil
 from fnmatch import fnmatch
@@ -30,10 +28,10 @@ from fnmatch import fnmatch
 TEST_VERSIONS=('2.3.7', '2.4.6', '2.5.6', '2.6.9',
                'pypy-2.4.0', 'pypy-2.6.1',
                'pypy-5.0.1', 'pypy-5.3.1',
-               '2.7.10', '2.7.11', '2.7.12',
+               '2.7.10', '2.7.11', '2.7.12', '2.7.13',
                '3.0.1', '3.1.5', '3.2.6',
                '3.3.5', '3.3.6',
-               '3.4.2', '3.5.1')
+               '3.4.2', '3.5.1', '3.6.0')
 
 target_base = '/tmp/py-dis/'
 lib_prefix = os.path.join(os.environ['HOME'], '.pyenv/versions')
@@ -106,28 +104,40 @@ def do_tests(src_dir, patterns, target_dir, start_with=None, do_verify=False):
 if __name__ == '__main__':
     import getopt, sys
 
-    do_verify = False
+    do_coverage = do_verify = False
     test_dirs = []
     start_with = None
 
     test_options_keys = list(test_options.keys())
     test_options_keys.sort()
     opts, args = getopt.getopt(sys.argv[1:], '',
-                               ['start-with=', 'verify', 'weak-verify', 'all', ] \
+                               ['start-with=', 'verify', 'weak-verify',
+                                'coverage', 'all', ] \
                                + test_options_keys )
+    vers = ''
     for opt, val in opts:
         if opt == '--verify':
             do_verify = True
         if opt == '--weak-verify':
             do_verify = 'weak'
+        if opt == '--coverage':
+            do_coverage = True
         elif opt == '--start-with':
             start_with = val
         elif opt[2:] in test_options_keys:
-            test_dirs.append(test_options[opt[2:]])
+            triple = test_options[opt[2:]]
+            vers = triple[-1]
+            test_dirs.append(triple)
         elif opt == '--all':
+            vers = 'all'
             for val in test_options_keys:
                 test_dirs.append(test_options[val])
 
+    if do_coverage:
+        os.environ['SPARK_PARSER_COVERAGE'] = (
+            '/tmp/spark-grammar-%s.cover' % vers
+            )
+
     for src_dir, pattern, target_dir in test_dirs:
         if os.path.exists(src_dir):
             target_dir = os.path.join(target_base, target_dir)

test/test_pythonlib.py

@@ -27,8 +27,6 @@ Step 2: Run the test:
   test_pythonlib.py --mylib --verify # decompile verify 'mylib'
 """
 
-from __future__ import print_function
-
 import getopt, os, py_compile, sys, shutil, tempfile, time
 
 from uncompyle6 import PYTHON_VERSION
@@ -127,8 +125,10 @@ def do_tests(src_dir, obj_patterns, target_dir, opts):
     if opts['do_compile']:
         compiled_version = opts['compiled_version']
         if compiled_version and PYTHON_VERSION != compiled_version:
-            print("Not compiling: desired Python version is %s but we are running %s" %
-                  (compiled_version, PYTHON_VERSION), file=sys.stderr)
+            sys.stderr.write("Not compiling: "
+                             "desired Python version is %s "
+                             "but we are running %s" %
+                             (compiled_version, PYTHON_VERSION))
         else:
             for root, dirs, basenames in os.walk(src_dir):
                 file_matches(files, root, basenames, PY)
@@ -146,8 +146,8 @@ def do_tests(src_dir, obj_patterns, target_dir, opts):
         file_matches(files, dirname, basenames, obj_patterns)
 
     if not files:
-        print("Didn't come up with any files to test! Try with --compile?",
-              file=sys.stderr)
+        sys.stderr.write("Didn't come up with any files to test! "
+                         "Try with --compile?")
         exit(1)
 
     os.chdir(cwd)
@@ -161,9 +161,9 @@ def do_tests(src_dir, obj_patterns, target_dir, opts):
         except ValueError:
             pass
 
-    print(time.ctime())
-    print('Source directory: ', src_dir)
-    print('Output directory: ', target_dir)
+    print time.ctime()
+    print 'Source directory: ', src_dir
+    print 'Output directory: ', target_dir
     try:
         _, _, failed_files, failed_verify = \
           main(src_dir, target_dir, files, [],
@@ -190,6 +190,7 @@ if __name__ == '__main__':
     test_options_keys.sort()
     opts, args = getopt.getopt(sys.argv[1:], '',
                                ['start-with=', 'verify', 'weak-verify', 'all', 'compile',
+                                'coverage',
                                 'no-rm'] \
                                + test_options_keys )
     if not opts: help()
@@ -198,7 +199,8 @@ if __name__ == '__main__':
         'do_compile': False,
         'do_verify': False,
         'start_with': None,
-        'rmtree' : True
+        'rmtree' : True,
+        'coverage' : False
         }
 
     for opt, val in opts:
@@ -217,24 +219,30 @@ if __name__ == '__main__':
         elif opt == '--all':
             for val in test_options_keys:
                 test_dirs.append(test_options[val])
+        elif opt == '--coverage':
+            test_opts['coverage'] = True
         else:
             help()
             pass
         pass
 
+    if test_opts['coverage']:
+        os.environ['SPARK_PARSER_COVERAGE'] = (
+            '/tmp/spark-grammar-python-lib%s.cover' % test_dirs[0][-1]
+            )
+
     last_compile_version = None
     for src_dir, pattern, target_dir, compiled_version in test_dirs:
         if os.path.isdir(src_dir):
             checked_dirs.append([src_dir, pattern, target_dir])
         else:
-            print("Can't find directory %s. Skipping" % src_dir,
-                  file=sys.stderr)
+            sys.stderr.write("Can't find directory %s. Skipping" % src_dir)
             continue
         last_compile_version = compiled_version
         pass
 
     if not checked_dirs:
-        print("No directories found to check", file=sys.stderr)
+        sys.stderr.write("No directories found to check\n")
         sys.exit(1)
 
     test_opts['compiled_version'] = last_compile_version

uncompyle6/__init__.py

@@ -41,13 +41,18 @@ PYTHON_VERSION_STR  = "%s.%s" % (sys.version_info[0], sys.version_info[1])
 
 IS_PYPY = '__pypy__' in sys.builtin_module_names
 
-sys.setrecursionlimit(5000)
+if hasattr(sys, 'setrecursionlimit'):
+    # pyston doesn't have setrecursionlimit
+    sys.setrecursionlimit(5000)
 
 import uncompyle6.semantics.pysource
 import uncompyle6.semantics.fragments
 
 # Export some functions
-from uncompyle6.main import uncompyle_file
+from uncompyle6.main import decompile_file
+
+# For compaitility
+uncompyle_file = decompile_file
 
 # Conventience functions so you can say:
 # from uncompyle6 import deparse_code

uncompyle6/bin/pydisassemble.py

@@ -3,7 +3,6 @@
 #
 # Copyright (c) 2015-2016 by Rocky Bernstein <rb@dustyfeet.com>
 #
-from __future__ import print_function
 import sys, os, getopt
 
 from uncompyle6.disas import disassemble_file
@@ -26,7 +25,7 @@ Options:
   -V | --version     show version and stop
   -h | --help        show this message
 
-""".format(program)
+""" % (program, program)
 
 PATTERNS = ('*.pyc', '*.pyo')
 
@@ -37,15 +36,15 @@ Type -h for for full help.""" % program
     native = True
 
     if len(sys.argv) == 1:
-        print("No file(s) given", file=sys.stderr)
-        print(Usage_short, file=sys.stderr)
+        sys.stderr.write("No file(s) given\n")
+        sys.stderr.write(Usage_short)
         sys.exit(1)
 
     try:
         opts, files = getopt.getopt(sys.argv[1:], 'hVU',
                                     ['help', 'version', 'uncompyle6'])
-    except getopt.GetoptError as e:
-        print('%s: %s' % (os.path.basename(sys.argv[0]), e),  file=sys.stderr)
+    except getopt.GetoptError(e):
+        sys.stderr.write('%s: %s' % (os.path.basename(sys.argv[0]), e))
         sys.exit(-1)
 
     for opt, val in opts:
@@ -59,16 +58,14 @@ Type -h for for full help.""" % program
             native = False
         else:
             print(opt)
-            print(Usage_short, file=sys.stderr)
+            sys.stderr.write(Usage_short)
             sys.exit(1)
 
-
     for file in files:
         if os.path.exists(files[0]):
             disassemble_file(file, sys.stdout, native)
         else:
-            print("Can't read %s - skipping" % files[0],
-                  file=sys.stderr)
+            sys.stderr.write("Can't read %s - skipping\n" % files[0])
             pass
         pass
     return

uncompyle6/bin/uncompile.py

@@ -4,7 +4,6 @@
 # Copyright (c) 2015-2016 by Rocky Bernstein
 # Copyright (c) 2000-2002 by hartmut Goebel <h.goebel@crazy-compilers.com>
 #
-from __future__ import print_function
 import sys, os, getopt, time
 
 program, ext = os.path.splitext(os.path.basename(__file__))
@@ -65,11 +64,11 @@ def usage():
 
 
 def main_bin():
-    if not (sys.version_info[0:2] in ((2, 6), (2, 7),
-                                      (3, 1), (3, 2), (3, 3),
+    if not (sys.version_info[0:2] in ((2, 4), (2, 5), (2, 6), (2, 7),
+                                      (3, 2), (3, 3),
                                       (3, 4), (3, 5), (3, 6))):
-        print('Error: %s requires Python 2.6-2.7, or 3.1-3.6' % program,
-              file=sys.stderr)
+        sys.stderr.write('Error: %s requires Python 2.4 2.5 2.6, 2.7, '
+                         '3.2, 3.3, 3.4, 3.5, or 3.6' % program)
         sys.exit(-1)
 
     do_verify = recurse_dirs = False
@@ -84,8 +83,8 @@ def main_bin():
         opts, files = getopt.getopt(sys.argv[1:], 'hagtdrVo:c:p:',
                                     'help asm grammar linemaps recurse timestamp tree '
                                     'verify version showgrammar'.split(' '))
-    except getopt.GetoptError as e:
-        print('%s: %s' % (os.path.basename(sys.argv[0]), e),  file=sys.stderr)
+    except getopt.GetoptError(e):
+        sys.stderr.write('%s: %s\n' % (os.path.basename(sys.argv[0]), e))
         sys.exit(-1)
 
     options = {}
@@ -119,7 +118,7 @@ def main_bin():
         elif opt in ('--recurse', '-r'):
             recurse_dirs = True
         else:
-            print(opt, file=sys.stderr)
+            sys.stderr.write(opt)
             usage()
 
     # expand directory if specified
@@ -144,7 +143,7 @@ def main_bin():
         files = [f[sb_len:] for f in files]
 
     if not files:
-        print("No files given", file=sys.stderr)
+        sys.stderr.write("No files given\n")
         usage()
 
     if outfile == '-':
@@ -224,7 +223,6 @@ def main_bin():
         except (KeyboardInterrupt, OSError):
             pass
 
-
     if timestamp:
         print(time.strftime(timestampfmt))
 

uncompyle6/disas.py

@@ -16,8 +16,6 @@ Second, we need structured instruction information for the
 want to run on Python 2.7.
 """
 
-from __future__ import print_function
-
 import sys
 from collections import deque
 
@@ -37,10 +35,9 @@ def disco(version, co, out=None, is_pypy=False):
 
     # store final output stream for case of error
     real_out = out or sys.stdout
-    print('# Python %s' % version, file=real_out)
+    real_out.write('# Python %s\n' % version)
     if co.co_filename:
-        print('# Embedded file name: %s' % co.co_filename,
-              file=real_out)
+        real_out.write('# Embedded file name: %s\n' % co.co_filename)
 
     scanner = get_scanner(version, is_pypy=is_pypy)
 
@@ -52,16 +49,15 @@ def disco_loop(disasm, queue, real_out):
     while len(queue) > 0:
         co = queue.popleft()
         if co.co_name != '<module>':
-            print('\n# %s line %d of %s' %
-                      (co.co_name, co.co_firstlineno, co.co_filename),
-                      file=real_out)
+            real_out.write('\n# %s line %d of %s\n' %
+                      (co.co_name, co.co_firstlineno, co.co_filename))
         tokens, customize = disasm(co)
         for t in tokens:
             if iscode(t.pattr):
                 queue.append(t.pattr)
             elif iscode(t.attr):
                 queue.append(t.attr)
-            print(t, file=real_out)
+            real_out.write(t)
             pass
         pass
 

uncompyle6/linenumbers.py

@@ -10,7 +10,7 @@ def line_number_mapping(pyc_filename, src_filename):
      source_size) = load_module(pyc_filename)
     try:
         code2 = load_file(src_filename)
-    except SyntaxError as e:
+    except SyntaxError, e:
         return str(e)
 
     queue = deque([code1, code2])

uncompyle6/main.py

@@ -1,4 +1,3 @@
-from __future__ import print_function
 import datetime, os, subprocess, sys, tempfile
 
 from uncompyle6 import verify, IS_PYPY
@@ -11,7 +10,7 @@ from uncompyle6.linenumbers import line_number_mapping
 
 from xdis.load import load_module
 
-def uncompyle(
+def decompile(
         bytecode_version, co, out=None, showasm=None, showast=False,
         timestamp=None, showgrammar=False, code_objects={},
         source_size=None, is_pypy=False, magic_int=None):
@@ -22,34 +21,41 @@ def uncompyle(
 
     # store final output stream for case of error
     real_out = out or sys.stdout
-    co_pypy_str = 'PyPy ' if is_pypy else ''
-    run_pypy_str = 'PyPy ' if IS_PYPY else ''
-    print('# uncompyle6 version %s\n'
-          '# %sPython bytecode %s%s\n# Decompiled from: %sPython %s' %
+    if is_pypy:
+        co_pypy_str = 'PyPy '
+    else:
+        co_pypy_str = ''
+
+    if IS_PYPY:
+        run_pypy_str = 'PyPy '
+    else:
+        run_pypy_str = ''
+
+    if magic_int:
+        m = str(magic_int)
+    else:
+        m = ""
+    real_out.write('# uncompyle6 version %s\n'
+          '# %sPython bytecode %s%s\n# Decompiled from: %sPython %s\n' %
           (VERSION, co_pypy_str, bytecode_version,
-           " (%d)" % magic_int if magic_int else "",
-           run_pypy_str, '\n# '.join(sys.version.split('\n'))),
-           file=real_out)
+           " (%s)" % m, run_pypy_str,
+           '\n# '.join(sys.version.split('\n'))))
     if co.co_filename:
-        print('# Embedded file name: %s' % co.co_filename,
-              file=real_out)
+        real_out.write('# Embedded file name: %s\n' % co.co_filename)
     if timestamp:
-        print('# Compiled at: %s' % datetime.datetime.fromtimestamp(timestamp),
-              file=real_out)
+        real_out.write('# Compiled at: %s\n' %
+                       datetime.datetime.fromtimestamp(timestamp))
     if source_size:
-        print('# Size of source mod 2**32: %d bytes' % source_size,
-               file=real_out)
+        real_out.write('# Size of source mod 2**32: %d bytes\n' % source_size)
 
-    try:
-        pysource.deparse_code(bytecode_version, co, out, showasm, showast,
-                              showgrammar, code_objects=code_objects,
-                              is_pypy=is_pypy)
-    except pysource.SourceWalkerError as e:
-        # deparsing failed
-        raise pysource.SourceWalkerError(str(e))
+    pysource.deparse_code(bytecode_version, co, out, showasm, showast,
+                          showgrammar, code_objects=code_objects,
+                          is_pypy=is_pypy)
 
+# For compatiblity
+uncompyle = decompile
 
-def uncompyle_file(filename, outstream=None, showasm=None, showast=False,
+def decompile_file(filename, outstream=None, showasm=None, showast=False,
                    showgrammar=False):
     """
     decompile Python byte-code file (.pyc)
@@ -62,16 +68,20 @@ def uncompyle_file(filename, outstream=None, showasm=None, showast=False,
 
     if type(co) == list:
         for con in co:
-            uncompyle(version, con, outstream, showasm, showast,
+            decompile(version, con, outstream, showasm, showast,
                       timestamp, showgrammar, code_objects=code_objects,
                       is_pypy=is_pypy, magic_int=magic_int)
     else:
-        uncompyle(version, co, outstream, showasm, showast,
+        decompile(version, co, outstream, showasm, showast,
                   timestamp, showgrammar,
                   code_objects=code_objects, source_size=source_size,
                   is_pypy=is_pypy, magic_int=magic_int)
     co = None
 
+# For compatiblity
+uncompyle_file = decompile_file
+
+
 # FIXME: combine into an options parameter
 def main(in_base, out_base, files, codes, outfile=None,
          showasm=None, showast=False, do_verify=False,
@@ -101,12 +111,6 @@ def main(in_base, out_base, files, codes, outfile=None,
 
     tot_files = okay_files = failed_files = verify_failed_files = 0
 
-    # for code in codes:
-    #    version = sys.version[:3] # "2.5"
-    #    with open(code, "r") as f:
-    #        co = compile(f.read(), "", "exec")
-    #    uncompyle(sys.version[:3], co, sys.stdout, showasm=showasm, showast=showast)
-
     for filename in files:
         infile = os.path.join(in_base, filename)
         if not os.path.exists(infile):
@@ -126,8 +130,12 @@ def main(in_base, out_base, files, codes, outfile=None,
                     prefix = prefix[:-len('.py')]
                 junk, outfile = tempfile.mkstemp(suffix=".py",
                                              prefix=prefix)
-                # Unbuffer output
-                sys.stdout = os.fdopen(sys.stdout.fileno(), 'w', 0)
+                # Unbuffer output if possible
+                if sys.stdout.isatty():
+                    buffering = -1
+                else:
+                    buffering = 0
+                sys.stdout = os.fdopen(sys.stdout.fileno(), 'w', buffering)
                 tee = subprocess.Popen(["tee", outfile], stdin=subprocess.PIPE)
                 os.dup2(tee.stdin.fileno(), sys.stdout.fileno())
                 os.dup2(tee.stdin.fileno(), sys.stderr.fileno())
@@ -141,11 +149,11 @@ def main(in_base, out_base, files, codes, outfile=None,
 
         # Try to uncompile the input file
         try:
-            uncompyle_file(infile, outstream, showasm, showast, showgrammar)
+            decompile_file(infile, outstream, showasm, showast, showgrammar)
             tot_files += 1
-        except (ValueError, SyntaxError, ParserError, pysource.SourceWalkerError) as e:
+        except (ValueError, SyntaxError, ParserError, pysource.SourceWalkerError):
             sys.stdout.write("\n")
-            sys.stderr.write("\n# file %s\n# %s\n" % (infile, e))
+            sys.stderr.write("# file %s\n" % (infile))
             failed_files += 1
         except KeyboardInterrupt:
             if outfile:
@@ -179,31 +187,35 @@ def main(in_base, out_base, files, codes, outfile=None,
                         msg = verify.compare_code_with_srcfile(infile, outfile, weak_verify=weak_verify)
                         if not outfile:
                             if not msg:
-                                print('\n# okay decompiling %s' % infile)
+                                print '\n# okay decompiling %s' % infile
                                 okay_files += 1
                             else:
-                                print('\n# %s\n\t%s', infile, msg)
-                    except verify.VerifyCmpError as e:
+                                print '\n# %s\n\t%s', infile, msg
+                    except verify.VerifyCmpError, e:
                         print(e)
                         verify_failed_files += 1
                         os.rename(outfile, outfile + '_unverified')
                         sys.stderr.write("### Error Verifying %s\n" % filename)
                         sys.stderr.write(str(e) + "\n")
                         if not outfile:
+                            sys.stder.write("### Error Verifiying %s" %
+                                            filename)
+                            sys.stderr.write(e)
                             if raise_on_error:
                                 raise
                             pass
                         pass
                 pass
             elif do_verify:
-                sys.stderr.write("\n### uncompile successful, but no file to compare against\n")
+                sys.stderr.write("\n### uncompile successful, "
+                                 "but no file to compare against")
                 pass
             else:
                 okay_files += 1
                 if not outfile:
                     mess = '\n# okay decompiling'
                     # mem_usage = __memUsage()
-                    print(mess, infile)
+                    print mess, infile
         if outfile:
             sys.stdout.write("%s\r" %
                              status_msg(do_verify, tot_files, okay_files, failed_files, verify_failed_files))
@@ -230,11 +242,11 @@ def status_msg(do_verify, tot_files, okay_files, failed_files,
                verify_failed_files):
     if tot_files == 1:
         if failed_files:
-            return "decompile failed"
+            return "\n# decompile failed"
         elif verify_failed_files:
-            return "decompile verify failed"
+            return "\n# decompile verify failed"
         else:
-            return "Successfully decompiled file"
+            return "\n# Successfully decompiled file"
             pass
         pass
     mess = "decompiled %i files: %i okay, %i failed" % (tot_files, okay_files, failed_files)

uncompyle6/parser.py

@@ -1,4 +1,4 @@
-#  Copyright (c) 2015-2016 Rocky Bernstein
+#  Copyright (c) 2015-2017 Rocky Bernstein
 #  Copyright (c) 2005 by Dan Pascu <dan@windowmaker.org>
 #  Copyright (c) 2000-2002 by hartmut Goebel <h.goebel@crazy-compilers.com>
 #  Copyright (c) 1999 John Aycock
@@ -6,8 +6,6 @@
 Common uncompyle parser routines.
 """
 
-from __future__ import print_function
-
 import sys
 
 from xdis.code import iscode
@@ -28,6 +26,16 @@ nop_func = lambda self, args: None
 
 class PythonParser(GenericASTBuilder):
 
+    def __init__(self, AST, start, debug):
+        super(PythonParser, self).__init__(AST, start, debug)
+        self.collect = [
+            'stmts', 'except_stmts', '_stmts',
+            'exprlist', 'kvlist', 'kwargs', 'come_froms',
+            # Python < 3
+            'print_items',
+            # PyPy:
+            'kvlist_n']
+
     def add_unique_rule(self, rule, opname, count, customize):
         """Add rule to grammar, but only if it hasn't been added previously
            opname and count are used in the customize() semantic the actions
@@ -76,7 +84,10 @@ class PythonParser(GenericASTBuilder):
         def fix(c):
             s = str(c)
             i = s.find('_')
-            return s if i == -1 else s[:i]
+            if i == -1:
+                return s
+            else:
+                return s[:i]
 
         prefix = ''
         if parent and tokens:
@@ -107,7 +118,10 @@ class PythonParser(GenericASTBuilder):
         err_token = instructions[index]
         print("Instruction context:")
         for i in range(start, finish):
-            indent = '   ' if i != index else '-> '
+            if i != index:
+                indent = '   '
+            else:
+                indent = '-> '
             print("%s%s" % (indent, instructions[i]))
         raise ParserError(err_token, err_token.offset)
 
@@ -115,11 +129,7 @@ class PythonParser(GenericASTBuilder):
         return token.type
 
     def nonterminal(self, nt, args):
-        collect = ('stmts', 'exprlist', 'kvlist', '_stmts', 'print_items', 'kwargs',
-                   # PYPY:
-                   'kvlist_n')
-
-        if nt in collect and len(args) > 1:
+        if nt in self.collect and len(args) > 1:
             #
             #  Collect iterated thingies together. That is rather than
             #  stmts -> stmts stmt -> stmts stmt -> ...

uncompyle6/parsers/astnode.py

@@ -28,9 +28,9 @@ class AST(spark_AST):
         i = 0
         for node in self:
             if hasattr(node, '__repr1__'):
-               if enumerate_children:
+                if enumerate_children:
                     child =  node.__repr1__(indent, i)
-               else:
+                else:
                     child = node.__repr1__(indent, None)
             else:
                 inst = node.format(line_prefix='L.')

uncompyle6/parsers/parse2.py

@@ -12,8 +12,6 @@ If we succeed in creating a parse tree, then we have a Python program
 that a later phase can turn into a sequence of ASCII text.
 """
 
-from __future__ import print_function
-
 from uncompyle6.parser import PythonParser, PythonParserSingle, nop_func
 from uncompyle6.parsers.astnode import AST
 from spark_parser import DEFAULT_DEBUG as PARSER_DEFAULT_DEBUG
@@ -126,6 +124,7 @@ class Python2Parser(PythonParser):
         assert_expr_and ::= assert_expr jmp_false expr
 
         ifstmt ::= testexpr _ifstmts_jump
+        ifstmt ::= testexpr return_if_stmts COME_FROM
 
         testexpr ::= testfalse
         testexpr ::= testtrue
@@ -144,6 +143,8 @@ class Python2Parser(PythonParser):
 
         ifelsestmtr ::= testexpr return_if_stmts return_stmts
 
+        ifelsestmtr ::= testexpr return_if_stmts COME_FROM return_stmts
+
         ifelsestmtl ::= testexpr c_stmts_opt JUMP_BACK else_suitel
 
 
@@ -272,7 +273,7 @@ class Python2Parser(PythonParser):
                 continue
             elif opname_base in ('BUILD_LIST', 'BUILD_TUPLE', 'BUILD_SET'):
                 thousands = (v//1024)
-                thirty32s = ((v//32)%32)
+                thirty32s = ((v//32) % 32)
                 if thirty32s > 0:
                     rule = "expr32 ::=%s" % (' expr' * 32)
                     self.add_unique_rule(rule, opname_base, v, customize)
@@ -282,7 +283,7 @@ class Python2Parser(PythonParser):
                                          opname_base, v, customize)
                     self.seen1024 = True
                 rule = ('build_list ::= ' + 'expr1024 '*thousands +
-                                        'expr32 '*thirty32s + 'expr '*(v%32) + opname)
+                                        'expr32 '*thirty32s + 'expr '*(v % 32) + opname)
             elif opname == 'LOOKUP_METHOD':
                 # A PyPy speciality - DRY with parse3
                 self.add_unique_rule("load_attr ::= expr LOOKUP_METHOD",

uncompyle6/parsers/parse25.py

@@ -1,4 +1,4 @@
-#  Copyright (c) 2016 Rocky Bernstein
+#  Copyright (c) 2016-2017 Rocky Bernstein
 """
 spark grammar differences over Python2.6 for Python 2.5.
 """
@@ -13,27 +13,46 @@ class Python25Parser(Python26Parser):
         self.customized = {}
 
     def p_misc25(self, args):
-        '''
+        """
         # If "return_if_stmt" is in a loop, a JUMP_BACK can be emitted. In 2.6 the
         # JUMP_BACK doesn't appear
 
         return_if_stmt ::= ret_expr  RETURN_END_IF JUMP_BACK
 
         # Python 2.6 uses ROT_TWO instead of the STORE_xxx
+        # withas is allowed as a "from future" in 2.5
         setupwithas ::= DUP_TOP LOAD_ATTR store LOAD_ATTR CALL_FUNCTION_0
                         setup_finally
 
         store ::= STORE_FAST
         store ::= STORE_NAME
 
-        # Python 2.6 omits ths LOAD_FAST DELETE_FAST below
+        tryelsestmt    ::= SETUP_EXCEPT suite_stmts_opt POP_BLOCK
+                           try_middle else_suite COME_FROM
+
+        # Python 2.6 omits the LOAD_FAST DELETE_FAST below
+        # withas is allowed as a "from future" in 2.5
         withasstmt ::= expr setupwithas designator suite_stmts_opt
                        POP_BLOCK LOAD_CONST COME_FROM
                        with_cleanup
 
         with_cleanup ::= LOAD_FAST DELETE_FAST WITH_CLEANUP END_FINALLY
         with_cleanup ::= LOAD_NAME DELETE_NAME WITH_CLEANUP END_FINALLY
-        '''
+        """
+
+    def add_custom_rules(self, tokens, customize):
+        super(Python25Parser, self).add_custom_rules(tokens, customize)
+        if self.version == 2.5:
+            self.check_reduce['tryelsestmt'] = 'tokens'
+
+    def reduce_is_invalid(self, rule, ast, tokens, first, last):
+        invalid = super(Python25Parser,
+                        self).reduce_is_invalid(rule, ast,
+                                                tokens, first, last)
+        if invalid:
+            return invalid
+        return False
+
 
 class Python25ParserSingle(Python26Parser, PythonParserSingle):
     pass

uncompyle6/parsers/parse26.py

@@ -1,4 +1,4 @@
-#  Copyright (c) 2016 Rocky Bernstein
+#  Copyright (c) 2017 Rocky Bernstein
 """
 spark grammar differences over Python2 for Python 2.6.
 """
@@ -22,27 +22,24 @@ class Python26Parser(Python2Parser):
                          JUMP_IF_FALSE POP_TOP POP_TOP designator POP_TOP
 
         try_middle   ::= JUMP_FORWARD COME_FROM except_stmts
-                         come_from_pop END_FINALLY COME_FROM
+                         come_from_pop END_FINALLY come_froms
+
+        try_middle   ::= JUMP_FORWARD COME_FROM except_stmts END_FINALLY
+                         come_froms
 
         try_middle   ::= jmp_abs COME_FROM except_stmts
                          POP_TOP END_FINALLY
 
         try_middle   ::= jmp_abs COME_FROM except_stmts
-                         POP_TOP END_FINALLY
-
-        trystmt      ::= SETUP_EXCEPT suite_stmts_opt POP_TOP
-                         try_middle
+                         END_FINALLY JUMP_FORWARD
 
         # Sometimes we don't put in COME_FROM to the next statement
         # like we do in 2.7. Perhaps we should?
         trystmt      ::= SETUP_EXCEPT suite_stmts_opt POP_BLOCK
                          try_middle
 
-        trystmt      ::= SETUP_EXCEPT suite_stmts_opt POP_BLOCK
-                         try_middle come_froms
-
         tryelsestmt    ::= SETUP_EXCEPT suite_stmts_opt POP_BLOCK
-                           try_middle else_suite come_froms
+                           try_middle else_suite COME_FROM
 
         _ifstmts_jump ::= c_stmts_opt JUMP_FORWARD COME_FROM POP_TOP
 
@@ -113,16 +110,10 @@ class Python26Parser(Python2Parser):
 
         break_stmt ::= BREAK_LOOP JUMP_BACK
 
-        # Semantic actions want the else to be at position 3
-        ifelsestmt  ::= testexpr c_stmts_opt jf_cf_pop else_suite come_froms
-        ifelsestmt  ::= testexpr c_stmts_opt filler else_suitel come_froms POP_TOP
-
         # Semantic actions want else_suitel to be at index 3
         ifelsestmtl ::= testexpr c_stmts_opt jb_cf_pop else_suitel
         ifelsestmtc ::= testexpr c_stmts_opt ja_cf_pop else_suitec
 
-        iflaststmt  ::= testexpr c_stmts_opt JUMP_ABSOLUTE come_froms POP_TOP
-
         # Semantic actions want suite_stmts_opt to be at index 3
         withstmt ::= expr setupwith SETUP_FINALLY suite_stmts_opt
                      POP_BLOCK LOAD_CONST COME_FROM WITH_CLEANUP END_FINALLY
@@ -157,8 +148,33 @@ class Python26Parser(Python2Parser):
         iflaststmtl ::= testexpr c_stmts_opt JUMP_BACK come_from_pop
         iflaststmt  ::= testexpr c_stmts_opt JUMP_ABSOLUTE come_from_pop
 
+        lastc_stmt ::= iflaststmt COME_FROM
+
         while1stmt ::= SETUP_LOOP l_stmts_opt JUMP_BACK COME_FROM
 
+        ifstmt         ::= testexpr_then _ifstmts_jump
+
+        # Semantic actions want the else to be at position 3
+        ifelsestmt     ::= testexpr      c_stmts_opt jf_cf_pop else_suite come_froms
+        ifelsestmt     ::= testexpr_then c_stmts_opt jf_cf_pop else_suite come_froms
+        ifelsestmt     ::= testexpr      c_stmts_opt filler else_suitel come_froms POP_TOP
+        ifelsestmt     ::= testexpr_then c_stmts_opt filler else_suitel come_froms POP_TOP
+
+        # Semantic actions want else_suitel to be at index 3
+        ifelsestmtl    ::= testexpr_then c_stmts_opt jb_cf_pop else_suitel
+        ifelsestmtc    ::= testexpr_then c_stmts_opt ja_cf_pop else_suitec
+
+        iflaststmt     ::= testexpr_then c_stmts_opt JUMP_ABSOLUTE come_froms POP_TOP
+        iflaststmt     ::= testexpr      c_stmts_opt JUMP_ABSOLUTE come_froms POP_TOP
+
+        testexpr_then  ::= testtrue_then
+        testexpr_then  ::= testfalse_then
+        testtrue_then  ::= expr jmp_true_then
+        testfalse_then ::= expr jmp_false_then
+
+        jmp_false_then ::= JUMP_IF_FALSE THEN POP_TOP
+        jmp_true_then  ::= JUMP_IF_TRUE THEN POP_TOP
+
         # Common with 2.7
         while1stmt ::= SETUP_LOOP return_stmts bp_come_from
         while1stmt ::= SETUP_LOOP return_stmts COME_FROM
@@ -177,11 +193,11 @@ class Python26Parser(Python2Parser):
 
         list_iter  ::= list_if JUMP_BACK
         list_iter  ::= list_if JUMP_BACK COME_FROM POP_TOP
-	list_compr ::= BUILD_LIST_0 DUP_TOP
-		       designator list_iter del_stmt
-	list_compr ::= BUILD_LIST_0 DUP_TOP
-		       designator list_iter JUMP_BACK del_stmt
-	lc_body    ::= LOAD_NAME expr LIST_APPEND
+        list_compr ::= BUILD_LIST_0 DUP_TOP
+                       designator list_iter del_stmt
+        list_compr ::= BUILD_LIST_0 DUP_TOP
+	               designator list_iter JUMP_BACK del_stmt
+        lc_body    ::= LOAD_NAME expr LIST_APPEND
 	lc_body    ::= LOAD_FAST expr LIST_APPEND
 
         comp_for ::= SETUP_LOOP expr _for designator comp_iter jb_bp_come_from
@@ -196,16 +212,16 @@ class Python26Parser(Python2Parser):
         genexpr_func ::= setup_loop_lf FOR_ITER designator comp_iter JUMP_BACK come_from_pop
                          jb_bp_come_from
         genexpr ::= LOAD_GENEXPR MAKE_FUNCTION_0 expr GET_ITER CALL_FUNCTION_1 COME_FROM
-
+        list_if ::= list_if ::= expr jmp_false_then list_iter
         '''
 
     def p_ret26(self, args):
         '''
-        ret_and  ::= expr jmp_false ret_expr_or_cond COME_FROM
-        ret_or   ::= expr jmp_true ret_expr_or_cond COME_FROM
-        ret_cond ::= expr jmp_false expr RETURN_END_IF POP_TOP ret_expr_or_cond
-        ret_cond ::= expr jmp_false expr ret_expr_or_cond
-        ret_cond_not ::= expr jmp_true expr RETURN_END_IF POP_TOP ret_expr_or_cond
+        ret_and      ::= expr jmp_false ret_expr_or_cond COME_FROM
+        ret_or       ::= expr jmp_true ret_expr_or_cond COME_FROM
+        ret_cond     ::= expr jmp_false_then expr RETURN_END_IF POP_TOP ret_expr_or_cond
+        ret_cond     ::= expr jmp_false_then expr ret_expr_or_cond
+        ret_cond_not ::= expr jmp_true_then  expr RETURN_END_IF POP_TOP ret_expr_or_cond
 
         return_if_stmt ::= ret_expr RETURN_END_IF POP_TOP
         return_stmt ::= ret_expr RETURN_VALUE POP_TOP
@@ -215,17 +231,37 @@ class Python26Parser(Python2Parser):
         '''
 
     def p_except26(self, args):
-        '''
+        """
         except_suite ::= c_stmts_opt jmp_abs POP_TOP
-        '''
+        """
 
     def p_misc26(self, args):
-        '''
+        """
         conditional  ::= expr jmp_false expr jf_cf_pop expr come_from_opt
         and  ::= expr JUMP_IF_FALSE POP_TOP expr JUMP_IF_FALSE POP_TOP
         cmp_list ::= expr cmp_list1 ROT_TWO COME_FROM POP_TOP _come_from
-        '''
 
+        conditional_lambda ::= expr jmp_false_then return_if_stmt return_stmt LAMBDA_MARKER
+        """
+
+    def add_custom_rules(self, tokens, customize):
+        super(Python26Parser, self).add_custom_rules(tokens, customize)
+        self.check_reduce['and'] = 'AST'
+
+    def reduce_is_invalid(self, rule, ast, tokens, first, last):
+        invalid = super(Python26Parser,
+                        self).reduce_is_invalid(rule, ast,
+                                                tokens, first, last)
+        if invalid:
+            return invalid
+        if rule == ('and', ('expr', 'jmp_false', 'expr', '\\e_come_from_opt')):
+            # Test that jmp_false jumps to the end of "and"
+            # or that it jumps to the same place as the end of "and"
+            jmp_false = ast[1][0]
+            jmp_target = jmp_false.offset + jmp_false.attr + 3
+            return not (jmp_target == tokens[last].offset or
+                        tokens[last].pattr == jmp_false.pattr)
+        return False
 class Python26ParserSingle(Python2Parser, PythonParserSingle):
     pass
 

uncompyle6/parsers/parse27.py

@@ -61,7 +61,7 @@ class Python27Parser(Python2Parser):
 
         ret_and  ::= expr JUMP_IF_FALSE_OR_POP ret_expr_or_cond COME_FROM
         ret_or   ::= expr JUMP_IF_TRUE_OR_POP ret_expr_or_cond COME_FROM
-        ret_cond ::= expr POP_JUMP_IF_FALSE expr RETURN_END_IF ret_expr_or_cond
+        ret_cond ::= expr POP_JUMP_IF_FALSE expr RETURN_END_IF COME_FROM ret_expr_or_cond
         ret_cond_not ::= expr POP_JUMP_IF_TRUE expr RETURN_END_IF ret_expr_or_cond
 
         or   ::= expr JUMP_IF_TRUE_OR_POP expr COME_FROM
@@ -115,8 +115,10 @@ if __name__ == '__main__':
             """.split()))
         remain_tokens = set(tokens) - opcode_set
         import re
-        remain_tokens = set([re.sub('_\d+$','', t) for t in remain_tokens])
-        remain_tokens = set([re.sub('_CONT$','', t) for t in remain_tokens])
+        remain_tokens = set([re.sub('_\d+$', '', t)
+                             for t in remain_tokens])
+        remain_tokens = set([re.sub('_CONT$', '', t)
+                             for t in remain_tokens])
         remain_tokens = set(remain_tokens) - opcode_set
         print(remain_tokens)
         # p.dumpGrammar()

uncompyle6/parsers/parse3.py

@@ -15,8 +15,6 @@ If we succeed in creating a parse tree, then we have a Python program
 that a later phase can turn into a sequence of ASCII text.
 """
 
-from __future__ import print_function
-
 from uncompyle6.parser import PythonParser, PythonParserSingle, nop_func
 from uncompyle6.parsers.astnode import AST
 from spark_parser import DEFAULT_DEBUG as PARSER_DEFAULT_DEBUG
@@ -44,6 +42,10 @@ class Python3Parser(PythonParser):
 
         list_for ::= expr FOR_ITER designator list_iter jb_or_c
 
+        # This is seen in PyPy, but possibly it appears on other Python 3?
+        list_if     ::= expr jmp_false list_iter COME_FROM
+        list_if_not ::= expr jmp_true list_iter COME_FROM
+
         jb_or_c ::= JUMP_BACK
         jb_or_c ::= CONTINUE
 
@@ -52,6 +54,9 @@ class Python3Parser(PythonParser):
         setcomp_func ::= BUILD_SET_0 LOAD_FAST FOR_ITER designator comp_iter
                 JUMP_BACK RETURN_VALUE RETURN_LAST
 
+        setcomp_func ::= BUILD_SET_0 LOAD_FAST FOR_ITER designator comp_iter
+                COME_FROM JUMP_BACK RETURN_VALUE RETURN_LAST
+
         comp_body ::= dict_comp_body
         comp_body ::= set_comp_body
         dict_comp_body ::= expr expr MAP_ADD
@@ -113,9 +118,11 @@ class Python3Parser(PythonParser):
         classdefdeco1 ::= expr classdefdeco1 CALL_FUNCTION_1
         classdefdeco1 ::= expr classdefdeco2 CALL_FUNCTION_1
 
-        assert ::= assert_expr jmp_true LOAD_ASSERT RAISE_VARARGS_1
-        assert2 ::= assert_expr jmp_true LOAD_ASSERT expr CALL_FUNCTION_1 RAISE_VARARGS_1
-        assert2 ::= assert_expr jmp_true LOAD_ASSERT expr RAISE_VARARGS_2
+        assert ::= assert_expr jmp_true LOAD_ASSERT RAISE_VARARGS_1 COME_FROM
+        assert2 ::= assert_expr jmp_true LOAD_ASSERT expr CALL_FUNCTION_1
+                    RAISE_VARARGS_1 COME_FROM
+        assert2 ::= assert_expr jmp_true LOAD_ASSERT expr
+                    RAISE_VARARGS_2 COME_FROM
 
         assert_expr ::= expr
         assert_expr ::= assert_expr_or
@@ -132,6 +139,7 @@ class Python3Parser(PythonParser):
 
         _ifstmts_jump ::= return_if_stmts
         _ifstmts_jump ::= c_stmts_opt JUMP_FORWARD COME_FROM
+        _ifstmts_jump ::= c_stmts_opt COME_FROM
 
         iflaststmt ::= testexpr c_stmts_opt JUMP_ABSOLUTE
 
@@ -155,6 +163,7 @@ class Python3Parser(PythonParser):
         ifelsestmtr ::= testexpr return_if_stmts return_stmts
 
         ifelsestmtl ::= testexpr c_stmts_opt JUMP_BACK else_suitel
+        ifelsestmtl ::= testexpr c_stmts_opt COME_FROM JUMP_BACK else_suitel
 
         # FIXME: this feels like a hack. Is it just 1 or two
         # COME_FROMs?  the parsed tree for this and even with just the
@@ -329,6 +338,9 @@ class Python3Parser(PythonParser):
         forelselaststmtl  ::= SETUP_LOOP expr _for designator for_block POP_BLOCK else_suitel
                               COME_FROM_LOOP
 
+        whilestmt         ::= SETUP_LOOP testexpr l_stmts_opt COME_FROM JUMP_BACK POP_BLOCK
+                              COME_FROM_LOOP
+
         whilestmt         ::= SETUP_LOOP testexpr l_stmts_opt JUMP_BACK POP_BLOCK
                               COME_FROM_LOOP
 
@@ -356,6 +368,9 @@ class Python3Parser(PythonParser):
         while1stmt        ::= SETUP_LOOP l_stmts
         while1stmt        ::= SETUP_LOOP l_stmts COME_FROM_LOOP
 
+        while1stmt        ::= SETUP_LOOP l_stmts COME_FROM JUMP_BACK COME_FROM_LOOP
+
+
         # FIXME: investigate - can code really produce a NOP?
         whileTruestmt     ::= SETUP_LOOP l_stmts_opt JUMP_BACK NOP
                               COME_FROM_LOOP
@@ -453,6 +468,15 @@ class Python3Parser(PythonParser):
                 ('kwarg ' * args_kw) +
                 'expr ' * nak + token.type)
         self.add_unique_rule(rule, token.type, args_pos, customize)
+        if self.version >= 3.5:
+            rule = ('async_call_function ::= expr ' +
+                    ('pos_arg ' * args_pos) +
+                    ('kwarg ' * args_kw) +
+                    'expr ' * nak + token.type +
+                    ' GET_AWAITABLE LOAD_CONST YIELD_FROM')
+            self.add_unique_rule(rule, token.type, args_pos, customize)
+            self.add_unique_rule('expr ::= async_call_function', token.type, args_pos, customize)
+
         rule = ('classdefdeco2 ::= LOAD_BUILD_CLASS mkfunc %s%s_%d'
                 %  (('expr ' * (args_pos-1)), opname, args_pos))
         self.add_unique_rule(rule, token.type, args_pos, customize)
@@ -461,7 +485,10 @@ class Python3Parser(PythonParser):
         """Python 3.3 added a an addtional LOAD_CONST before MAKE_FUNCTION and
         this has an effect on many rules.
         """
-        new_rule = rule % (('LOAD_CONST ') * (1 if  self.version >= 3.3 else 0))
+        if self.version >= 3.3:
+            new_rule = rule % (('LOAD_CONST ') * 1)
+        else:
+            new_rule = rule % (('LOAD_CONST ') * 0)
         self.add_unique_rule(new_rule, opname, attr, customize)
 
     def add_custom_rules(self, tokens, customize):
@@ -476,8 +503,15 @@ class Python3Parser(PythonParser):
 
             # build_class (see load_build_class)
 
-            build_list  ::= {expr}^n BUILD_LIST_n
-            build_list  ::= {expr}^n BUILD_TUPLE_n
+            # Even the below say _list, in the semantic rules we
+            # disambiguate tuples, and sets from lists
+
+            build_list ::= {expr}^n BUILD_LIST_n
+            build_list ::= {expr}^n BUILD_TUPLE_n
+            build_list ::= {expr}^n BUILD_SET_n
+            build_list ::= {expr}^n BUILD_LIST_UNPACK_n
+            build_list ::= {expr}^n BUILD_SET_UNPACK_n
+            build_list ::= {expr}^n BUILD_TUPLE_UNPACK_n
 
             load_closure  ::= {LOAD_CLOSURE}^n BUILD_TUPLE_n
             # call_function (see custom_classfunc_rule)
@@ -550,7 +584,8 @@ class Python3Parser(PythonParser):
             elif opname_base in ('BUILD_LIST', 'BUILD_TUPLE', 'BUILD_SET'):
                 v = token.attr
                 rule = ('build_list ::= ' + 'expr1024 ' * int(v//1024) +
-                                        'expr32 ' * int((v//32)%32) + 'expr '*(v%32) + opname)
+                        'expr32 ' * int((v//32) % 32) +
+                        'expr ' * (v % 32) + opname)
                 self.add_unique_rule(rule, opname, token.attr, customize)
                 if opname_base == 'BUILD_TUPLE':
                     rule = ('load_closure ::= %s%s' % (('LOAD_CLOSURE ' * v), opname))
@@ -590,15 +625,21 @@ class Python3Parser(PythonParser):
                     self.add_unique_rule(rule, 'kvlist_n', 1, customize)
                     rule = "mapexpr ::=  BUILD_MAP_n kvlist_n"
                 elif self.version >= 3.5:
-                    if  opname != 'BUILD_MAP_WITH_CALL':
+                    if opname != 'BUILD_MAP_WITH_CALL':
                         rule = kvlist_n + ' ::= ' + 'expr ' * (token.attr*2)
                         self.add_unique_rule(rule, opname, token.attr, customize)
                         rule = "mapexpr ::=  %s %s" % (kvlist_n, opname)
+
                 else:
                     rule = kvlist_n + ' ::= ' + 'expr expr STORE_MAP ' * token.attr
                     self.add_unique_rule(rule, opname, token.attr, customize)
                     rule = "mapexpr ::=  %s %s" % (opname, kvlist_n)
                 self.add_unique_rule(rule, opname, token.attr, customize)
+            elif opname_base == 'BUILD_CONST_KEY_MAP':
+                # This is in 3.6+
+                kvlist_n = 'expr ' * (token.attr)
+                rule = "mapexpr ::= %sLOAD_CONST %s" % (kvlist_n, opname)
+                self.add_unique_rule(rule, opname, token.attr, customize)
             elif opname_base in ('UNPACK_EX',):
                 before_count, after_count = token.attr
                 rule = 'unpack ::= ' + opname + ' designator' * (before_count + after_count + 1)
@@ -635,6 +676,27 @@ class Python3Parser(PythonParser):
                     rule = ('mkfunc ::= kwargs %sexpr %s' %
                             ('pos_arg ' * args_pos, opname))
                 self.add_unique_rule(rule, opname, token.attr, customize)
+                if opname.startswith('MAKE_FUNCTION_A'):
+                    # rule = ('mkfunc2 ::= %s%sEXTENDED_ARG %s' %
+                    #         ('pos_arg ' * (args_pos), 'kwargs ' * (annotate_args-1), opname))
+                    self.add_unique_rule(rule, opname, token.attr, customize)
+                    if self.version >= 3.3:
+                        rule = ('mkfunc_annotate ::= %s%sannotate_tuple LOAD_CONST LOAD_CONST EXTENDED_ARG %s' %
+                                (('pos_arg ' * (args_pos)),
+                                 ('call_function ' * (annotate_args-1)), opname))
+                        self.add_unique_rule(rule, opname, token.attr, customize)
+                        rule = ('mkfunc_annotate ::= %s%sannotate_tuple LOAD_CONST LOAD_CONST EXTENDED_ARG %s' %
+                                (('pos_arg ' * (args_pos)),
+                                 ('annotate_arg ' * (annotate_args-1)), opname))
+                    else:
+                        rule = ('mkfunc_annotate ::= %s%sannotate_tuple LOAD_CONST EXTENDED_ARG %s' %
+                                (('pos_arg ' * (args_pos)),
+                                 ('annotate_arg ' * (annotate_args-1)), opname))
+                        self.add_unique_rule(rule, opname, token.attr, customize)
+                        rule = ('mkfunc_annotate ::= %s%sannotate_tuple LOAD_CONST EXTENDED_ARG %s' %
+                                (('pos_arg ' * (args_pos)),
+                                 ('call_function ' * (annotate_args-1)), opname))
+                    self.add_unique_rule(rule, opname, token.attr, customize)
             elif opname_base == 'CALL_METHOD':
                 # PyPy only - DRY with parse2
 
@@ -695,12 +757,28 @@ class Python3Parser(PythonParser):
         self.check_reduce['augassign1'] = 'AST'
         self.check_reduce['augassign2'] = 'AST'
         self.check_reduce['while1stmt'] = 'noAST'
+        self.check_reduce['annotate_tuple'] = 'noAST'
+        self.check_reduce['kwarg'] = 'noAST'
+        # FIXME: remove parser errors caused by the below
+        # self.check_reduce['while1elsestmt'] = 'noAST'
         return
 
     def reduce_is_invalid(self, rule, ast, tokens, first, last):
         lhs = rule[0]
         if lhs in ('augassign1', 'augassign2') and ast[0][0] == 'and':
             return True
+        elif lhs == 'annotate_tuple':
+            return not isinstance(tokens[first].attr, tuple)
+        elif lhs == 'kwarg':
+            return not (isinstance(tokens[first].attr, unicode) or
+                        isinstance(tokens[first].attr, str))
+        elif lhs == 'while1elsestmt':
+            # if SETUP_LOOP target spans the else part, then this is
+            # not while1else. Also do for whileTrue?
+            last += 1
+            while isinstance(tokens[last].offset, str):
+                last += 1
+            return tokens[first].attr == tokens[last].offset
         elif lhs == 'while1stmt':
             if tokens[last] in ('COME_FROM_LOOP', 'JUMP_BACK'):
                 # jump_back should be right afer SETUP_LOOP. Test?

uncompyle6/parsers/parse30.py

@@ -2,12 +2,11 @@
 """
 spark grammar differences over Python 3.1 for Python 3.0.
 """
-from __future__ import print_function
 
 from uncompyle6.parser import PythonParserSingle
-from uncompyle6.parsers.parse3 import Python3Parser
+from uncompyle6.parsers.parse31 import Python31Parser
 
-class Python30Parser(Python3Parser):
+class Python30Parser(Python31Parser):
 
     def p_30(self, args):
         """
@@ -15,6 +14,11 @@ class Python30Parser(Python3Parser):
         stmt         ::= store_locals
         store_locals ::= LOAD_FAST STORE_LOCALS
 
+        # FIXME: combine with parse3.2
+        whileTruestmt     ::= SETUP_LOOP l_stmts_opt JUMP_BACK
+                              COME_FROM_LOOP
+        whileTruestmt     ::= SETUP_LOOP return_stmts
+                              COME_FROM_LOOP
 
         # In many ways Python 3.0 code generation is more like Python 2.6 than
         # it is 2.7 or 3.1. So we have a number of 2.6ish (and before) rules below
@@ -39,5 +43,10 @@ class Python30Parser(Python3Parser):
         setup_finally ::= STORE_FAST SETUP_FINALLY LOAD_FAST DELETE_FAST
         """
 
+    def add_custom_rules(self, tokens, customize):
+        super(Python30Parser, self).add_custom_rules(tokens, customize)
+        return
+    pass
+
 class Python30ParserSingle(Python30Parser, PythonParserSingle):
     pass

uncompyle6/parsers/parse31.py

@@ -2,7 +2,6 @@
 """
 spark grammar differences over Python 3.2 for Python 3.1.
 """
-from __future__ import print_function
 
 from uncompyle6.parser import PythonParserSingle
 from uncompyle6.parsers.parse32 import Python32Parser
@@ -36,16 +35,8 @@ class Python31Parser(Python32Parser):
 
     def add_custom_rules(self, tokens, customize):
         super(Python31Parser, self).add_custom_rules(tokens, customize)
-        for i, token in enumerate(tokens):
-            opname = token.type
-            if opname.startswith('MAKE_FUNCTION_A'):
-                args_pos, args_kw, annotate_args  = token.attr
-                # Check that there are 2 annotated params?
-                # rule = ('mkfunc2 ::= %s%sEXTENDED_ARG %s' %
-                #         ('pos_arg ' * (args_pos), 'kwargs ' * (annotate_args-1), opname))
-                rule = ('mkfunc_annotate ::= %s%sannotate_tuple LOAD_CONST EXTENDED_ARG %s' %
-                        (('pos_arg ' * (args_pos)),
-                         ('annotate_arg ' * (annotate_args-1)), opname))
-                self.add_unique_rule(rule, opname, token.attr, customize)
+        return
+    pass
+
 class Python31ParserSingle(Python31Parser, PythonParserSingle):
     pass

uncompyle6/parsers/parse32.py

@@ -2,8 +2,6 @@
 """
 spark grammar differences over Python 3 for Python 3.2.
 """
-from __future__ import print_function
-
 from uncompyle6.parser import PythonParserSingle
 from uncompyle6.parsers.parse3 import Python3Parser
 
@@ -19,6 +17,11 @@ class Python32Parser(Python3Parser):
                               COME_FROM_LOOP
         whileTruestmt     ::= SETUP_LOOP return_stmts
                               COME_FROM_LOOP
+
+        # Python 3.2+ has more loop optimization that removes
+        # JUMP_FORWARD in some cases, and hence we also don't
+        # see COME_FROM
+        _ifstmts_jump ::= c_stmts_opt
         """
     pass
 
@@ -41,6 +44,9 @@ class Python32Parser(Python3Parser):
                         (('pos_arg ' * (args_pos)),
                          ('annotate_arg ' * (annotate_args-1)), opname))
                 self.add_unique_rule(rule, opname, token.attr, customize)
+                pass
+            return
+        pass
 
 
 class Python32ParserSingle(Python32Parser, PythonParserSingle):

uncompyle6/parsers/parse33.py

@@ -2,7 +2,6 @@
 """
 spark grammar differences over Python 3.2 for Python 3.3.
 """
-from __future__ import print_function
 
 from uncompyle6.parser import PythonParserSingle
 from uncompyle6.parsers.parse32 import Python32Parser
@@ -21,11 +20,6 @@ class Python33Parser(Python32Parser):
         iflaststmt    ::= testexpr c_stmts_opt33
         c_stmts_opt33 ::= JUMP_BACK JUMP_ABSOLUTE c_stmts_opt
         _ifstmts_jump ::= c_stmts_opt JUMP_FORWARD _come_from
-
-        # Python 3.3+ has more loop optimization that removes
-        # JUMP_FORWARD in some cases, and hence we also don't
-        # see COME_FROM
-        _ifstmts_jump ::= c_stmts_opt
         """
 
 class Python33ParserSingle(Python33Parser, PythonParserSingle):

uncompyle6/parsers/parse35.py

@@ -2,7 +2,6 @@
 """
 spark grammar differences over Python 3.4 for Python 3.5.
 """
-from __future__ import print_function
 
 from uncompyle6.parser import PythonParserSingle
 from spark_parser import DEFAULT_DEBUG as PARSER_DEFAULT_DEBUG
@@ -16,36 +15,103 @@ class Python35Parser(Python34Parser):
 
     def p_35on(self, args):
         """
+        # The number of canned instructions in new statements is mind boggling.
+        # I'm sure by the time Python 4 comes around these will be turned
+        # into special opcodes
+
+        while1stmt  ::= SETUP_LOOP l_stmts COME_FROM JUMP_BACK
+                        POP_BLOCK COME_FROM_LOOP
+
+        # Python 3.5+ Await statement
+        stmt ::= await_stmt
+        await_stmt ::= call_function GET_AWAITABLE LOAD_CONST YIELD_FROM POP_TOP
+
         # Python 3.5+ has WITH_CLEANUP_START/FINISH
 
-        withstmt ::= expr SETUP_WITH exprlist suite_stmts_opt
-                    POP_BLOCK LOAD_CONST COME_FROM_WITH
-                    WITH_CLEANUP_START WITH_CLEANUP_FINISH END_FINALLY
+        withstmt    ::= expr
+                        SETUP_WITH exprlist suite_stmts_opt
+                        POP_BLOCK LOAD_CONST COME_FROM_WITH
+                        WITH_CLEANUP_START WITH_CLEANUP_FINISH END_FINALLY
 
-        withstmt ::= expr SETUP_WITH POP_TOP suite_stmts_opt
-                     POP_BLOCK LOAD_CONST COME_FROM_WITH
-                     WITH_CLEANUP_START WITH_CLEANUP_FINISH END_FINALLY
+        withstmt   ::= expr
+                       SETUP_WITH POP_TOP suite_stmts_opt
+                       POP_BLOCK LOAD_CONST COME_FROM_WITH
+                       WITH_CLEANUP_START WITH_CLEANUP_FINISH END_FINALLY
+
+        withasstmt ::= expr
+                       SETUP_WITH designator suite_stmts_opt
+                       POP_BLOCK LOAD_CONST COME_FROM_WITH
+                       WITH_CLEANUP_START WITH_CLEANUP_FINISH END_FINALLY
+
+
+        # Python 3.5+ async additions
+
+        stmt            ::= async_with_stmt
+        async_with_stmt ::= expr
+                            BEFORE_ASYNC_WITH GET_AWAITABLE LOAD_CONST YIELD_FROM
+                            SETUP_ASYNC_WITH POP_TOP suite_stmts_opt
+                            POP_BLOCK LOAD_CONST
+                            WITH_CLEANUP_START
+                            GET_AWAITABLE LOAD_CONST YIELD_FROM
+                            WITH_CLEANUP_FINISH END_FINALLY
+
+        stmt               ::= async_with_as_stmt
+        async_with_as_stmt ::= expr
+                               BEFORE_ASYNC_WITH GET_AWAITABLE LOAD_CONST YIELD_FROM
+                               SETUP_ASYNC_WITH designator suite_stmts_opt
+                               POP_BLOCK LOAD_CONST
+                               WITH_CLEANUP_START
+                               GET_AWAITABLE LOAD_CONST YIELD_FROM
+                               WITH_CLEANUP_FINISH END_FINALLY
+
+
+        stmt               ::= async_for_stmt
+        async_for_stmt     ::= SETUP_LOOP expr
+                               GET_AITER
+                               LOAD_CONST YIELD_FROM SETUP_EXCEPT GET_ANEXT LOAD_CONST
+                               YIELD_FROM
+                               designator
+                               POP_BLOCK JUMP_FORWARD COME_FROM_EXCEPT DUP_TOP
+                               LOAD_GLOBAL COMPARE_OP POP_JUMP_IF_FALSE
+                               POP_TOP POP_TOP POP_TOP POP_EXCEPT POP_BLOCK
+                               JUMP_ABSOLUTE END_FINALLY COME_FROM
+                               for_block POP_BLOCK JUMP_ABSOLUTE
+                               opt_come_from_loop
+
+        stmt               ::= async_forelse_stmt
+        async_forelse_stmt ::= SETUP_LOOP expr
+                               GET_AITER
+                               LOAD_CONST YIELD_FROM SETUP_EXCEPT GET_ANEXT LOAD_CONST
+                               YIELD_FROM
+                               designator
+                               POP_BLOCK JUMP_FORWARD COME_FROM_EXCEPT DUP_TOP
+                               LOAD_GLOBAL COMPARE_OP POP_JUMP_IF_FALSE
+                               POP_TOP POP_TOP POP_TOP POP_EXCEPT POP_BLOCK
+                               JUMP_ABSOLUTE END_FINALLY COME_FROM
+                               for_block POP_BLOCK JUMP_ABSOLUTE
+                               else_suite COME_FROM_LOOP
 
-        withasstmt ::= expr SETUP_WITH designator suite_stmts_opt
-                POP_BLOCK LOAD_CONST COME_FROM_WITH
-                WITH_CLEANUP_START WITH_CLEANUP_FINISH END_FINALLY
 
         inplace_op ::= INPLACE_MATRIX_MULTIPLY
         binary_op  ::= BINARY_MATRIX_MULTIPLY
 
         # Python 3.5+ does jump optimization
         # In <.3.5 the below is a JUMP_FORWARD to a JUMP_ABSOLUTE.
-        # in return_stmt, we will need the semantic actions in pysource.py
-        # to work out whether to dedent or not based on the presence of
-        # RETURN_END_IF vs RETURN_VALUE
+
+        return_if_stmt ::= ret_expr RETURN_END_IF POP_BLOCK
 
         ifelsestmtc ::= testexpr c_stmts_opt JUMP_FORWARD else_suitec
+        ifelsestmtc ::= testexpr c_stmts_opt jf_else else_suitec
+
         # ifstmt ::= testexpr c_stmts_opt
 
+        iflaststmt ::= testexpr c_stmts_opt JUMP_FORWARD
+
         # Python 3.3+ also has yield from. 3.5 does it
         # differently than 3.3, 3.4
 
         yield_from ::= expr GET_YIELD_FROM_ITER LOAD_CONST YIELD_FROM
+
         """
 
     def add_custom_rules(self, tokens, customize):

uncompyle6/parsers/parse36.py

@@ -2,7 +2,6 @@
 """
 spark grammar differences over Python 3.5 for Python 3.6.
 """
-from __future__ import print_function
 
 from uncompyle6.parser import PythonParserSingle
 from spark_parser import DEFAULT_DEBUG as PARSER_DEFAULT_DEBUG
@@ -21,6 +20,9 @@ class Python36Parser(Python35Parser):
 
         func_args36   ::= expr BUILD_TUPLE_0
         call_function ::= func_args36 unmapexpr CALL_FUNCTION_EX
+
+        withstmt ::= expr SETUP_WITH POP_TOP suite_stmts_opt POP_BLOCK LOAD_CONST
+                     WITH_CLEANUP_START WITH_CLEANUP_FINISH END_FINALLY
         """
 
     def add_custom_rules(self, tokens, customize):
@@ -69,8 +71,8 @@ if __name__ == '__main__':
             """.split()))
         remain_tokens = set(tokens) - opcode_set
         import re
-        remain_tokens = set([re.sub('_\d+$','', t) for t in remain_tokens])
-        remain_tokens = set([re.sub('_CONT$','', t) for t in remain_tokens])
+        remain_tokens = set([re.sub('_\d+$', '', t) for t in remain_tokens])
+        remain_tokens = set([re.sub('_CONT$', '', t) for t in remain_tokens])
         remain_tokens = set(remain_tokens) - opcode_set
         print(remain_tokens)
         # print(sorted(p.rule2name.items()))

uncompyle6/scanner.py

@@ -10,8 +10,6 @@ scanner/ingestion module. From here we call various version-specific
 scanners, e.g. for Python 2.7 or 3.4.
 """
 
-from __future__ import print_function
-
 import sys
 
 from uncompyle6 import PYTHON3, IS_PYPY
@@ -226,9 +224,15 @@ class Scanner(object):
         for given opcode <op>.
         """
         if op < self.opc.HAVE_ARGUMENT:
-            return 1
+            if self.version >= 3.6:
+                return 2
+            else:
+                return 1
         else:
-            return 2 if self.version >= 3.6 else 3
+            if self.version >= 3.6:
+                return 2
+            else:
+                return 3
 
     def remove_mid_line_ifs(self, ifs):
         """

uncompyle6/scanners/scanner15.py

@@ -13,15 +13,15 @@ import uncompyle6.scanners.scanner21 as scan
 from xdis.opcodes import opcode_15
 JUMP_OPs = opcode_15.JUMP_OPs
 
-# We base this off of 2.2 instead of the other way around
+# We base this off of 2.1 instead of the other way around
 # because we cleaned things up this way.
 # The history is that 2.7 support is the cleanest,
 # then from that we got 2.6 and so on.
 class Scanner15(scan.Scanner21):
     def __init__(self, show_asm=False):
-        scan.Scanner21.__init__(self, show_asm)
+        scan.Scanner21.__init__(self, show_asm=False)
         self.opc = opcode_15
         self.opname = opcode_15.opname
         self.version = 1.5
-        self.genexpr_name = '<generator expression>';
+        self.genexpr_name = '<generator expression>'
         return

uncompyle6/scanners/scanner2.py

@@ -20,9 +20,13 @@ For example:
 Finally we save token information.
 """
 
-from __future__ import print_function
+from uncompyle6 import PYTHON_VERSION
+
+if PYTHON_VERSION < 2.6:
+    from xdis.namedtuple25 import namedtuple
+else:
+    from collections import namedtuple
 
-from collections import namedtuple
 from array import array
 
 from uncompyle6.scanner import op_has_argument
@@ -37,7 +41,7 @@ class Scanner2(scan.Scanner):
         self.jump_forward = frozenset([self.opc.JUMP_ABSOLUTE, self.opc.JUMP_FORWARD])
         # This is the 2.5+ default
         # For <2.5 it is <generator expression>
-        self.genexpr_name = '<genexpr>';
+        self.genexpr_name = '<genexpr>'
 
     @staticmethod
     def unmangle_name(name, classname):
@@ -68,7 +72,6 @@ class Scanner2(scan.Scanner):
             varnames = co.co_varnames
         return free, names, varnames
 
-
     def ingest(self, co, classname=None, code_objects={}, show_asm=None):
         """
         Pick out tokens from an uncompyle6 code object, and transform them,
@@ -85,7 +88,9 @@ class Scanner2(scan.Scanner):
         cause specific rules for the specific number of arguments they take.
         """
 
-        show_asm = self.show_asm if not show_asm else show_asm
+        if not show_asm:
+            show_asm = self.show_asm
+
         # show_asm = 'after'
         if show_asm in ('both', 'before'):
             from xdis.bytecode import Bytecode
@@ -98,14 +103,14 @@ class Scanner2(scan.Scanner):
 
         customize = {}
         if self.is_pypy:
-            customize['PyPy'] = 1;
+            customize['PyPy'] = 1
 
         Token = self.Token # shortcut
 
-        n = self.setup_code(co)
+        codelen = self.setup_code(co)
 
-        self.build_lines_data(co, n)
-        self.build_prev_op(n)
+        self.build_lines_data(co, codelen)
+        self.build_prev_op(codelen)
 
         free, names, varnames = self.unmangle_code_names(co, classname)
         self.names = names
@@ -114,7 +119,7 @@ class Scanner2(scan.Scanner):
         # turn 'LOAD_GLOBAL' to 'LOAD_ASSERT'.
         # 'LOAD_ASSERT' is used in assert statements.
         self.load_asserts = set()
-        for i in self.op_range(0, n):
+        for i in self.op_range(0, codelen):
             # We need to detect the difference between:
             #   raise AssertionError
             #  and
@@ -139,7 +144,7 @@ class Scanner2(scan.Scanner):
         last_stmt = self.next_stmt[0]
         i = self.next_stmt[last_stmt]
         replace = {}
-        while i < n-1:
+        while i < codelen - 1:
             if self.lines[last_stmt].next > i:
                 # Distinguish "print ..." from "print ...,"
                 if self.code[last_stmt] == self.opc.PRINT_ITEM:
@@ -151,7 +156,7 @@ class Scanner2(scan.Scanner):
             i = self.next_stmt[i]
 
         extended_arg = 0
-        for offset in self.op_range(0, n):
+        for offset in self.op_range(0, codelen):
             if offset in jump_targets:
                 jump_idx = 0
                 # We want to process COME_FROMs to the same offset to be in *descending*
@@ -160,7 +165,6 @@ class Scanner2(scan.Scanner):
                 # we sort them). That way, specific COME_FROM tags will match up
                 # properly. For example, a "loop" with an "if" nested in it should have the
                 # "loop" tag last so the grammar rule matches that properly.
-                # last_offset = -1
                 for jump_offset  in sorted(jump_targets[offset], reverse=True):
                     # if jump_offset == last_offset:
                     #     continue
@@ -177,6 +181,7 @@ class Scanner2(scan.Scanner):
                         offset="%s_%d" % (offset, jump_idx),
                         has_arg = True))
                     jump_idx += 1
+                    pass
 
             op = self.code[offset]
             op_name = self.opc.opname[op]
@@ -265,13 +270,14 @@ class Scanner2(scan.Scanner):
                 # rule for that.
                 target = self.get_target(offset)
                 if target <= offset:
+                    op_name = 'JUMP_BACK'
                     if (offset in self.stmts
                         and self.code[offset+3] not in (self.opc.END_FINALLY,
-                                                        self.opc.POP_BLOCK)
-                        and offset not in self.not_continue):
-                        op_name = 'CONTINUE'
-                    else:
-                        op_name = 'JUMP_BACK'
+                                                        self.opc.POP_BLOCK)):
+                        if ((offset in self.linestartoffsets and
+                            self.code[self.prev[offset]] == self.opc.JUMP_ABSOLUTE)
+                            or offset not in self.not_continue):
+                            op_name = 'CONTINUE'
 
             elif op == self.opc.LOAD_GLOBAL:
                 if offset in self.load_asserts:
@@ -427,7 +433,7 @@ class Scanner2(scan.Scanner):
                 j = self.prev[s]
                 while code[j] in self.designator_ops:
                     j = self.prev[j]
-                if self.version >= 2.1 and code[j] == self.opc.FOR_ITER:
+                if self.version > 2.1 and code[j] == self.opc.FOR_ITER:
                     stmts.remove(s)
                     continue
             last_stmt = s
@@ -459,7 +465,6 @@ class Scanner2(scan.Scanner):
                         self.ignore_if.add(except_match)
                         return None
 
-
                 self.ignore_if.add(except_match)
                 self.not_continue.add(jmp)
                 return jmp
@@ -479,26 +484,27 @@ class Scanner2(scan.Scanner):
             elif op in self.setup_ops:
                 count_SETUP_ += 1
 
-    def detect_structure(self, offset, op):
+    def detect_control_flow(self, offset, op):
         """
         Detect type of block structures and their boundaries to fix optimized jumps
         in python2.3+
         """
 
-        # TODO: check the struct boundaries more precisely -Dan
-
         code = self.code
 
         # Detect parent structure
         parent = self.structs[0]
         start  = parent['start']
         end    = parent['end']
+
+        # Pick inner-most parent for our offset
         for struct in self.structs:
-            _start = struct['start']
-            _end   = struct['end']
-            if (_start <= offset < _end) and (_start >= start and _end <= end):
-                start  = _start
-                end    = _end
+            current_start = struct['start']
+            current_end   = struct['end']
+            if ((current_start <= offset < current_end)
+                and (current_start >= start and current_end <= end)):
+                start  = current_start
+                end    = current_end
                 parent = struct
 
         if op == self.opc.SETUP_LOOP:
@@ -589,7 +595,7 @@ class Scanner2(scan.Scanner):
 
                 target = self.get_target(jump_back, self.opc.JUMP_ABSOLUTE)
 
-                if (self.version >= 2.0 and
+                if (self.version > 2.1 and
                     code[target] in (self.opc.FOR_ITER, self.opc.GET_ITER)):
                     loop_type = 'for'
                 else:
@@ -617,7 +623,7 @@ class Scanner2(scan.Scanner):
                                        'start': jump_back+3,
                                        'end':   end})
         elif op == self.opc.SETUP_EXCEPT:
-            start  = offset+3
+            start  = offset + self.op_size(op)
             target = self.get_target(offset, op)
             end    = self.restrict_to_parent(target, parent)
             if target != end:
@@ -630,9 +636,23 @@ class Scanner2(scan.Scanner):
             # Now isolate the except and else blocks
             end_else = start_else = self.get_target(self.prev[end])
 
+
+            end_finally_offset = end
+            setup_except_nest = 0
+            while end_finally_offset < len(self.code):
+                if self.code[end_finally_offset] == self.opc.END_FINALLY:
+                    if setup_except_nest == 0:
+                        break
+                    else:
+                        setup_except_nest -= 1
+                elif self.code[end_finally_offset] == self.opc.SETUP_EXCEPT:
+                    setup_except_nest += 1
+                end_finally_offset += self.op_size(code[end_finally_offset])
+                pass
+
             # Add the except blocks
             i = end
-            while i < len(self.code) and self.code[i] != self.opc.END_FINALLY:
+            while i < len(self.code) and i < end_finally_offset:
                 jmp = self.next_except_jump(i)
                 if jmp is None: # check
                     i = self.next_stmt[i]
@@ -677,6 +697,8 @@ class Scanner2(scan.Scanner):
                 self.fixed_jumps[offset] = rtarget
                 return
 
+            jump_if_offset = offset
+
             start = offset+3
             pre = self.prev
 
@@ -699,6 +721,10 @@ class Scanner2(scan.Scanner):
                                        'end':   pre[target]})
                 return
 
+            # The op offset just before the target jump offset is important
+            # in making a determination of what we have. Save that.
+            pre_rtarget = pre[rtarget]
+
             # Is it an "and" inside an "if" or "while" block
             if op == self.opc.PJIF:
 
@@ -709,22 +735,22 @@ class Scanner2(scan.Scanner):
 
                 # If we still have any offsets in set, start working on it
                 if match:
-                    if code[pre[rtarget]] in self.jump_forward \
-                            and pre[rtarget] not in self.stmts \
-                            and self.restrict_to_parent(self.get_target(pre[rtarget]), parent) == rtarget:
-                        if code[pre[pre[rtarget]]] == self.opc.JUMP_ABSOLUTE \
+                    if code[pre_rtarget] in self.jump_forward \
+                            and pre_rtarget not in self.stmts \
+                            and self.restrict_to_parent(self.get_target(pre_rtarget), parent) == rtarget:
+                        if code[pre[pre_rtarget]] == self.opc.JUMP_ABSOLUTE \
                                 and self.remove_mid_line_ifs([offset]) \
-                                and target == self.get_target(pre[pre[rtarget]]) \
-                                and (pre[pre[rtarget]] not in self.stmts or self.get_target(pre[pre[rtarget]]) > pre[pre[rtarget]])\
-                                and 1 == len(self.remove_mid_line_ifs(self.rem_or(start, pre[pre[rtarget]], self.pop_jump_if, target))):
+                                and target == self.get_target(pre[pre_rtarget]) \
+                                and (pre[pre_rtarget] not in self.stmts or self.get_target(pre[pre_rtarget]) > pre[pre_rtarget])\
+                                and 1 == len(self.remove_mid_line_ifs(self.rem_or(start, pre[pre_rtarget], self.pop_jump_if, target))):
                             pass
-                        elif code[pre[pre[rtarget]]] == self.opc.RETURN_VALUE \
+                        elif code[pre[pre_rtarget]] == self.opc.RETURN_VALUE \
                                 and self.remove_mid_line_ifs([offset]) \
                                 and 1 == (len(set(self.remove_mid_line_ifs(self.rem_or(start,
-                                                                                       pre[pre[rtarget]],
+                                                                                       pre[pre_rtarget],
                                                                                        self.pop_jump_if, target)))
-                                              | set(self.remove_mid_line_ifs(self.rem_or(start, pre[pre[rtarget]],
-                                                            (self.opc.PJIF, self.opc.PJIT, self.opc.JUMP_ABSOLUTE), pre[rtarget], True))))):
+                                              | set(self.remove_mid_line_ifs(self.rem_or(start, pre[pre_rtarget],
+                                                            (self.opc.PJIF, self.opc.PJIT, self.opc.JUMP_ABSOLUTE), pre_rtarget, True))))):
                             pass
                         else:
                             fix = None
@@ -742,7 +768,7 @@ class Scanner2(scan.Scanner):
                     else:
                         if (self.version < 2.7
                             and parent['type'] in ('root', 'for-loop', 'if-then',
-                                                   'if-else', 'try')):
+                                                   'else', 'try')):
                             self.fixed_jumps[offset] = rtarget
                         else:
                             # note test for < 2.7 might be superflous although informative
@@ -757,7 +783,7 @@ class Scanner2(scan.Scanner):
                 else:
                     assert_offset = offset + 3
                 if (assert_offset) in self.load_asserts:
-                    if code[pre[rtarget]] == self.opc.RAISE_VARARGS:
+                    if code[pre_rtarget] == self.opc.RAISE_VARARGS:
                         return
                     self.load_asserts.remove(assert_offset)
 
@@ -766,7 +792,7 @@ class Scanner2(scan.Scanner):
                     pass
                 elif code[next] in self.jump_forward and target == self.get_target(next):
                     if code[pre[next]] == self.opc.PJIF:
-                        if code[next] == self.opc.JUMP_FORWARD or target != rtarget or code[pre[pre[rtarget]]] not in (self.opc.JUMP_ABSOLUTE, self.opc.RETURN_VALUE):
+                        if code[next] == self.opc.JUMP_FORWARD or target != rtarget or code[pre[pre_rtarget]] not in (self.opc.JUMP_ABSOLUTE, self.opc.RETURN_VALUE):
                             self.fixed_jumps[offset] = pre[next]
                             return
                 elif code[next] == self.opc.JUMP_ABSOLUTE and code[target] in self.jump_forward:
@@ -783,20 +809,32 @@ class Scanner2(scan.Scanner):
                 return
 
             if self.version == 2.7:
-                if code[pre[rtarget]] == self.opc.JUMP_ABSOLUTE and pre[rtarget] in self.stmts \
-                        and pre[rtarget] != offset and pre[pre[rtarget]] != offset:
+                if code[pre_rtarget] == self.opc.JUMP_ABSOLUTE and pre_rtarget in self.stmts \
+                        and pre_rtarget != offset and pre[pre_rtarget] != offset:
                     if code[rtarget] == self.opc.JUMP_ABSOLUTE and code[rtarget+3] == self.opc.POP_BLOCK:
-                        if code[pre[pre[rtarget]]] != self.opc.JUMP_ABSOLUTE:
+                        if code[pre[pre_rtarget]] != self.opc.JUMP_ABSOLUTE:
                             pass
-                        elif self.get_target(pre[pre[rtarget]]) != target:
+                        elif self.get_target(pre[pre_rtarget]) != target:
                             pass
                         else:
-                            rtarget = pre[rtarget]
+                            rtarget = pre_rtarget
                     else:
-                        rtarget = pre[rtarget]
+                        rtarget = pre_rtarget
 
-            # Does the "if" jump just beyond a jump op, then this is probably an if statement
-            pre_rtarget = pre[rtarget]
+            # Does the "jump if" jump beyond a jump op?
+            # That is, we have something like:
+            #  POP_JUMP_IF_FALSE HERE
+            #  ...
+            # JUMP_FORWARD
+            # HERE:
+            #
+            # If so, this can be block inside an "if" statement
+            # or a conditional assignment like:
+            #   x = 1 if x else 2
+            #
+            # There are other contexts we may need to consider
+            # like whether the target is "END_FINALLY"
+            # or if the condition jump is to a forward location
             code_pre_rtarget = code[pre_rtarget]
 
             if code_pre_rtarget in self.jump_forward:
@@ -816,21 +854,87 @@ class Scanner2(scan.Scanner):
                             jump_target = self.get_target(next_offset, next_op)
                             if jump_target in self.setup_loops:
                                 self.structs.append({'type':  'while-loop',
-                                       'start': start - 3,
+                                       'start': jump_if_offset,
                                        'end':   jump_target})
-                                self.fixed_jumps[start-3] = jump_target
+                                self.fixed_jumps[jump_if_offset] = jump_target
                                 return
 
                 end = self.restrict_to_parent(if_end, parent)
 
-                self.structs.append({'type':  'if-then',
-                                       'start': start-3,
-                                       'end':   pre_rtarget})
+                if_then_maybe = None
+
+                if 2.2 <= self.version <= 2.6:
+                    # Take the JUMP_IF target. In an "if/then", it will be
+                    # a POP_TOP instruction and the instruction before it
+                    # will be a JUMP_FORWARD to just after the POP_TOP.
+                    # For example:
+                    # Good:
+                    # 3  JUMP_IF_FALSE        33  'to 39'
+                    # ..
+                    # 36  JUMP_FORWARD          1  'to 40'
+                    # 39  POP_TOP
+                    # 40 ...
+                    # example:
+
+                    # BAD (is an "and"):
+                    # 28  JUMP_IF_FALSE         4  'to 35'
+                    # ...
+                    # 32  JUMP_ABSOLUTE        40  'to 40' # should be 36 or there should
+                    #                                      # be a COME_FROM at the pop top
+                    #                                      # before 40 to 35
+                    # 35  POP_TOP
+                    # 36 ...
+                    # 39  POP_TOP
+                    # 39_0  COME_FROM 3
+                    # 40 ...
+
+                    if self.opc.opname[code[jump_if_offset]].startswith('JUMP_IF'):
+                        jump_if_target = code[jump_if_offset+1]
+                        if self.opc.opname[code[jump_if_target + jump_if_offset + 3]] == 'POP_TOP':
+                            jump_inst = jump_if_target + jump_if_offset
+                            jump_offset = code[jump_inst+1]
+                            jump_op = self.opc.opname[code[jump_inst]]
+                            if (jump_op == 'JUMP_FORWARD' and jump_offset == 1):
+                                self.structs.append({'type':  'if-then',
+                                                     'start': start-3,
+                                                     'end':   pre_rtarget})
+
+                                self.thens[start] = end
+                            elif jump_op == 'JUMP_ABSOLUTE':
+                                if_then_maybe = {'type':  'if-then',
+                                                 'start': start-3,
+                                                 'end':   pre_rtarget}
+
+                elif self.version == 2.7:
+                    self.structs.append({'type':  'if-then',
+                                         'start': start-3,
+                                         'end':   pre_rtarget})
 
                 self.not_continue.add(pre_rtarget)
 
                 if rtarget < end:
-                    self.structs.append({'type':  'if-else',
+                    # We have an "else" block  of some kind.
+                    # Is it associated with "if_then_maybe" seen above?
+                    # These will be linked in this funny way:
+
+                    # 198  JUMP_IF_FALSE        18  'to 219'
+                    # 201  POP_TOP
+                    # ...
+                    # 216  JUMP_ABSOLUTE       256  'to 256'
+                    # 219  POP_TOP
+                    # ...
+                    # 252  JUMP_FORWARD          1  'to 256'
+                    # 255  POP_TOP
+                    # 256
+                    if if_then_maybe and jump_op == 'JUMP_ABSOLUTE':
+                        jump_target = self.get_target(jump_inst, code[jump_inst])
+                        if self.opc.opname[code[end]] == 'JUMP_FORWARD':
+                            end_target = self.get_target(end, code[end])
+                            if jump_target == end_target:
+                                self.structs.append(if_then_maybe)
+                                self.thens[start] = end
+
+                    self.structs.append({'type':  'else',
                                        'start': rtarget,
                                        'end':   end})
             elif code_pre_rtarget == self.opc.RETURN_VALUE:
@@ -838,7 +942,9 @@ class Scanner2(scan.Scanner):
                     self.structs.append({'type':  'if-then',
                                            'start': start,
                                            'end':   rtarget})
+                    self.thens[start] = rtarget
                     if self.version == 2.7 or code[pre_rtarget+1] != self.opc.JUMP_FORWARD:
+                        self.fixed_jumps[offset] = rtarget
                         self.return_end_ifs.add(pre_rtarget)
 
         elif op in self.pop_jump_if_or_pop:
@@ -871,11 +977,12 @@ class Scanner2(scan.Scanner):
         self.ignore_if = set()
         self.build_statement_indices()
 
-        # Containers filled by detect_structure()
+        # Containers filled by detect_control_flow()
         self.not_continue = set()
         self.return_end_ifs = set()
         self.setup_loop_targets = {}  # target given setup_loop offset
         self.setup_loops = {}  # setup_loop offset given target
+        self.thens = {} # JUMP_IF's that separate the 'then' part of an 'if'
 
         targets = {}
         for offset in self.op_range(0, n):
@@ -883,7 +990,7 @@ class Scanner2(scan.Scanner):
 
             # Determine structures and fix jumps in Python versions
             # since 2.3
-            self.detect_structure(offset, op)
+            self.detect_control_flow(offset, op)
 
             if op_has_argument(op, self.opc):
                 label = self.fixed_jumps.get(offset)
@@ -891,8 +998,8 @@ class Scanner2(scan.Scanner):
 
                 if label is None:
                     if op in self.opc.hasjrel and self.opc.opname[op] != 'FOR_ITER':
-                    # if (op in self.opc.hasjrel and
-                    #     (self.version < 2.0 or op != self.opc.FOR_ITER)):
+                        # if (op in self.opc.hasjrel and
+                        #     (self.version < 2.0 or op != self.opc.FOR_ITER)):
                         label = offset + 3 + oparg
                     elif self.version == 2.7 and op in self.opc.hasjabs:
                         if op in (self.opc.JUMP_IF_FALSE_OR_POP,
@@ -922,8 +1029,10 @@ class Scanner2(scan.Scanner):
 
                             # FIXME: rocky: I think we need something like this...
                             if offset not in set(self.ignore_if) or self.version == 2.7:
-                                source = (self.setup_loops[label]
-                                          if label in self.setup_loops else offset)
+                                if label in self.setup_loops:
+                                    source = self.setup_loops[label]
+                                else:
+                                    source = offset
                                 targets[label] = targets.get(label, []) + [source]
                             pass
 

uncompyle6/scanners/scanner21.py

@@ -19,9 +19,9 @@ JUMP_OPs = opcode_21.JUMP_OPs
 # then from that we got 2.6 and so on.
 class Scanner21(scan.Scanner22):
     def __init__(self, show_asm=False):
-        scan.Scanner22.__init__(self, show_asm)
+        scan.Scanner22.__init__(self, show_asm=False)
         self.opc = opcode_21
         self.opname = opcode_21.opname
         self.version = 2.1
-        self.genexpr_name = '<generator expression>';
+        self.genexpr_name = '<generator expression>'
         return

uncompyle6/scanners/scanner22.py

@@ -19,11 +19,11 @@ JUMP_OPs = opcode_22.JUMP_OPs
 # then from that we got 2.6 and so on.
 class Scanner22(scan.Scanner23):
     def __init__(self, show_asm=False):
-        scan.Scanner23.__init__(self, show_asm)
+        scan.Scanner23.__init__(self, show_asm=False)
         self.opc = opcode_22
         self.opname = opcode_22.opname
         self.version = 2.2
-        self.genexpr_name = '<generator expression>';
+        self.genexpr_name = '<generator expression>'
         self.parent_ingest = self.ingest
         self.ingest = self.ingest22
         return

uncompyle6/scanners/scanner23.py

@@ -2,9 +2,8 @@
 """
 Python 2.3 bytecode scanner/deparser
 
-This overlaps Python's 2.3's dis module, but it can be run from
-Python 3 and other versions of Python. Also, we save token
-information for later use in deparsing.
+This massages tokenized 2.3 bytecode to make it more amenable for
+grammar parsing.
 """
 
 import uncompyle6.scanners.scanner24 as scan
@@ -13,12 +12,12 @@ import uncompyle6.scanners.scanner24 as scan
 from xdis.opcodes import opcode_23
 JUMP_OPs = opcode_23.JUMP_OPs
 
-# We base this off of 2.5 instead of the other way around
+# We base this off of 2.4 instead of the other way around
 # because we cleaned things up this way.
 # The history is that 2.7 support is the cleanest,
 # then from that we got 2.6 and so on.
 class Scanner23(scan.Scanner24):
-    def __init__(self, show_asm):
+    def __init__(self, show_asm=False):
         scan.Scanner24.__init__(self, show_asm)
         self.opc = opcode_23
         self.opname = opcode_23.opname

uncompyle6/scanners/scanner24.py

@@ -2,9 +2,8 @@
 """
 Python 2.4 bytecode scanner/deparser
 
-This overlaps Python's 2.4's dis module, but it can be run from
-Python 3 and other versions of Python. Also, we save token
-information for later use in deparsing.
+This massages tokenized 2.7 bytecode to make it more amenable for
+grammar parsing.
 """
 
 import uncompyle6.scanners.scanner25 as scan
@@ -18,7 +17,7 @@ JUMP_OPs = opcode_24.JUMP_OPs
 # The history is that 2.7 support is the cleanest,
 # then from that we got 2.6 and so on.
 class Scanner24(scan.Scanner25):
-    def __init__(self, show_asm):
+    def __init__(self, show_asm=False):
         scan.Scanner25.__init__(self, show_asm)
         # These are the only differences in initialization between
         # 2.4, 2.5 and 2.6

uncompyle6/scanners/scanner25.py

@@ -18,7 +18,7 @@ JUMP_OPs = opcode_25.JUMP_OPs
 # The history is that 2.7 support is the cleanest,
 # then from that we got 2.6 and so on.
 class Scanner25(scan.Scanner26):
-    def __init__(self, show_asm):
+    def __init__(self, show_asm=False):
         # There are no differences in initialization between
         # 2.5 and 2.6
         self.opc = opcode_25

uncompyle6/scanners/scanner26.py

@@ -86,7 +86,9 @@ class Scanner26(scan.Scanner2):
         cause specific rules for the specific number of arguments they take.
         """
 
-        show_asm = self.show_asm if not show_asm else show_asm
+        if not show_asm:
+            show_asm = self.show_asm
+
         # show_asm = 'after'
         if show_asm in ('both', 'before'):
             from xdis.bytecode import Bytecode
@@ -94,35 +96,32 @@ class Scanner26(scan.Scanner2):
             for instr in bytecode.get_instructions(co):
                 print(instr._disassemble())
 
-        # from xdis.bytecode import Bytecode
-        # bytecode = Bytecode(co, self.opc)
-        # for instr in bytecode.get_instructions(co):
-        #     print(instr._disassemble())
-
         # Container for tokens
         tokens = []
 
         customize = {}
+        if self.is_pypy:
+            customize['PyPy'] = 1
+
         Token = self.Token # shortcut
 
-        n = self.setup_code(co)
+        codelen = self.setup_code(co)
 
-        self.build_lines_data(co, n)
-        self.build_prev_op(n)
+        self.build_lines_data(co, codelen)
+        self.build_prev_op(codelen)
 
         free, names, varnames = self.unmangle_code_names(co, classname)
         self.names = names
 
-        codelen = len(self.code)
-
         # Scan for assertions. Later we will
         # turn 'LOAD_GLOBAL' to 'LOAD_ASSERT'.
         # 'LOAD_ASSERT' is used in assert statements.
         self.load_asserts = set()
-        for i in self.op_range(0, n):
-            # We need to detect the difference between
-            # "raise AssertionError" and
-            # "assert"
+        for i in self.op_range(0, codelen):
+            # We need to detect the difference between:
+            #   raise AssertionError
+            #  and
+            #   assert ...
             if (self.code[i] == self.opc.JUMP_IF_TRUE and
                 i + 4 < codelen and
                 self.code[i+3] == self.opc.POP_TOP and
@@ -161,12 +160,20 @@ class Scanner26(scan.Scanner2):
                 # we sort them). That way, specific COME_FROM tags will match up
                 # properly. For example, a "loop" with an "if" nested in it should have the
                 # "loop" tag last so the grammar rule matches that properly.
+                last_jump_offset = -1
                 for jump_offset  in sorted(jump_targets[offset], reverse=True):
-                    tokens.append(Token(
-                        'COME_FROM', None, repr(jump_offset),
-                        offset="%s_%d" % (offset, jump_idx),
-                        has_arg = True))
-                    jump_idx += 1
+                    if jump_offset != last_jump_offset:
+                        tokens.append(Token(
+                            'COME_FROM', None, repr(jump_offset),
+                            offset="%s_%d" % (offset, jump_idx),
+                            has_arg = True))
+                        jump_idx += 1
+                        last_jump_offset = jump_offset
+            elif offset in self.thens:
+                tokens.append(Token(
+                    'THEN', None, self.thens[offset],
+                    offset="%s_0" % offset,
+                    has_arg = True))
 
             has_arg = (op >= self.opc.HAVE_ARGUMENT)
             if has_arg:
@@ -240,16 +247,21 @@ class Scanner26(scan.Scanner2):
                 # boundaries The continue-type jumps help us get
                 # "continue" statements with would otherwise be turned
                 # into a "pass" statement because JUMPs are sometimes
-                # ignored in rules as just boundary overhead.
+                # ignored in rules as just boundary overhead.  In
+                # comprehensions we might sometimes classify JUMP_BACK
+                # as CONTINUE, but that's okay since we add a grammar
+                # rule for that.
                 target = self.get_target(offset)
                 if target <= offset:
+                    op_name = 'JUMP_BACK'
                     if (offset in self.stmts
                         and self.code[offset+3] not in (self.opc.END_FINALLY,
-                                                          self.opc.POP_BLOCK)
-                        and offset not in self.not_continue):
-                        op_name = 'CONTINUE'
+                                                          self.opc.POP_BLOCK)):
+                        if ((offset in self.linestartoffsets and
+                            tokens[-1].type == 'JUMP_BACK')
+                            or offset not in self.not_continue):
+                            op_name = 'CONTINUE'
                     else:
-                        op_name = 'JUMP_BACK'
                         # FIXME: this is a hack to catch stuff like:
                         #   if x: continue
                         # the "continue" is not on a new line.

uncompyle6/scanners/scanner27.py

@@ -7,8 +7,6 @@ grammar parsing.
 """
 
 
-from __future__ import print_function
-
 from uncompyle6.scanners.scanner2 import Scanner2
 
 from uncompyle6 import PYTHON3

uncompyle6/scanners/scanner3.py

@@ -20,9 +20,13 @@ For example:
 Finally we save token information.
 """
 
-from __future__ import print_function
+from uncompyle6 import PYTHON_VERSION
+
+if PYTHON_VERSION < 2.6:
+    from xdis.namedtuple25 import namedtuple
+else:
+    from collections import namedtuple
 
-from collections import namedtuple
 from array import array
 
 from uncompyle6.scanner import Scanner, op_has_argument
@@ -126,7 +130,12 @@ class Scanner3(Scanner):
 
         if is_pypy:
             varargs_ops.add(self.opc.CALL_METHOD)
+        if self.version >= 3.6:
+            varargs_ops.add(self.opc.BUILD_CONST_KEY_MAP)
+
         self.varargs_ops = frozenset(varargs_ops)
+        # FIXME: remove the above in favor of:
+        # self.varargs_ops = frozenset(self.opc.hasvargs)
 
     def opName(self, offset):
         return self.opc.opname[self.code[offset]]
@@ -147,7 +156,9 @@ class Scanner3(Scanner):
         cause specific rules for the specific number of arguments they take.
         """
 
-        show_asm = self.show_asm if not show_asm else show_asm
+        if not show_asm:
+            show_asm = self.show_asm
+
         # show_asm = 'after'
         if show_asm in ('both', 'before'):
             bytecode = Bytecode(co, self.opc)
@@ -159,7 +170,7 @@ class Scanner3(Scanner):
 
         customize = {}
         if self.is_pypy:
-            customize['PyPy'] = 1;
+            customize['PyPy'] = 1
 
         self.code = array('B', co.co_code)
         self.build_lines_data(co)
@@ -199,6 +210,7 @@ class Scanner3(Scanner):
         # Get jump targets
         # Format: {target offset: [jump offsets]}
         jump_targets = self.find_jump_targets(show_asm)
+        last_op_was_break = False
 
         for inst in bytecode:
 
@@ -321,11 +333,15 @@ class Scanner3(Scanner):
                 if target <= inst.offset:
                     next_opname = self.opname[self.code[inst.offset+3]]
                     if (inst.offset in self.stmts and
-                        next_opname not in ('END_FINALLY', 'POP_BLOCK',
+                        (self.version != 3.0 or (hasattr(inst, 'linestart'))) and
+                        (next_opname not in ('END_FINALLY', 'POP_BLOCK',
                                             # Python 3.0 only uses POP_TOP
-                                            'POP_TOP')
-                        and inst.offset not in self.not_continue):
-                        opname = 'CONTINUE'
+                                            'POP_TOP'))):
+                        if (self.version >= 3.4 or
+                            (inst.offset not in self.not_continue) or
+                            (tokens[-1].type == 'RETURN_VALUE')):
+                            opname = 'CONTINUE'
+                            pass
                     else:
                         opname = 'JUMP_BACK'
                         # FIXME: this is a hack to catch stuff like:
@@ -334,15 +350,21 @@ class Scanner3(Scanner):
                         # There are other situations where we don't catch
                         # CONTINUE as well.
                         if tokens[-1].type == 'JUMP_BACK' and tokens[-1].attr <= argval:
-                            # intern is used because we are changing the *previous* token
-                            tokens[-1].type = intern('CONTINUE')
-
+                            if tokens[-2].type == 'BREAK_LOOP':
+                                del tokens[-1]
+                            else:
+                                # intern is used because we are changing the *previous* token
+                                tokens[-1].type = intern('CONTINUE')
+                    if last_op_was_break and opname == 'CONTINUE':
+                        last_op_was_break = False
+                        continue
             elif op == self.opc.RETURN_VALUE:
                 if inst.offset in self.return_end_ifs:
                     opname = 'RETURN_END_IF'
             elif inst.offset in self.load_asserts:
                 opname = 'LOAD_ASSERT'
 
+            last_op_was_break = opname == 'BREAK_LOOP'
             tokens.append(
                 Token(
                     type_ = opname,
@@ -434,7 +456,7 @@ class Scanner3(Scanner):
         self.build_statement_indices()
         self.else_start = {}
 
-        # Containers filled by detect_structure()
+        # Containers filled by detect_control_flow()
         self.not_continue = set()
         self.return_end_ifs = set()
         self.setup_loop_targets = {}  # target given setup_loop offset
@@ -446,16 +468,20 @@ class Scanner3(Scanner):
 
             # Determine structures and fix jumps in Python versions
             # since 2.3
-            self.detect_structure(offset, targets)
+            self.detect_control_flow(offset, targets)
 
             has_arg = (op >= op3.HAVE_ARGUMENT)
             if has_arg:
                 label = self.fixed_jumps.get(offset)
-                oparg = code[offset+1] + code[offset+2] * 256
+                if self.version >= 3.6:
+                    oparg = code[offset+1]
+                else:
+                    oparg = code[offset+1] + code[offset+2] * 256
+                next_offset = offset + self.op_size(op)
 
                 if label is None:
                     if op in op3.hasjrel and op != self.opc.FOR_ITER:
-                        label = offset + 3 + oparg
+                        label = next_offset + oparg
                     elif op in op3.hasjabs:
                         if op in self.jump_if_pop:
                             if oparg > offset:
@@ -485,7 +511,7 @@ class Scanner3(Scanner):
         prelim = self.all_instr(start, end, self.statement_opcodes)
 
         # Initialize final container with statements with
-        # preliminnary data
+        # preliminary data
         stmts = self.stmts = set(prelim)
 
         # Same for opcode sequences
@@ -564,16 +590,16 @@ class Scanner3(Scanner):
         op = self.code[offset]
         if self.version  >= 3.6:
             target = self.code[offset+1]
-            if op in op3.hasjrel:
+            if op in self.opc.hasjrel:
                 target += offset + 2
         else:
             target = self.code[offset+1] + self.code[offset+2] * 256
-            if op in op3.hasjrel:
+            if op in self.opc.hasjrel:
                 target += offset + 3
 
         return target
 
-    def detect_structure(self, offset, targets):
+    def detect_control_flow(self, offset, targets):
         """
         Detect structures and their boundaries to fix optimized jumps
         in python2.3+
@@ -600,13 +626,15 @@ class Scanner3(Scanner):
                 parent = struct
 
         if op == self.opc.SETUP_LOOP:
-
             # We categorize loop types: 'for', 'while', 'while 1' with
             # possibly suffixes '-loop' and '-else'
             # Try to find the jump_back instruction of the loop.
             # It could be a return instruction.
 
-            start = offset+3
+            if self.version <= 3.5:
+                start = offset+3
+            else:
+                start = offset+2
             target = self.get_target(offset)
             end    = self.restrict_to_parent(target, parent)
             self.setup_loop_targets[offset] = target
@@ -616,22 +644,35 @@ class Scanner3(Scanner):
                 self.fixed_jumps[offset] = end
             (line_no, next_line_byte) = self.lines[offset]
             jump_back = self.last_instr(start, end, self.opc.JUMP_ABSOLUTE,
-                                          next_line_byte, False)
+                                            next_line_byte, False)
 
-            if jump_back and jump_back != self.prev_op[end] and self.is_jump_forward(jump_back+3):
-                if (code[self.prev_op[end]] == self.opc.RETURN_VALUE
-                    or (code[self.prev_op[end]] == self.opc.POP_BLOCK
-                         and code[self.prev_op[self.prev_op[end]]] == self.opc.RETURN_VALUE)):
+            if jump_back:
+                jump_forward_offset = jump_back+3
+            else:
+                jump_forward_offset = None
+
+            return_val_offset1 = self.prev[self.prev[end]]
+
+            if (jump_back and jump_back != self.prev_op[end]
+                and self.is_jump_forward(jump_forward_offset)):
+                if (code[self.prev_op[end]] == self.opc.RETURN_VALUE or
+                    (code[self.prev_op[end]] == self.opc.POP_BLOCK
+                     and code[return_val_offset1] == self.opc.RETURN_VALUE)):
                     jump_back = None
-            if not jump_back: # loop suite ends in return. wtf right?
-                jump_back = self.last_instr(start, end, self.opc.RETURN_VALUE) + 1
+            if not jump_back:
+                jump_back = self.last_instr(start, end, self.opc.RETURN_VALUE)
                 if not jump_back:
                     return
+
+                jump_back += 2
+                if_offset = None
                 if code[self.prev_op[next_line_byte]] not in POP_JUMP_TF:
-                    loop_type = 'for'
-                else:
+                    if_offset = self.prev[next_line_byte]
+                if if_offset:
                     loop_type = 'while'
-                    self.ignore_if.add(self.prev_op[next_line_byte])
+                    self.ignore_if.add(if_offset)
+                else:
+                    loop_type = 'for'
                 target = next_line_byte
                 end = jump_back + 3
             else:
@@ -645,6 +686,7 @@ class Scanner3(Scanner):
                 elif target < offset:
                     self.fixed_jumps[offset] = jump_back+4
                     end = jump_back+4
+
                 target = self.get_target(jump_back)
 
                 if code[target] in (self.opc.FOR_ITER, self.opc.GET_ITER):
@@ -652,6 +694,7 @@ class Scanner3(Scanner):
                 else:
                     loop_type = 'while'
                     test = self.prev_op[next_line_byte]
+
                     if test == offset:
                         loop_type = 'while 1'
                     elif self.code[test] in op3.hasjabs+op3.hasjrel:
@@ -692,38 +735,40 @@ class Scanner3(Scanner):
                                      'end': prev_op[target]})
                 return
 
-            # Is it an "and" inside an "if" block
+            # The op offset just before the target jump offset is important
+            # in making a determination of what we have. Save that.
+            pre_rtarget = prev_op[rtarget]
+
+            # Is it an "and" inside an "if" or "while" block
             if op == self.opc.POP_JUMP_IF_FALSE:
+
                 # Search for another POP_JUMP_IF_FALSE targetting the same op,
                 # in current statement, starting from current offset, and filter
                 # everything inside inner 'or' jumps and midline ifs
                 match = self.rem_or(start, self.next_stmt[offset],
                                     self.opc.POP_JUMP_IF_FALSE, target)
-                ## We can't remove mid-line ifs because line structures have changed
-                ## from restructBytecode().
-                ##  match = self.remove_mid_line_ifs(match)
 
                 # If we still have any offsets in set, start working on it
                 if match:
-                    is_jump_forward = self.is_jump_forward(prev_op[rtarget])
-                    if (is_jump_forward and prev_op[rtarget] not in self.stmts and
-                        self.restrict_to_parent(self.get_target(prev_op[rtarget]), parent) == rtarget):
-                        if (code[prev_op[prev_op[rtarget]]] == self.opc.JUMP_ABSOLUTE
+                    is_jump_forward = self.is_jump_forward(pre_rtarget)
+                    if (is_jump_forward and pre_rtarget not in self.stmts and
+                        self.restrict_to_parent(self.get_target(pre_rtarget), parent) == rtarget):
+                        if (code[prev_op[pre_rtarget]] == self.opc.JUMP_ABSOLUTE
                             and self.remove_mid_line_ifs([offset]) and
-                            target == self.get_target(prev_op[prev_op[rtarget]]) and
-                            (prev_op[prev_op[rtarget]] not in self.stmts or
-                             self.get_target(prev_op[prev_op[rtarget]]) > prev_op[prev_op[rtarget]]) and
-                            1 == len(self.remove_mid_line_ifs(self.rem_or(start, prev_op[prev_op[rtarget]], POP_JUMP_TF, target)))):
+                            target == self.get_target(prev_op[pre_rtarget]) and
+                            (prev_op[pre_rtarget] not in self.stmts or
+                             self.get_target(prev_op[pre_rtarget]) > prev_op[pre_rtarget]) and
+                            1 == len(self.remove_mid_line_ifs(self.rem_or(start, prev_op[pre_rtarget], POP_JUMP_TF, target)))):
                             pass
-                        elif (code[prev_op[prev_op[rtarget]]] == self.opc.RETURN_VALUE
+                        elif (code[prev_op[pre_rtarget]] == self.opc.RETURN_VALUE
                               and self.remove_mid_line_ifs([offset]) and
-                              1 == (len(set(self.remove_mid_line_ifs(self.rem_or(start, prev_op[prev_op[rtarget]],
+                              1 == (len(set(self.remove_mid_line_ifs(self.rem_or(start, prev_op[pre_rtarget],
                                                                                  POP_JUMP_TF, target))) |
-                                    set(self.remove_mid_line_ifs(self.rem_or(start, prev_op[prev_op[rtarget]],
+                                    set(self.remove_mid_line_ifs(self.rem_or(start, prev_op[pre_rtarget],
                                                                              (self.opc.POP_JUMP_IF_FALSE,
                                                                               self.opc.POP_JUMP_IF_TRUE,
                                                                               self.opc.JUMP_ABSOLUTE),
-                                                                             prev_op[rtarget], True)))))):
+                                                                             pre_rtarget, True)))))):
                             pass
                         else:
                             fix = None
@@ -751,7 +796,7 @@ class Scanner3(Scanner):
                     if code[prev_op[next]] == self.opc.POP_JUMP_IF_FALSE:
                         if (code[next] == self.opc.JUMP_FORWARD
                             or target != rtarget
-                            or code[prev_op[prev_op[rtarget]]] not in
+                            or code[prev_op[pre_rtarget]] not in
                             (self.opc.JUMP_ABSOLUTE, self.opc.RETURN_VALUE)):
                             self.fixed_jumps[offset] = prev_op[next]
                             return
@@ -764,14 +809,14 @@ class Scanner3(Scanner):
             if offset in self.ignore_if:
                 return
 
-            if (code[prev_op[rtarget]] == self.opc.JUMP_ABSOLUTE and
-                prev_op[rtarget] in self.stmts and
-                prev_op[rtarget] != offset and
-                prev_op[prev_op[rtarget]] != offset and
+            if (code[pre_rtarget] == self.opc.JUMP_ABSOLUTE and
+                pre_rtarget in self.stmts and
+                pre_rtarget != offset and
+                prev_op[pre_rtarget] != offset and
                 not (code[rtarget] == self.opc.JUMP_ABSOLUTE and
                      code[rtarget+3] == self.opc.POP_BLOCK and
-                     code[prev_op[prev_op[rtarget]]] != self.opc.JUMP_ABSOLUTE)):
-                rtarget = prev_op[rtarget]
+                     code[prev_op[pre_rtarget]] != self.opc.JUMP_ABSOLUTE)):
+                rtarget = pre_rtarget
 
             # Does the "jump if" jump beyond a jump op?
             # That is, we have something like:
@@ -787,12 +832,11 @@ class Scanner3(Scanner):
             # There are other contexts we may need to consider
             # like whether the target is "END_FINALLY"
             # or if the condition jump is to a forward location
-            if self.is_jump_forward(prev_op[rtarget]):
-                rrtarget = prev_op[rtarget]
-                if_end = self.get_target(rrtarget)
+            if self.is_jump_forward(pre_rtarget):
+                if_end = self.get_target(pre_rtarget)
 
                 # If the jump target is back, we are looping
-                if (if_end < rrtarget and
+                if (if_end < pre_rtarget and
                     (code[prev_op[if_end]] == self.opc.SETUP_LOOP)):
                     if (if_end > start):
                         return
@@ -801,19 +845,26 @@ class Scanner3(Scanner):
 
                 self.structs.append({'type': 'if-then',
                                      'start': start,
-                                     'end': prev_op[rtarget]})
-                self.not_continue.add(prev_op[rtarget])
+                                     'end': pre_rtarget})
+                self.not_continue.add(pre_rtarget)
 
                 if rtarget < end and (
                         code[rtarget] not in (self.opc.END_FINALLY,
                                               self.opc.JUMP_ABSOLUTE) and
-                        code[prev_op[rrtarget]] not in (self.opc.POP_EXCEPT,
+                        code[prev_op[pre_rtarget]] not in (self.opc.POP_EXCEPT,
                                                         self.opc.END_FINALLY)):
                     self.structs.append({'type': 'else',
                                          'start': rtarget,
                                          'end': end})
                     self.else_start[rtarget] = end
-            elif code[prev_op[rtarget]] == self.opc.RETURN_VALUE:
+            elif self.is_jump_back(pre_rtarget):
+                if_end = rtarget
+                self.structs.append({'type': 'if-then',
+                                     'start': start,
+                                     'end': pre_rtarget})
+                self.not_continue.add(pre_rtarget)
+            elif code[pre_rtarget] in (self.opc.RETURN_VALUE,
+                                       self.opc.BREAK_LOOP):
                 self.structs.append({'type': 'if-then',
                                      'start': start,
                                      'end': rtarget})
@@ -843,8 +894,24 @@ class Scanner3(Scanner):
                                 return
                             pass
                     pass
-                self.return_end_ifs.add(prev_op[rtarget])
+                if code[pre_rtarget] == self.opc.RETURN_VALUE:
+                    self.return_end_ifs.add(pre_rtarget)
+                else:
+                    self.fixed_jumps[offset] = rtarget
+                    self.not_continue.add(pre_rtarget)
+            else:
+                # For now, we'll only tag forward jump.
+                if rtarget > offset:
+                    self.fixed_jumps[offset] = rtarget
 
+        elif op == self.opc.SETUP_EXCEPT:
+            target = self.get_target(offset)
+            end    = self.restrict_to_parent(target, parent)
+            self.fixed_jumps[offset] = end
+        elif op == self.opc.SETUP_FINALLY:
+            target = self.get_target(offset)
+            end    = self.restrict_to_parent(target, parent)
+            self.fixed_jumps[offset] = end
         elif op in self.jump_if_pop:
             target = self.get_target(offset)
             if target > offset:
@@ -882,6 +949,16 @@ class Scanner3(Scanner):
                 pass
         return
 
+    def is_jump_back(self, offset):
+        """
+        Return True if the code at offset is some sort of jump back.
+        That is, it is ether "JUMP_FORWARD" or an absolute jump that
+        goes forward.
+        """
+        if self.code[offset] != self.opc.JUMP_ABSOLUTE:
+            return False
+        return offset > self.get_target(offset)
+
     def next_except_jump(self, start):
         """
         Return the next jump that was generated by an except SomeException:

uncompyle6/scanners/scanner30.py

@@ -1,4 +1,4 @@
-#  Copyright (c) 2016 by Rocky Bernstein
+#  Copyright (c) 2016, 2017 by Rocky Bernstein
 """
 Python 3.0 bytecode scanner/deparser
 
@@ -6,12 +6,12 @@ This sets up opcodes Python's 3.0 and calls a generalized
 scanner routine for Python 3.
 """
 
-from __future__ import print_function
-
 # bytecode verification, verify(), uses JUMP_OPs from here
 from xdis.opcodes import opcode_30 as opc
 JUMP_OPs = map(lambda op: opc.opname[op], opc.hasjrel + opc.hasjabs)
 
+JUMP_TF = frozenset([opc.JUMP_IF_FALSE, opc.JUMP_IF_TRUE])
+
 from uncompyle6.scanners.scanner3 import Scanner3
 class Scanner30(Scanner3):
 
@@ -20,6 +20,373 @@ class Scanner30(Scanner3):
         return
     pass
 
+    def detect_control_flow(self, offset, targets):
+        """
+        Detect structures and their boundaries to fix optimized jumps
+        Python 3.0 is more like Python 2.6 than it is Python 3.x.
+        So we have a special routine here.
+        """
+
+        code = self.code
+        op = code[offset]
+
+        # Detect parent structure
+        parent = self.structs[0]
+        start = parent['start']
+        end = parent['end']
+
+        # Pick inner-most parent for our offset
+        for struct in self.structs:
+            current_start = struct['start']
+            current_end   = struct['end']
+            if ((current_start <= offset < current_end)
+                and (current_start >= start and current_end <= end)):
+                start = current_start
+                end = current_end
+                parent = struct
+
+        if op == self.opc.SETUP_LOOP:
+            # We categorize loop types: 'for', 'while', 'while 1' with
+            # possibly suffixes '-loop' and '-else'
+            # Try to find the jump_back instruction of the loop.
+            # It could be a return instruction.
+
+            start = offset+3
+            target = self.get_target(offset)
+            end    = self.restrict_to_parent(target, parent)
+            self.setup_loop_targets[offset] = target
+            self.setup_loops[target] = offset
+
+            if target != end:
+                self.fixed_jumps[offset] = end
+
+            (line_no, next_line_byte) = self.lines[offset]
+            jump_back = self.last_instr(start, end, self.opc.JUMP_ABSOLUTE,
+                                            next_line_byte, False)
+
+            if jump_back:
+                jump_forward_offset = jump_back+3
+            else:
+                jump_forward_offset = None
+
+            return_val_offset1 = self.prev[self.prev[end]]
+
+            if (jump_back and jump_back != self.prev_op[end]
+                and self.is_jump_forward(jump_forward_offset)):
+                if (code[self.prev_op[end]] == self.opc.RETURN_VALUE or
+                    (code[self.prev_op[end]] == self.opc.POP_BLOCK
+                     and code[return_val_offset1] == self.opc.RETURN_VALUE)):
+                    jump_back = None
+            if not jump_back:
+                # loop suite ends in return
+                jump_back = self.last_instr(start, end, self.opc.RETURN_VALUE)
+                if not jump_back:
+                    return
+
+                jump_back += 2
+                if_offset = None
+                if code[self.prev_op[next_line_byte]] not in JUMP_TF:
+                    if_offset = self.prev[next_line_byte]
+                if if_offset:
+                    loop_type = 'while'
+                    self.ignore_if.add(if_offset)
+                else:
+                    loop_type = 'for'
+                target = next_line_byte
+                end = jump_back + 3
+            else:
+                if self.get_target(jump_back) >= next_line_byte:
+                    jump_back = self.last_instr(start, end, self.opc.JUMP_ABSOLUTE, start, False)
+                if end > jump_back+4 and self.is_jump_forward(end):
+                    if self.is_jump_forward(jump_back+4):
+                        if self.get_target(jump_back+4) == self.get_target(end):
+                            self.fixed_jumps[offset] = jump_back+4
+                            end = jump_back+4
+                elif target < offset:
+                    self.fixed_jumps[offset] = jump_back+4
+                    end = jump_back+4
+
+                target = self.get_target(jump_back)
+
+                if code[target] in (self.opc.FOR_ITER, self.opc.GET_ITER):
+                    loop_type = 'for'
+                else:
+                    loop_type = 'while'
+                    test = self.prev_op[next_line_byte]
+
+                    if test == offset:
+                        loop_type = 'while 1'
+                    elif self.code[test] in opc.hasjabs+opc.hasjrel:
+                        self.ignore_if.add(test)
+                        test_target = self.get_target(test)
+                        if test_target > (jump_back+3):
+                            jump_back = test_target
+                self.not_continue.add(jump_back)
+            self.loops.append(target)
+            self.structs.append({'type': loop_type + '-loop',
+                                   'start': target,
+                                   'end':   jump_back})
+            if jump_back+3 != end:
+                self.structs.append({'type': loop_type + '-else',
+                                       'start': jump_back+3,
+                                       'end':   end})
+        elif op in JUMP_TF:
+            start = offset + self.op_size(op)
+            target = self.get_target(offset)
+            rtarget = self.restrict_to_parent(target, parent)
+            prev_op = self.prev_op
+
+            # Do not let jump to go out of parent struct bounds
+            if target != rtarget and parent['type'] == 'and/or':
+                self.fixed_jumps[offset] = rtarget
+                return
+
+            # Does this jump to right after another conditional jump that is
+            # not myself?  If so, it's part of a larger conditional.
+            # rocky: if we have a conditional jump to the next instruction, then
+            # possibly I am "skipping over" a "pass" or null statement.
+
+            if ((code[prev_op[target]] in self.pop_jump_if_pop) and
+                (target > offset) and prev_op[target] != offset):
+                self.fixed_jumps[offset] = prev_op[target]
+                self.structs.append({'type': 'and/or',
+                                     'start': start,
+                                     'end': prev_op[target]})
+                return
+
+            # The op offset just before the target jump offset is important
+            # in making a determination of what we have. Save that.
+            pre_rtarget = prev_op[rtarget]
+
+            # Is it an "and" inside an "if" or "while" block
+            if op == opc.JUMP_IF_FALSE:
+
+                # Search for another JUMP_IF_FALSE targetting the same op,
+                # in current statement, starting from current offset, and filter
+                # everything inside inner 'or' jumps and midline ifs
+                match = self.rem_or(start, self.next_stmt[offset],
+                                    opc.JUMP_IF_FALSE, target)
+
+                # If we still have any offsets in set, start working on it
+                if match:
+                    is_jump_forward = self.is_jump_forward(pre_rtarget)
+                    if (is_jump_forward and pre_rtarget not in self.stmts and
+                        self.restrict_to_parent(self.get_target(pre_rtarget), parent) == rtarget):
+                        if (code[prev_op[pre_rtarget]] == self.opc.JUMP_ABSOLUTE
+                            and self.remove_mid_line_ifs([offset]) and
+                            target == self.get_target(prev_op[pre_rtarget]) and
+                            (prev_op[pre_rtarget] not in self.stmts or
+                             self.get_target(prev_op[pre_rtarget]) > prev_op[pre_rtarget]) and
+                            1 == len(self.remove_mid_line_ifs(self.rem_or(start, prev_op[pre_rtarget], JUMP_TF, target)))):
+                            pass
+                        elif (code[prev_op[pre_rtarget]] == self.opc.RETURN_VALUE
+                              and self.remove_mid_line_ifs([offset]) and
+                              1 == (len(set(self.remove_mid_line_ifs(self.rem_or(start, prev_op[pre_rtarget],
+                                                                                 JUMP_TF, target))) |
+                                    set(self.remove_mid_line_ifs(self.rem_or(start, prev_op[pre_rtarget],
+                                                                             (opc.JUMP_IF_FALSE,
+                                                                              opc.JUMP_IF_TRUE,
+                                                                              opc.JUMP_ABSOLUTE),
+                                                                             pre_rtarget, True)))))):
+                            pass
+                        else:
+                            fix = None
+                            jump_ifs = self.all_instr(start, self.next_stmt[offset],
+                                                      opc.JUMP_IF_FALSE)
+                            last_jump_good = True
+                            for j in jump_ifs:
+                                if target == self.get_target(j):
+                                    if self.lines[j].next == j + 3 and last_jump_good:
+                                        fix = j
+                                        break
+                                else:
+                                    last_jump_good = False
+                            self.fixed_jumps[offset] = fix or match[-1]
+                            return
+                    else:
+                        self.fixed_jumps[offset] = match[-1]
+                        return
+            # op == JUMP_IF_TRUE
+            else:
+                next = self.next_stmt[offset]
+                if prev_op[next] == offset:
+                    pass
+                elif self.is_jump_forward(next) and target == self.get_target(next):
+                    if code[prev_op[next]] == opc.JUMP_IF_FALSE:
+                        if (code[next] == self.opc.JUMP_FORWARD
+                            or target != rtarget
+                            or code[prev_op[pre_rtarget]] not in
+                            (self.opc.JUMP_ABSOLUTE, self.opc.RETURN_VALUE)):
+                            self.fixed_jumps[offset] = prev_op[next]
+                            return
+                elif (code[next] == self.opc.JUMP_ABSOLUTE and self.is_jump_forward(target) and
+                      self.get_target(target) == self.get_target(next)):
+                    self.fixed_jumps[offset] = prev_op[next]
+                    return
+
+            # Don't add a struct for a while test, it's already taken care of
+            if offset in self.ignore_if:
+                return
+
+            if (code[pre_rtarget] == self.opc.JUMP_ABSOLUTE and
+                pre_rtarget in self.stmts and
+                pre_rtarget != offset and
+                prev_op[pre_rtarget] != offset and
+                not (code[rtarget] == self.opc.JUMP_ABSOLUTE and
+                     code[rtarget+3] == self.opc.POP_BLOCK and
+                     code[prev_op[pre_rtarget]] != self.opc.JUMP_ABSOLUTE)):
+                rtarget = pre_rtarget
+
+            # Does the "jump if" jump beyond a jump op?
+            # That is, we have something like:
+            #  JUMP_IF_FALSE HERE
+            #  ...
+            # JUMP_FORWARD
+            # HERE:
+            #
+            # If so, this can be block inside an "if" statement
+            # or a conditional assignment like:
+            #   x = 1 if x else 2
+            #
+            # There are other contexts we may need to consider
+            # like whether the target is "END_FINALLY"
+            # or if the condition jump is to a forward location
+            if self.is_jump_forward(pre_rtarget):
+                if_end = self.get_target(pre_rtarget)
+
+                # If the jump target is back, we are looping
+                if (if_end < pre_rtarget and
+                    (code[prev_op[if_end]] == self.opc.SETUP_LOOP)):
+                    if (if_end > start):
+                        return
+
+                end = self.restrict_to_parent(if_end, parent)
+
+                self.structs.append({'type': 'if-then',
+                                     'start': start,
+                                     'end': pre_rtarget})
+                self.not_continue.add(pre_rtarget)
+
+                # if rtarget < end and (
+                #         code[rtarget] not in (self.opc.END_FINALLY,
+                #                               self.opc.JUMP_ABSOLUTE) and
+                #         code[prev_op[pre_rtarget]] not in (self.opc.POP_EXCEPT,
+                #                                         self.opc.END_FINALLY)):
+                #     self.structs.append({'type': 'else',
+                #                          'start': rtarget,
+                #                          'end': end})
+                #     self.else_start[rtarget] = end
+            elif self.is_jump_back(pre_rtarget):
+                if_end = rtarget
+                self.structs.append({'type': 'if-then',
+                                     'start': start,
+                                     'end': pre_rtarget})
+                self.not_continue.add(pre_rtarget)
+            elif code[pre_rtarget] in (self.opc.RETURN_VALUE,
+                                       self.opc.BREAK_LOOP):
+                self.structs.append({'type': 'if-then',
+                                     'start': start,
+                                     'end': rtarget})
+                # It is important to distingish if this return is inside some sort
+                # except block return
+                jump_prev = prev_op[offset]
+                if self.is_pypy and code[jump_prev] == self.opc.COMPARE_OP:
+                    if self.opc.cmp_op[code[jump_prev+1]] == 'exception match':
+                        return
+                if self.version >= 3.5:
+                    # Python 3.5 may remove as dead code a JUMP
+                    # instruction after a RETURN_VALUE. So we check
+                    # based on seeing SETUP_EXCEPT various places.
+                    if code[rtarget] == self.opc.SETUP_EXCEPT:
+                        return
+                    # Check that next instruction after pops and jump is
+                    # not from SETUP_EXCEPT
+                    next_op = rtarget
+                    if code[next_op] == self.opc.POP_BLOCK:
+                        next_op += self.op_size(self.code[next_op])
+                    if code[next_op] == self.opc.JUMP_ABSOLUTE:
+                        next_op += self.op_size(self.code[next_op])
+                    if next_op in targets:
+                        for try_op in targets[next_op]:
+                            come_from_op = code[try_op]
+                            if come_from_op == self.opc.SETUP_EXCEPT:
+                                return
+                            pass
+                    pass
+                if code[pre_rtarget] == self.opc.RETURN_VALUE:
+                    self.return_end_ifs.add(pre_rtarget)
+                else:
+                    self.fixed_jumps[offset] = rtarget
+                    self.not_continue.add(pre_rtarget)
+
+
+        elif op == self.opc.SETUP_EXCEPT:
+            target = self.get_target(offset)
+            end    = self.restrict_to_parent(target, parent)
+            self.fixed_jumps[offset] = end
+        elif op == self.opc.SETUP_FINALLY:
+            target = self.get_target(offset)
+            end    = self.restrict_to_parent(target, parent)
+            self.fixed_jumps[offset] = end
+        elif op in self.jump_if_pop:
+            target = self.get_target(offset)
+            if target > offset:
+                unop_target = self.last_instr(offset, target, self.opc.JUMP_FORWARD, target)
+                if unop_target and code[unop_target+3] != self.opc.ROT_TWO:
+                    self.fixed_jumps[offset] = unop_target
+                else:
+                    self.fixed_jumps[offset] = self.restrict_to_parent(target, parent)
+                    pass
+                pass
+        elif self.version >= 3.5:
+            # 3.5+ has Jump optimization which too often causes RETURN_VALUE to get
+            # misclassified as RETURN_END_IF. Handle that here.
+            # In RETURN_VALUE, JUMP_ABSOLUTE, RETURN_VALUE is never RETURN_END_IF
+            if op == self.opc.RETURN_VALUE:
+                if (offset+1 < len(code) and code[offset+1] == self.opc.JUMP_ABSOLUTE and
+                    offset in self.return_end_ifs):
+                    self.return_end_ifs.remove(offset)
+                    pass
+                pass
+            elif op == self.opc.JUMP_FORWARD:
+                # If we have:
+                #   JUMP_FORWARD x, [non-jump, insns], RETURN_VALUE, x:
+                # then RETURN_VALUE is not RETURN_END_IF
+                rtarget = self.get_target(offset)
+                rtarget_prev = self.prev[rtarget]
+                if (code[rtarget_prev] == self.opc.RETURN_VALUE and
+                    rtarget_prev in self.return_end_ifs):
+                    i = rtarget_prev
+                    while i != offset:
+                        if code[i] in [opc.JUMP_FORWARD, opc.JUMP_ABSOLUTE]:
+                            return
+                        i = self.prev[i]
+                    self.return_end_ifs.remove(rtarget_prev)
+                pass
+        return
+
+    def rem_or(self, start, end, instr, target=None, include_beyond_target=False):
+        """
+        Find offsets of all requested <instr> between <start> and <end>,
+        optionally <target>ing specified offset, and return list found
+        <instr> offsets which are not within any POP_JUMP_IF_TRUE jumps.
+        """
+        assert(start>=0 and end<=len(self.code) and start <= end)
+
+        # Find all offsets of requested instructions
+        instr_offsets = self.all_instr(start, end, instr, target, include_beyond_target)
+        # Get all JUMP_IF_TRUE (or) offsets
+        pjit_offsets = self.all_instr(start, end, opc.JUMP_IF_TRUE)
+        filtered = []
+        for pjit_offset in pjit_offsets:
+            pjit_tgt = self.get_target(pjit_offset) - 3
+            for instr_offset in instr_offsets:
+                if instr_offset <= pjit_offset or instr_offset >= pjit_tgt:
+                    filtered.append(instr_offset)
+            instr_offsets = filtered
+            filtered = []
+        return instr_offsets
+
 if __name__ == "__main__":
     from uncompyle6 import PYTHON_VERSION
     if PYTHON_VERSION == 3.0:

uncompyle6/scanners/scanner31.py

@@ -6,8 +6,6 @@ This sets up opcodes Python's 3.1 and calls a generalized
 scanner routine for Python 3.
 """
 
-from __future__ import print_function
-
 # bytecode verification, verify(), uses JUMP_OPs from here
 from xdis.opcodes import opcode_31 as opc
 JUMP_OPs = map(lambda op: opc.opname[op], opc.hasjrel + opc.hasjabs)

uncompyle6/scanners/scanner32.py

@@ -6,8 +6,6 @@ This sets up opcodes Python's 3.2 and calls a generalized
 scanner routine for Python 3.
 """
 
-from __future__ import print_function
-
 # bytecode verification, verify(), uses JUMP_OPs from here
 from xdis.opcodes import opcode_32 as opc
 JUMP_OPs = map(lambda op: opc.opname[op], opc.hasjrel + opc.hasjabs)

uncompyle6/scanners/scanner33.py

@@ -6,8 +6,6 @@ This sets up opcodes Python's 3.3 and calls a generalized
 scanner routine for Python 3.
 """
 
-from __future__ import print_function
-
 # bytecode verification, verify(), uses JUMP_OPs from here
 from xdis.opcodes import opcode_33 as opc
 JUMP_OPs = map(lambda op: opc.opname[op], opc.hasjrel + opc.hasjabs)

uncompyle6/scanners/scanner34.py

@@ -6,8 +6,6 @@ This sets up opcodes Python's 3.4 and calls a generalized
 scanner routine for Python 3.
 """
 
-from __future__ import print_function
-
 from xdis.opcodes import opcode_34 as opc
 
 # bytecode verification, verify(), uses JUMP_OPs from here

uncompyle6/scanners/scanner35.py

@@ -6,8 +6,6 @@ This sets up opcodes Python's 3.5 and calls a generalized
 scanner routine for Python 3.
 """
 
-from __future__ import print_function
-
 from uncompyle6.scanners.scanner3 import Scanner3
 
 # bytecode verification, verify(), uses JUMP_OPs from here

uncompyle6/scanners/scanner36.py

@@ -6,8 +6,6 @@ This sets up opcodes Python's 3.5 and calls a generalized
 scanner routine for Python 3.
 """
 
-from __future__ import print_function
-
 from uncompyle6.scanners.scanner3 import Scanner3
 
 # bytecode verification, verify(), uses JUMP_OPs from here

uncompyle6/scanners/tok.py

@@ -56,17 +56,24 @@ class Token:
         return self.format(line_prefix='')
 
     def format(self, line_prefix=''):
-        prefix = ('\n%s%4d  ' % (line_prefix, self.linestart)
-                  if self.linestart else (' ' * (6 + len(line_prefix))))
+        if self.linestart:
+            prefix = '\n%s%4d  ' % (line_prefix, self.linestart)
+        else:
+            prefix = ' ' * (6 + len(line_prefix))
         offset_opname = '%6s  %-17s' % (self.offset, self.type)
         if not self.has_arg:
             return "%s%s" % (prefix, offset_opname)
-        argstr = "%6d " % self.attr if isinstance(self.attr, int) else (' '*7)
+
+        if isinstance(self.attr, int):
+            argstr = "%6d " % self.attr
+        else:
+            argstr = ' '*7
         if self.pattr:
             pattr = self.pattr
             if self.opc:
                 if self.op in self.opc.hasjrel:
-                    pattr = "to " + self.pattr
+                    if not self.pattr.startswith('to '):
+                        pattr = "to " + self.pattr
                 elif self.op in self.opc.hasjabs:
                     self.pattr= str(self.pattr)
                     if not self.pattr.startswith('to '):

uncompyle6/semantics/consts.py

@@ -0,0 +1,347 @@
+#  Copyright (c) 2017 by Rocky Bernstein
+"""Constants used in pysource.py"""
+
+import re, sys
+from uncompyle6.parsers.astnode import AST
+from uncompyle6 import PYTHON3
+from uncompyle6.scanners.tok import Token, NoneToken
+
+if PYTHON3:
+    minint = -sys.maxsize-1
+    maxint = sys.maxsize
+else:
+    minint = -sys.maxint-1
+    maxint = sys.maxint
+
+
+LINE_LENGTH = 80
+
+# Some ASTs used for comparing code fragments (like 'return None' at
+# the end of functions).
+
+RETURN_LOCALS = AST('return_stmt',
+                    [ AST('ret_expr', [AST('expr', [ Token('LOAD_LOCALS') ])]),
+                      Token('RETURN_VALUE')])
+
+NONE = AST('expr', [ NoneToken ] )
+
+RETURN_NONE = AST('stmt',
+                  [ AST('return_stmt',
+                        [ NONE, Token('RETURN_VALUE')]) ])
+
+PASS = AST('stmts',
+           [ AST('sstmt',
+                 [ AST('stmt',
+                       [ AST('passstmt', [])])])])
+
+ASSIGN_DOC_STRING = lambda doc_string: \
+  AST('stmt',
+      [ AST('assign',
+            [ AST('expr', [ Token('LOAD_CONST', pattr=doc_string) ]),
+              AST('designator', [ Token('STORE_NAME', pattr='__doc__')])
+            ])])
+
+NAME_MODULE = AST('stmt',
+                [ AST('assign',
+                    [ AST('expr',
+                          [Token('LOAD_NAME', pattr='__name__', offset=0, has_arg=True)]),
+                      AST('designator',
+                          [ Token('STORE_NAME', pattr='__module__', offset=3, has_arg=True)])
+                      ])])
+
+# God intended \t, but Python has decided to use 4 spaces.
+# If you want real tabs, use Go.
+# TAB = '\t'
+TAB = ' ' * 4
+INDENT_PER_LEVEL = ' ' # additional intent per pretty-print level
+
+TABLE_R = {
+    'STORE_ATTR':	( '%c.%[1]{pattr}', 0),
+#   'STORE_SUBSCR':	( '%c[%c]', 0, 1 ),
+    'DELETE_ATTR':	( '%|del %c.%[-1]{pattr}\n', 0 ),
+#   'EXEC_STMT':	( '%|exec %c in %[1]C\n', 0, (0,maxint,', ') ),
+}
+
+TABLE_R0 = {
+#    'BUILD_LIST':	( '[%C]',      (0,-1,', ') ),
+#    'BUILD_TUPLE':	( '(%C)',      (0,-1,', ') ),
+#    'CALL_FUNCTION':	( '%c(%P)', 0, (1,-1,', ') ),
+}
+TABLE_DIRECT = {
+    'BINARY_ADD':	( '+' ,),
+    'BINARY_SUBTRACT':	( '-' ,),
+    'BINARY_MULTIPLY':	( '*' ,),
+    'BINARY_DIVIDE':	( '/' ,),
+    'BINARY_MATRIX_MULTIPLY':	( '@' ,),
+    'BINARY_TRUE_DIVIDE':	( '/' ,),   # Not in <= 2.1
+    'BINARY_FLOOR_DIVIDE':	( '//' ,),
+    'BINARY_MODULO':	( '%%',),
+    'BINARY_POWER':	( '**',),
+    'BINARY_LSHIFT':	( '<<',),
+    'BINARY_RSHIFT':	( '>>',),
+    'BINARY_AND':	( '&' ,),
+    'BINARY_OR':	( '|' ,),
+    'BINARY_XOR':	( '^' ,),
+    'INPLACE_ADD':	( '+=' ,),
+    'INPLACE_SUBTRACT':	( '-=' ,),
+    'INPLACE_MULTIPLY':	( '*=' ,),
+    'INPLACE_MATRIX_MULTIPLY':	( '@=' ,),
+    'INPLACE_DIVIDE':	( '/=' ,),
+    'INPLACE_TRUE_DIVIDE':	( '/=' ,),  # Not in <= 2.1; 2.6 generates INPLACE_DIVIDE only?
+    'INPLACE_FLOOR_DIVIDE':	( '//=' ,),
+    'INPLACE_MODULO':	( '%%=',),
+    'INPLACE_POWER':	( '**=',),
+    'INPLACE_LSHIFT':	( '<<=',),
+    'INPLACE_RSHIFT':	( '>>=',),
+    'INPLACE_AND':	( '&=' ,),
+    'INPLACE_OR':	( '|=' ,),
+    'INPLACE_XOR':	( '^=' ,),
+    'binary_expr':	( '%c %c %c', 0, -1, 1 ),
+
+    'UNARY_POSITIVE':	( '+',),
+    'UNARY_NEGATIVE':	( '-',),
+    'UNARY_INVERT':	( '~%c'),
+    'unary_expr':   ( '%c%c', 1, 0),
+
+    'unary_not':	( 'not %c', 0 ),
+    'unary_convert':	( '`%c`', 0 ),
+    'get_iter':	( 'iter(%c)', 0 ),
+    'slice0':		( '%c[:]', 0 ),
+    'slice1':		( '%c[%p:]', 0, (1, 100) ),
+    'slice2':		( '%c[:%p]', 0, (1, 100) ),
+    'slice3':		( '%c[%p:%p]', 0, (1, 100), (2, 100) ),
+
+    'IMPORT_FROM':	( '%{pattr}', ),
+    'load_attr':	( '%c.%[1]{pattr}', 0),
+    'LOAD_FAST':	( '%{pattr}', ),
+    'LOAD_NAME':	( '%{pattr}', ),
+    'LOAD_CLASSNAME':	( '%{pattr}', ),
+    'LOAD_GLOBAL':	( '%{pattr}', ),
+    'LOAD_DEREF':	( '%{pattr}', ),
+    'LOAD_LOCALS':	( 'locals()', ),
+    'LOAD_ASSERT':  ( '%{pattr}', ),
+#   'LOAD_CONST':	( '%{pattr}', ),	# handled by n_LOAD_CONST
+    'DELETE_FAST':	( '%|del %{pattr}\n', ),
+    'DELETE_NAME':	( '%|del %{pattr}\n', ),
+    'DELETE_GLOBAL':	( '%|del %{pattr}\n', ),
+    'delete_subscr':	( '%|del %c[%c]\n', 0, 1,),
+    'binary_subscr':	( '%c[%p]', 0, (1, 100)),
+    'binary_subscr2':	( '%c[%p]', 0, (1, 100)),
+    'store_subscr':	( '%c[%c]', 0, 1),
+    'STORE_FAST':	( '%{pattr}', ),
+    'STORE_NAME':	( '%{pattr}', ),
+    'STORE_GLOBAL':	( '%{pattr}', ),
+    'STORE_DEREF':	( '%{pattr}', ),
+    'unpack':		( '%C%,', (1, maxint, ', ') ),
+
+    # This nonterminal we create on the fly in semantic routines
+    'unpack_w_parens':	( '(%C%,)', (1, maxint, ', ') ),
+
+    'unpack_list':	( '[%C]', (1, maxint, ', ') ),
+    'build_tuple2':	( '%P', (0, -1, ', ', 100) ),
+
+    # 'list_compr':	( '[ %c ]', -2),	# handled by n_list_compr
+    'list_iter':	( '%c', 0),
+    'list_for':		( ' for %c in %c%c', 2, 0, 3 ),
+    'list_if':		( ' if %c%c', 0, 2 ),
+    'list_if_not':		( ' if not %p%c', (0, 22), 2 ),
+    'lc_body':		( '', ),	# ignore when recusing
+
+    'comp_iter':	( '%c', 0),
+    'comp_if':		( ' if %c%c', 0, 2 ),
+    'comp_ifnot':	( ' if not %p%c', (0, 22), 2 ),
+    'comp_body':	( '', ),	# ignore when recusing
+    'set_comp_body':    ( '%c', 0 ),
+    'gen_comp_body':    ( '%c', 0 ),
+    'dict_comp_body':   ( '%c:%c', 1, 0 ),
+
+    'assign':		( '%|%c = %p\n', -1, (0, 200) ),
+
+    # The 2nd parameter should have a = suffix.
+    # There is a rule with a 4th parameter "designator"
+    # which we don't use here.
+    'augassign1':	( '%|%c %c %c\n', 0, 2, 1),
+
+    'augassign2':	( '%|%c.%[2]{pattr} %c %c\n', 0, -3, -4),
+    'designList':	( '%c = %c', 0, -1 ),
+    'and':          	( '%c and %c', 0, 2 ),
+    'ret_and':        	( '%c and %c', 0, 2 ),
+    'and2':          	( '%c', 3 ),
+    'or':           	( '%c or %c', 0, 2 ),
+    'ret_or':           	( '%c or %c', 0, 2 ),
+    'conditional':  ( '%p if %p else %p', (2, 27), (0, 27), (4, 27)),
+    'ret_cond':     ( '%p if %p else %p', (2, 27), (0, 27), (-1, 27)),
+    'conditionalnot':  ( '%p if not %p else %p', (2, 27), (0, 22), (4, 27)),
+    'ret_cond_not':  ( '%p if not %p else %p', (2, 27), (0, 22), (-1, 27)),
+    'conditional_lambda':  ( '(%c if %c else %c)', 2, 0, 3),
+    'return_lambda':    ('%c', 0),
+    'compare':		( '%p %[-1]{pattr} %p', (0, 19), (1, 19) ),
+    'cmp_list':		( '%p %p', (0, 20), (1, 19)),
+    'cmp_list1':	( '%[3]{pattr} %p %p', (0, 19), (-2, 19)),
+    'cmp_list2':	( '%[1]{pattr} %p', (0, 19)),
+#   'classdef': 	(), # handled by n_classdef()
+    'funcdef':  	( '\n\n%|def %c\n', -2), # -2 to handle closures
+    'funcdefdeco':  	( '\n\n%c', 0),
+    'mkfuncdeco':  	( '%|@%c\n%c', 0, 1),
+    'mkfuncdeco0':  	( '%|def %c\n', 0),
+    'classdefdeco':  	( '\n\n%c', 0),
+    'classdefdeco1':  	( '%|@%c\n%c', 0, 1),
+    'kwarg':    	( '%[0]{pattr}=%c', 1),
+    'kwargs':    	( '%D', (0, maxint, ', ') ),
+
+    'assert_expr_or': ( '%c or %c', 0, 2 ),
+    'assert_expr_and':    ( '%c and %c', 0, 2 ),
+    'print_items_stmt': ( '%|print %c%c,\n', 0, 2),  # Python 2 only
+    'print_items_nl_stmt': ( '%|print %c%c\n', 0, 2),
+    'print_item':  ( ', %c', 0),
+    'print_nl':	( '%|print\n', ),
+    'print_to':		( '%|print >> %c, %c,\n', 0, 1 ),
+    'print_to_nl':	( '%|print >> %c, %c\n', 0, 1 ),
+    'print_nl_to':	( '%|print >> %c\n', 0 ),
+    'print_to_items':	( '%C', (0, 2, ', ') ),
+
+    'call_stmt':	( '%|%p\n', (0, 200)),
+    'break_stmt':	( '%|break\n', ),
+    'continue_stmt':	( '%|continue\n', ),
+
+    'raise_stmt0':	( '%|raise\n', ),
+    'raise_stmt1':	( '%|raise %c\n', 0),
+    'raise_stmt3':	( '%|raise %c, %c, %c\n', 0, 1, 2),
+#    'yield':	( 'yield %c', 0),
+#    'return_stmt':	( '%|return %c\n', 0),
+
+    'ifstmt':		( '%|if %c:\n%+%c%-', 0, 1 ),
+    'iflaststmt':		( '%|if %c:\n%+%c%-', 0, 1 ),
+    'iflaststmtl':		( '%|if %c:\n%+%c%-', 0, 1 ),
+    'testtrue':         ( 'not %p', (0, 22) ),
+
+    'ifelsestmt':	( '%|if %c:\n%+%c%-%|else:\n%+%c%-', 0, 1, 3 ),
+    'ifelsestmtc':	( '%|if %c:\n%+%c%-%|else:\n%+%c%-', 0, 1, 3 ),
+    'ifelsestmtl':	( '%|if %c:\n%+%c%-%|else:\n%+%c%-', 0, 1, 3 ),
+    'ifelifstmt':	( '%|if %c:\n%+%c%-%c', 0, 1, 3 ),
+    'elifelifstmt':	( '%|elif %c:\n%+%c%-%c', 0, 1, 3 ),
+    'elifstmt':		( '%|elif %c:\n%+%c%-', 0, 1 ),
+    'elifelsestmt':	( '%|elif %c:\n%+%c%-%|else:\n%+%c%-', 0, 1, 3 ),
+    'ifelsestmtr':	( '%|if %c:\n%+%c%-%|else:\n%+%c%-', 0, 1, 2 ),
+    'ifelsestmtr2':	( '%|if %c:\n%+%c%-%|else:\n%+%c%-\n\n', 0, 1, 3 ), # has COME_FROM
+    'elifelsestmtr':	( '%|elif %c:\n%+%c%-%|else:\n%+%c%-\n\n', 0, 1, 2 ),
+    'elifelsestmtr2':	( '%|elif %c:\n%+%c%-%|else:\n%+%c%-\n\n', 0, 1, 3 ), # has COME_FROM
+
+    'whileTruestmt':	( '%|while True:\n%+%c%-\n\n', 1 ),
+    'whilestmt':	( '%|while %c:\n%+%c%-\n\n', 1, 2 ),
+    'while1stmt':	( '%|while 1:\n%+%c%-\n\n', 1 ),
+    'while1elsestmt':  ( '%|while 1:\n%+%c%-%|else:\n%+%c%-\n\n', 1, -2 ),
+    'whileelsestmt':	( '%|while %c:\n%+%c%-%|else:\n%+%c%-\n\n', 1, 2, -2 ),
+    'whileelselaststmt':	( '%|while %c:\n%+%c%-%|else:\n%+%c%-', 1, 2, -2 ),
+    'forstmt':		( '%|for %c in %c:\n%+%c%-\n\n', 3, 1, 4 ),
+    'forelsestmt':	(
+        '%|for %c in %c:\n%+%c%-%|else:\n%+%c%-\n\n', 3, 1, 4, -2),
+    'forelselaststmt':	(
+        '%|for %c in %c:\n%+%c%-%|else:\n%+%c%-', 3, 1, 4, -2),
+    'forelselaststmtl':	(
+        '%|for %c in %c:\n%+%c%-%|else:\n%+%c%-\n\n', 3, 1, 4, -2),
+    'trystmt':		( '%|try:\n%+%c%-%c\n\n', 1, 3 ),
+    'tryelsestmt':	( '%|try:\n%+%c%-%c%|else:\n%+%c%-\n\n', 1, 3, 4 ),
+    'tryelsestmtc':	( '%|try:\n%+%c%-%c%|else:\n%+%c%-', 1, 3, 4 ),
+    'tryelsestmtl':	( '%|try:\n%+%c%-%c%|else:\n%+%c%-', 1, 3, 4 ),
+    'tf_trystmt':	( '%c%-%c%+', 1, 3 ),
+    'tf_tryelsestmt':	( '%c%-%c%|else:\n%+%c', 1, 3, 4 ),
+    'tryfinallystmt':	( '%|try:\n%+%c%-%|finally:\n%+%c%-\n\n', 1, 5 ),
+    'except':           ( '%|except:\n%+%c%-', 3 ),
+    'except_cond1':	( '%|except %c:\n', 1 ),
+    'except_suite':     ( '%+%c%-%C', 0, (1, maxint, '') ),
+    'except_suite_finalize':     ( '%+%c%-%C', 1, (3, maxint, '') ),
+    'passstmt':		( '%|pass\n', ),
+    'STORE_FAST':	( '%{pattr}', ),
+    'kv':		( '%c: %c', 3, 1 ),
+    'kv2':		( '%c: %c', 1, 2 ),
+    'mapexpr':		( '{%[1]C}', (0, maxint, ', ') ),
+    'importstmt': ( '%|import %c\n', 2),
+    'importfrom': ( '%|from %[2]{pattr} import %c\n', 3 ),
+    'importstar': ( '%|from %[2]{pattr} import *\n', ),
+}
+
+
+MAP_DIRECT = (TABLE_DIRECT, )
+MAP_R0 = (TABLE_R0, -1, 0)
+MAP_R = (TABLE_R, -1)
+
+MAP = {
+    'stmt':		MAP_R,
+    'call_function':	MAP_R,
+    'del_stmt':		MAP_R,
+    'designator':	MAP_R,
+    'exprlist':		MAP_R0,
+}
+
+PRECEDENCE = {
+    'build_list':           0,
+    'mapexpr':              0,
+    'unary_convert':        0,
+    'dictcomp':             0,
+    'setcomp':              0,
+    'list_compr':           0,
+    'genexpr':              0,
+
+    'load_attr':            2,
+    'binary_subscr':        2,
+    'binary_subscr2':       2,
+    'slice0':               2,
+    'slice1':               2,
+    'slice2':               2,
+    'slice3':               2,
+    'buildslice2':          2,
+    'buildslice3':          2,
+    'call_function':        2,
+
+    'BINARY_POWER':         4,
+
+    'unary_expr':           6,
+
+    'BINARY_MULTIPLY':      8,
+    'BINARY_DIVIDE':        8,
+    'BINARY_TRUE_DIVIDE':   8,
+    'BINARY_FLOOR_DIVIDE':  8,
+    'BINARY_MODULO':        8,
+
+    'BINARY_ADD':           10,
+    'BINARY_SUBTRACT':      10,
+
+    'BINARY_LSHIFT':        12,
+    'BINARY_RSHIFT':        12,
+
+    'BINARY_AND':           14,
+
+    'BINARY_XOR':           16,
+
+    'BINARY_OR':            18,
+
+    'cmp':                  20,
+
+    'unary_not':            22,
+
+    'and':                  24,
+    'ret_and':              24,
+
+    'or':                   26,
+    'ret_or':               26,
+
+    'conditional':          28,
+    'conditionalnot':       28,
+    'ret_cond':             28,
+    'ret_cond_not':         28,
+
+    '_mklambda':            30,
+    'yield':                101,
+    'yield_from':           101
+}
+
+ASSIGN_TUPLE_PARAM = lambda param_name: \
+             AST('expr', [ Token('LOAD_FAST', pattr=param_name) ])
+
+escape = re.compile(r'''
+            (?P<prefix> [^%]* )
+            % ( \[ (?P<child> -? \d+ ) \] )?
+                ((?P<type> [^{] ) |
+                 ( [{] (?P<expr> [^}]* ) [}] ))
+        ''', re.VERBOSE)

uncompyle6/semantics/fragments.py

@@ -51,11 +51,9 @@ The node position 0 will be associated with "import".
 
 # FIXME: DRY code with pysource
 
-from __future__ import print_function
-
 import re, sys
 
-from uncompyle6 import PYTHON3, IS_PYPY
+from uncompyle6 import PYTHON3, IS_PYPY, PYTHON_VERSION
 from xdis.code import iscode
 from uncompyle6.semantics import pysource
 from uncompyle6 import parser
@@ -69,22 +67,25 @@ from uncompyle6.show import (
     maybe_show_ast_param_default,
 )
 
-from uncompyle6.semantics.pysource import AST, INDENT_PER_LEVEL, NONE, PRECEDENCE, \
-     ParserError, TABLE_DIRECT, escape, find_globals, minint, MAP
+from uncompyle6.parsers.astnode import AST
+
+from uncompyle6.semantics.pysource import (
+    ParserError, find_globals, StringIO)
+
+from uncompyle6.semantics.consts import (
+    INDENT_PER_LEVEL, NONE, PRECEDENCE,
+    TABLE_DIRECT, escape, minint, MAP
+    )
 
 from uncompyle6.semantics.make_function import find_all_globals, find_none
 
-if PYTHON3:
-    from itertools import zip_longest
-    from io import StringIO
-else:
-    from itertools import izip_longest as zip_longest
-    from StringIO import StringIO
-
-
 from spark_parser import DEFAULT_DEBUG as PARSER_DEFAULT_DEBUG
 
-from collections import namedtuple
+if PYTHON_VERSION < 2.6:
+    from xdis.namedtuple25 import namedtuple
+else:
+    from collections import namedtuple
+
 NodeInfo = namedtuple("NodeInfo", "node start finish")
 ExtractInfo = namedtuple("ExtractInfo",
                          "lineNo lineStartOffset markerLine selectedLine selectedText")
@@ -97,14 +98,14 @@ TABLE_DIRECT_FRAGMENT = {
     'importstmt':	( '%|import %c%x\n', 2, (2, (0, 1)), ),
     'importfrom':	( '%|from %[2]{pattr}%x import %c\n', (2, (0, 1)), 3),
     'importmultiple':   ( '%|import%b %c%c\n', 0, 2, 3 ),
-    'list_for':	  	(' for %c%x in %c%c', 2, (2,(1,)), 0, 3 ),
-    'forstmt':	  	( '%|for%b %c%x in %c:\n%+%c%-\n\n', 0, 3, (3, (2,)), 1, 4 ),
+    'list_for':	  	(' for %c%x in %c%c', 2, (2, (1, )), 0, 3 ),
+    'forstmt':	  	( '%|for%b %c%x in %c:\n%+%c%-\n\n', 0, 3, (3, (2, )), 1, 4 ),
     'forelsestmt': 	(
         '%|for %c in %c%x:\n%+%c%-%|else:\n%+%c%-\n\n', 3, (3, (2,)), 1, 4, -2),
     'forelselaststmt':	(
         '%|for %c%x in %c:\n%+%c%-%|else:\n%+%c%-', 3, (3, (2,)), 1, 4, -2),
     'forelselaststmtl':	(
-        '%|for %c%x in %c:\n%+%c%-%|else:\n%+%c%-\n\n', 3, (3, (2,)), 1, 4, -2),
+        '%|for %c%x in %c:\n%+%c%-%|else:\n%+%c%-\n\n', 3, (3, (2, )), 1, 4, -2),
 
     'whilestmt':	( '%|while%b %c:\n%+%c%-\n\n', 0, 1, 2 ),
     'whileelsestmt':	( '%|while%b %c:\n%+%c%-%|else:\n%+%c%-\n\n', 0, 1, 2, -2 ),
@@ -163,7 +164,7 @@ class FragmentsWalker(pysource.SourceWalker, object):
                  None)
 
     def set_pos_info(self, node, start, finish, name=None):
-        if name == None: name = self.name
+        if name is None: name = self.name
         if hasattr(node, 'offset'):
             self.offsets[name, node.offset] = \
               NodeInfo(node = node, start = start, finish = finish)
@@ -622,7 +623,7 @@ class FragmentsWalker(pysource.SourceWalker, object):
             n = ast[iter_index]
             assert n == 'list_iter'
 
-        ## FIXME: I'm not totally sure this is right.
+        # FIXME: I'm not totally sure this is right.
 
         # find innermost node
         if_node = None
@@ -736,7 +737,10 @@ class FragmentsWalker(pysource.SourceWalker, object):
     def n_genexpr(self, node):
         start = len(self.f.getvalue())
         self.write('(')
-        code_index = -6 if self.version > 3.2 else -5
+        if self.version > 3.2:
+            code_index = -6
+        else:
+            code_index = -5
         self.comprehension_walk(node, iter_index=3, code_index=code_index)
         self.write(')')
         self.set_pos_info(node, start, len(self.f.getvalue()))
@@ -890,7 +894,10 @@ class FragmentsWalker(pysource.SourceWalker, object):
                         subclass = n.attr
                         break
                     pass
-                subclass_info = node if node == 'classdefdeco2' else node[0]
+                if node == 'classdefdeco2':
+                    subclass_info = node
+                else:
+                    subclass_info = node[0]
             elif buildclass[1][0] == 'load_closure':
                 # Python 3 with closures not functions
                 load_closure = buildclass[1]
@@ -914,7 +921,10 @@ class FragmentsWalker(pysource.SourceWalker, object):
                 subclass = buildclass[1][0].attr
                 subclass_info = node[0]
         else:
-            buildclass = node if (node == 'classdefdeco2') else node[0]
+            if node == 'classdefdeco2':
+                buildclass = node
+            else:
+                buildclass = node[0]
             build_list = buildclass[1][0]
             if hasattr(buildclass[-3][0], 'attr'):
                 subclass = buildclass[-3][0].attr
@@ -980,7 +990,9 @@ class FragmentsWalker(pysource.SourceWalker, object):
             tokens.append(Token('LAMBDA_MARKER'))
             try:
                 ast = parser.parse(self.p, tokens, customize)
-            except (parser.ParserError, AssertionError) as e:
+            except parser.ParserError(e):
+                raise ParserError(e, tokens)
+            except AssertionError(e):
                 raise ParserError(e, tokens)
             maybe_show_ast(self.showast, ast)
             return ast
@@ -1005,7 +1017,9 @@ class FragmentsWalker(pysource.SourceWalker, object):
         # Build AST from disassembly.
         try:
             ast = parser.parse(self.p, tokens, customize)
-        except (parser.ParserError, AssertionError) as e:
+        except parser.ParserError(e):
+            raise ParserError(e, tokens)
+        except AssertionError(e):
             raise ParserError(e, tokens)
 
         maybe_show_ast(self.showast, ast)
@@ -1642,16 +1656,16 @@ class FragmentsWalker(pysource.SourceWalker, object):
                                  code._customize,
                                  isLambda = isLambda,
                                  noneInNames = ('None' in code.co_names))
-        except ParserError as p:
+        except ParserError(p):
             self.write(str(p))
             self.ERROR = p
             return
 
         # build parameters
 
+        tup = [paramnames, defparams]
         params = [build_param(ast, name, default) for
-                  name, default in zip_longest(paramnames, defparams, fillvalue=None)]
-
+              name, default in map(lambda *tup:tup, *tup)]
         params.reverse() # back to correct order
 
         if 4 & code.co_flags:	# flag 2 -> variable number of args

uncompyle6/semantics/make_function.py

@@ -6,14 +6,9 @@ All the crazy things we have to do to handle Python functions
 from xdis.code import iscode
 from uncompyle6.scanner import Code
 from uncompyle6.parsers.astnode import AST
-from uncompyle6 import PYTHON3
 from uncompyle6.semantics.parser_error import ParserError
 from uncompyle6.semantics.helper import print_docstring
 
-if PYTHON3:
-    from itertools import zip_longest
-else:
-    from itertools import izip_longest as zip_longest
 
 from uncompyle6.show import maybe_show_ast_param_default
 
@@ -71,7 +66,11 @@ def make_function3_annotate(self, node, isLambda, nested=1,
     # MAKE_FUNCTION_... or MAKE_CLOSURE_...
     assert node[-1].type.startswith('MAKE_')
 
-    annotate_tuple = node[annotate_last]
+    annotate_tuple = None
+    for annotate_last in range(len(node)-1, -1, -1):
+        if node[annotate_last] == 'annotate_tuple':
+            annotate_tuple = node[annotate_last]
+            break
     annotate_args = {}
 
     if (annotate_tuple == 'annotate_tuple'
@@ -122,7 +121,7 @@ def make_function3_annotate(self, node, isLambda, nested=1,
                              code._customize,
                              isLambda = isLambda,
                              noneInNames = ('None' in code.co_names))
-    except ParserError as p:
+    except ParserError, p:
         self.write(str(p))
         self.ERROR = p
         return
@@ -148,22 +147,34 @@ def make_function3_annotate(self, node, isLambda, nested=1,
     suffix = ''
     for param in paramnames[:i]:
         self.write(suffix, param)
-        if param in annotate_args:
-            value, string = annotate_args[param]
-            if string:
-                self.write(': "%s"' % value)
-            else:
-                self.write(': %s' % value)
         suffix = ', '
+        if param in annotate_tuple[0].attr:
+            p = [x for x in annotate_tuple[0].attr].index(param)
+            self.write(': ')
+            self.preorder(node[p])
+            if (line_number != self.line_number):
+                suffix = ",\n" + indent
+                line_number = self.line_number
+            # value, string = annotate_args[param]
+            # if string:
+            #     self.write(': "%s"' % value)
+            # else:
+            #     self.write(': %s' % value)
 
-    suffix = ', ' if i > 0 else ''
+    if i > 0:
+        suffix = ', '
+    else:
+        suffix = ''
     for n in node:
         if n == 'pos_arg':
             self.write(suffix)
             param = paramnames[i]
             self.write(param)
             if param in annotate_args:
-                self.write(':"%s' % annotate_args[param])
+                aa = annotate_args[param]
+                if isinstance(aa, tuple):
+                    aa = aa[0]
+                self.write(': "%s"' % aa)
             self.write('=')
             i += 1
             self.preorder(n)
@@ -189,6 +200,9 @@ def make_function3_annotate(self, node, isLambda, nested=1,
         i = 0
         for n in node[0]:
             if n == 'kwarg':
+                if (line_number != self.line_number):
+                    self.write("\n" + indent)
+                    line_number = self.line_number
                 self.write('%s=' % n[0].pattr)
                 self.preorder(n[1])
                 if i < last:
@@ -207,19 +221,24 @@ def make_function3_annotate(self, node, isLambda, nested=1,
         self.write(": ")
     else:
         self.write(')')
-        if 'return' in annotate_args:
-            value, string = annotate_args['return']
-            if string:
-                self.write(' -> "%s"' % value)
-            else:
-                self.write(' -> %s' % value)
+        if 'return' in annotate_tuple[0].attr:
+            if (line_number != self.line_number):
+                self.write("\n" + indent)
+                line_number = self.line_number
+            self.write(' -> ')
+            # value, string = annotate_args['return']
+            # if string:
+            #     self.write(' -> "%s"' % value)
+            # else:
+            #     self.write(' -> %s' % value)
+            self.preorder(node[annotate_last-1])
 
         self.println(":")
 
     if (len(code.co_consts) > 0 and
         code.co_consts[0] is not None and not isLambda): # ugly
         # docstring exists, dump it
-        print_docstring(self, indent, code.co_consts[0])
+        print_docstring(self, self.indent, code.co_consts[0])
 
     code._tokens = None # save memory
     assert ast == 'stmts'
@@ -301,17 +320,21 @@ def make_function2(self, node, isLambda, nested=1, codeNode=None):
                              code._customize,
                              isLambda = isLambda,
                              noneInNames = ('None' in code.co_names))
-    except ParserError as p:
+    except ParserError, p:
         self.write(str(p))
         self.ERROR = p
         return
 
-    kw_pairs = args_node.attr[1] if self.version >= 3.0 else 0
+    if self.version >= 3.0:
+        kw_pairs = args_node.attr[1]
+    else:
+        kw_pairs = 0
     indent = self.indent
 
     # build parameters
+    tup = [paramnames, defparams]
     params = [build_param(ast, name, default) for
-              name, default in zip_longest(paramnames, defparams, fillvalue=None)]
+              name, default in map(lambda *tup:tup, *tup)]
     params.reverse() # back to correct order
 
     if 4 & code.co_flags:	# flag 2 -> variable number of args
@@ -437,18 +460,22 @@ def make_function3(self, node, isLambda, nested=1, codeNode=None):
                              code._customize,
                              isLambda = isLambda,
                              noneInNames = ('None' in code.co_names))
-    except ParserError as p:
+    except ParserError, p:
         self.write(str(p))
         self.ERROR = p
         return
 
-    kw_pairs = args_node.attr[1] if self.version >= 3.0 else 0
+    if self.version >= 3.0:
+        kw_pairs = args_node.attr[1]
+    else:
+        kw_pairs = 0
     indent = self.indent
 
     # build parameters
     if self.version != 3.2:
+        tup = [paramnames, defparams]
         params = [build_param(ast, name, default) for
-                  name, default in zip_longest(paramnames, defparams, fillvalue=None)]
+              name, default in map(lambda *tup:tup, *tup)]
         params.reverse() # back to correct order
 
         if 4 & code.co_flags:	# flag 2 -> variable number of args
@@ -483,7 +510,10 @@ def make_function3(self, node, isLambda, nested=1, codeNode=None):
 
         i = len(paramnames) - len(defparams)
         self.write(", ".join(paramnames[:i]))
-        suffix = ', ' if i > 0 else ''
+        if i > 0:
+            suffix = ', '
+        else:
+            suffix = ''
         for n in node:
             if n == 'pos_arg':
                 self.write(suffix)