Get ready for release 3.1.2

which are created via a call to create a subclass.

Should be more general though.

.gitignore

@@ -21,3 +21,4 @@
 ChangeLog
 __pycache__
 build
+nohup.out

.travis.yml

@@ -3,7 +3,13 @@ language: python
 sudo: false
 
 python:
-  - '2.7' # this is a cheat here because travis doesn't do 2.4-2.6
+  - '3.5'
+  - '2.7'
+  - '2.6'
+  - '3.3'
+  - '3.4'
+  - '3.2'
+  - '3.6'
 
 install:
 - pip install -e .

Makefile

@@ -27,21 +27,18 @@ check:
 check-short: pytest
 	$(MAKE) -C test check-short
 
+# Note for 2.6 use <=3.0.1 see requirements-dev.txt
 #: Tests for Python 2.7, 3.3 and 3.4
-check-2.7 check-3.3 check-3.4: pytest
+check-2.6 check-2.7 check-3.3 check-3.4 check-3.5: pytest
 	$(MAKE) -C test $@
 
 #: Tests for Python 3.2 and 3.5 - pytest doesn't work here
 # Or rather 3.5 doesn't work not on Travis
-check-3.0 check-3.1 check-3.2 check-3.5 check-3.6:
+check-3.0 check-3.1 check-3.2 check-3.6:
 	$(MAKE) -C test $@
 
 check-3.7: pytest
 
-#:Tests for Python 2.6 (doesn't have pytest)
-check-2.4 check-2.5 check-2.6:
-	$(MAKE) -C test $@
-
 #:PyPy 2.6.1 PyPy 5.0.1, or PyPy 5.8.0-beta0
 # Skip for now
 2.6 5.0 5.3 5.6 5.8:

NEWS

@@ -1,3 +1,11 @@
+uncompyle6 3.1.2 2018-04-08 Eastern Orthodox Easter
+
+- Python 3.x subclass and call parsing fixes
+- Allow/note running on Python 3.1
+- improve 3.5+ BUILD_MAP_UNPACK
+- DRY instruction building code between 2.x and 3.x
+- expand testing
+
 uncompyle6 3.1.1 2018-04-01 Easter April Fool's
 
 Jesus on Friday's New York Times puzzle: "I'm stuck on 2A"

__pkginfo__.py

@@ -35,6 +35,8 @@ classifiers =  ['Development Status :: 5 - Production/Stable',
                 'Programming Language :: Python :: 2.5',
                 'Programming Language :: Python :: 2.6',
                 'Programming Language :: Python :: 2.7',
+                'Programming Language :: Python :: 3.1',
+                'Programming Language :: Python :: 3.2',
                 'Programming Language :: Python :: 3.3',
                 'Programming Language :: Python :: 3.4',
                 'Programming Language :: Python :: 3.5',
@@ -54,7 +56,8 @@ entry_points       = {
     ]}
 ftp_url            = None
 install_requires   = ['spark-parser >= 1.8.5, < 1.9.0',
-                      'xdis >= 3.7.0, < 3.8.0']
+                      'xdis >= 3.8.0, < 3.9.0', 'six']
+
 license            = 'GPL3'
 mailing_list       = 'python-debugger@googlegroups.com'
 modname            = 'uncompyle6'

admin-tools/how-to-make-a-release.txt

@@ -1,46 +0,0 @@
-git pull
-
-Change version in uncompyle6/version.py
-source uncompyle6/version.py
-echo $VERSION
-git commit -m"Get ready for release $VERSION" .
-
-Update ChangeLog:
-  make ChangeLog
-
-Update NEWS from ChangeLog
-make check
-
-git commit --amend .
-
-git push
-
-Make sure pyenv is running
-# Pyenv
-
-source admin-tools/check-newer-versions.sh
-
-
-# Switch to python-2.4 and build that first...
-source admin-tools/setup-python-2.4
-
-rm ChangeLog
-git merge master
-
-Update NEWS from master branch
-
-git commit -m"Get ready for release $VERSION" .
-
-source admin-tools/check-older-versions.sh
-source admin-tools/check-newer-versions.sh
-
-make-dist-older.sh
-
-git tag release-python-2.4-$VERSION
-
-./make-dist-newer.sh
-
-git tag release-$VERSION
-
-
-twine upload dist/uncompyle6-${VERSION}*

admin-tools/pyenv-newer-versions

@@ -5,4 +5,4 @@ if [[ $0 == ${BASH_SOURCE[0]} ]] ; then
     echo "This script should be *sourced* rather than run directly through bash"
     exit 1
 fi
-export PYVERSIONS='3.5.5 3.6.4 2.6.9 3.3.7 2.7.14 3.4.8'
+export PYVERSIONS='3.5.5 3.6.4 2.6.9 3.3.7 2.7.14 3.2.6 3.1.5 3.4.8'

admin-tools/update-sources.sh

@@ -1,3 +0,0 @@
-#!/bin/bash
-cd $(dirname ${BASH_SOURCE[0]})/..
-git pull

circle.yml

@@ -11,4 +11,4 @@ dependencies:
 test:
     override:
         - python ./setup.py develop && make check-2.7
-        - cd ./test/stdlib && pyenv local 2.7.10 && bash ./runtests.sh 'test_[f-i]*.py'
+        - cd ./test/stdlib && pyenv local 2.7.10 && bash ./runtests.sh 'test_[p-z]*.py'

pytest/test_deparse.py

@@ -30,7 +30,7 @@ def list_comp():
     [y for y in range(3)]
 
 def get_parsed_for_fn(fn):
-    code = fn.func_code
+    code = fn.__code__ if PYTHON3 else fn.func_code
     return deparse(code, version=PYTHON_VERSION)
 
 def check_expect(expect, parsed, fn_name):

pytest/test_docstring.py

@@ -10,7 +10,7 @@ else:
     maxint = sys.maxint
 from uncompyle6.semantics.helper import print_docstring
 
-class PrintFake:
+class PrintFake():
     def __init__(self):
         self.pending_newlines = 0
         self.f = StringIO()

pytest/test_fjt.py

@@ -1,8 +1,6 @@
 #!/usr/bin/env python
 from uncompyle6 import PYTHON_VERSION, IS_PYPY
 from uncompyle6.scanner import get_scanner
-from xdis.bytecode import Bytecode
-from array import array
 def bug(state, slotstate):
     if state:
         if slotstate is not None:
@@ -22,17 +20,10 @@ def bug_loop(disassemble, tb=None):
     disassemble(tb)
 
 def test_if_in_for():
-    code = bug.func_code
+    code = bug.__code__
     scan = get_scanner(PYTHON_VERSION)
     if 2.7 <= PYTHON_VERSION <= 3.0 and not IS_PYPY:
-        n = scan.setup_code(code)
-        bytecode = Bytecode(code, scan.opc)
-        scan.build_lines_data(code, n)
-        scan.insts = list(bytecode)
-        scan.offset2inst_index = {}
-        for i, inst in enumerate(scan.insts):
-            scan.offset2inst_index[inst.offset] = i
-        scan.build_prev_op(n)
+        scan.build_instructions(code)
         fjt = scan.find_jump_targets(False)
 
         ## FIXME: the data below is wrong.
@@ -47,14 +38,7 @@ def test_if_in_for():
         #    {'start': 62, 'end': 63, 'type': 'for-else'}]
 
         code = bug_loop.__code__
-        n = scan.setup_code(code)
-        bytecode = Bytecode(code, scan.opc)
-        scan.build_lines_data(code, n)
-        scan.insts = list(bytecode)
-        scan.build_prev_op(n)
-        scan.offset2inst_index = {}
-        for i, inst in enumerate(scan.insts):
-            scan.offset2inst_index[inst.offset] = i
+        scan.build_instructions(code)
         fjt = scan.find_jump_targets(False)
         assert{64: [42], 67: [42, 42], 42: [16, 41], 19: [6]} == fjt
         assert scan.structs == [
@@ -68,14 +52,7 @@ def test_if_in_for():
             {'start': 48, 'end': 67, 'type': 'while-loop'}]
 
     elif 3.2 < PYTHON_VERSION <= 3.4:
-        bytecode = Bytecode(code, scan.opc)
-        scan.code = array('B', code.co_code)
-        scan.lines = scan.build_lines_data(code)
-        scan.build_prev_op()
-        scan.insts = list(bytecode)
-        scan.offset2inst_index = {}
-        for i, inst in enumerate(scan.insts):
-            scan.offset2inst_index[inst.offset] = i
+        scan.build_instructions(code)
         fjt  = scan.find_jump_targets(False)
         assert {69: [66], 63: [18]} == fjt
         assert scan.structs == \
@@ -85,5 +62,6 @@ def test_if_in_for():
            {'end': 59, 'type': 'for-loop', 'start': 31},
            {'end': 63, 'type': 'for-else', 'start': 62}]
     else:
-        assert True, "FIXME: should note fixed"
+        print("FIXME: should fix for %s" % PYTHON_VERSION)
+        assert True
     return

pytest/test_fstring.py

@@ -0,0 +1,150 @@
+# std
+import os
+# test
+import pytest
+import hypothesis
+from hypothesis import strategies as st
+# uncompyle6
+from uncompyle6 import PYTHON_VERSION, deparse_code
+
+
+@st.composite
+def expressions(draw):
+    # todo : would be nice to generate expressions using hypothesis however
+    # this is pretty involved so for now just use a corpus of expressions
+    # from which to select.
+    return draw(st.sampled_from((
+        'abc',
+        'len(items)',
+        'x + 1',
+        'lineno',
+        'container',
+        'self.attribute',
+        'self.method()',
+        # These expressions are failing, I think these are control
+        # flow problems rather than problems with FORMAT_VALUE,
+        # however I need to confirm this...
+        #'sorted(items, key=lambda x: x.name)',
+        #'func(*args, **kwargs)',
+        #'text or default',
+        #'43 if life_the_universe and everything else None'
+    )))
+
+
+@st.composite
+def format_specifiers(draw):
+    """
+    Generate a valid format specifier using the rules:
+
+    format_spec ::=  [[fill]align][sign][#][0][width][,][.precision][type]
+    fill        ::=  <any character>
+    align       ::=  "<" | ">" | "=" | "^"
+    sign        ::=  "+" | "-" | " "
+    width       ::=  integer
+    precision   ::=  integer
+    type        ::=  "b" | "c" | "d" | "e" | "E" | "f" | "F" | "g" | "G" | "n" | "o" | "s" | "x" | "X" | "%"
+
+    See https://docs.python.org/2/library/string.html
+
+    :param draw: Let hypothesis draw from other strategies.
+
+    :return: An example format_specifier.
+    """
+    alphabet_strategy = st.characters(min_codepoint=ord('a'), max_codepoint=ord('z'))
+    fill = draw(st.one_of(alphabet_strategy, st.none()))
+    align = draw(st.sampled_from(list('<>=^')))
+    fill_align = (fill + align or '') if fill else ''
+
+    type_ = draw(st.sampled_from('bcdeEfFgGnosxX%'))
+    can_have_sign = type_ in 'deEfFgGnoxX%'
+    can_have_comma = type_ in 'deEfFgG%'
+    can_have_precision = type_ in 'fFgG'
+    can_have_pound = type_ in 'boxX%'
+    can_have_zero = type_ in 'oxX'
+
+    sign = draw(st.sampled_from(list('+- ') + [''])) if can_have_sign else ''
+    pound = draw(st.sampled_from(('#', '',))) if can_have_pound else ''
+    zero = draw(st.sampled_from(('0', '',))) if can_have_zero else ''
+
+    int_strategy = st.integers(min_value=1, max_value=1000)
+
+    width = draw(st.one_of(int_strategy, st.none()))
+    width = str(width) if width is not None else ''
+
+    comma = draw(st.sampled_from((',', '',))) if can_have_comma else ''
+    if can_have_precision:
+        precision = draw(st.one_of(int_strategy, st.none()))
+        precision = '.' + str(precision) if precision else ''
+    else:
+        precision = ''
+
+    return ''.join((fill_align, sign, pound, zero, width, comma, precision, type_,))
+
+
+@st.composite
+def fstrings(draw):
+    """
+    Generate a valid f-string.
+    See https://www.python.org/dev/peps/pep-0498/#specification
+
+    :param draw: Let hypothsis draw from other strategies.
+
+    :return: A valid f-string.
+    """
+    character_strategy = st.characters(
+        blacklist_characters='\r\n\'\\s{}',
+        min_codepoint=1,
+        max_codepoint=1000,
+    )
+    is_raw = draw(st.booleans())
+    integer_strategy = st.integers(min_value=0, max_value=3)
+    expression_count = draw(integer_strategy)
+    content = []
+    for _ in range(expression_count):
+        expression = draw(expressions())
+        conversion = draw(st.sampled_from(('', '!s', '!r', '!a',)))
+        has_specifier = draw(st.booleans())
+        specifier = ':' + draw(format_specifiers()) if has_specifier else ''
+        content.append('{{{}{}}}'.format(expression, conversion, specifier))
+        content.append(draw(st.text(character_strategy)))
+    content = ''.join(content)
+    return "f{}'{}'".format('r' if is_raw else '', content)
+
+
+@pytest.mark.skipif(PYTHON_VERSION < 3.6, reason='need at least python 3.6')
+@hypothesis.given(format_specifiers())
+def test_format_specifiers(format_specifier):
+    """Verify that format_specifiers generates valid specifiers"""
+    try:
+        exec('"{:' + format_specifier + '}".format(0)')
+    except ValueError as e:
+        if 'Unknown format code' not in str(e):
+            raise
+
+
+def run_test(text):
+    hypothesis.assume(len(text))
+    hypothesis.assume("f'{" in text)
+    expr = text + '\n'
+    code = compile(expr, '<string>', 'single')
+    deparsed = deparse_code(PYTHON_VERSION, code, compile_mode='single')
+    recompiled = compile(deparsed.text, '<string>', 'single')
+    if recompiled != code:
+        assert 'dis(' + deparsed.text.strip('\n') + ')' == 'dis(' + expr.strip('\n') + ')'
+
+
+@pytest.mark.skipif(PYTHON_VERSION < 3.6, reason='need at least python 3.6')
+@hypothesis.given(fstrings())
+def test_uncompyle_fstring(fstring):
+    """Verify uncompyling fstring bytecode"""
+    run_test(fstring)
+
+
+@pytest.mark.skipif(PYTHON_VERSION < 3.6, reason='need at least python 3.6')
+@pytest.mark.parametrize('fstring', [
+    "f'{abc}{abc!s}'",
+    "f'{abc}0'",
+])
+def test_uncompyle_direct(fstring):
+    """useful for debugging"""
+    run_test(fstring)

pytest/test_function_call.py

@@ -0,0 +1,184 @@
+# std
+import string
+# 3rd party
+from hypothesis import given, assume, example, settings, strategies as st
+import pytest
+# uncompyle
+from validate import validate_uncompyle
+from test_fstring import expressions
+from uncompyle6 import PYTHON_VERSION
+
+alpha = st.sampled_from(string.ascii_lowercase)
+numbers = st.sampled_from(string.digits)
+alphanum = st.sampled_from(string.ascii_lowercase + string.digits)
+
+
+@st.composite
+def function_calls(draw,
+                   min_keyword_args=0, max_keyword_args=5,
+                   min_positional_args=0, max_positional_args=5,
+                   min_star_args=0, max_star_args=1,
+                   min_double_star_args=0, max_double_star_args=1):
+    """
+    Strategy factory for generating function calls.
+
+    :param draw: Callable which draws examples from other strategies.
+
+    :return: The function call text.
+    """
+    st_positional_args = st.lists(
+        alpha,
+        min_size=min_positional_args,
+        max_size=max_positional_args
+    )
+    st_keyword_args = st.lists(
+        alpha,
+        min_size=min_keyword_args,
+        max_size=max_keyword_args
+    )
+    st_star_args = st.lists(
+        alpha,
+        min_size=min_star_args,
+        max_size=max_star_args
+    )
+    st_double_star_args = st.lists(
+        alpha,
+        min_size=min_double_star_args,
+        max_size=max_double_star_args
+    )
+
+    positional_args = draw(st_positional_args)
+    keyword_args = draw(st_keyword_args)
+    st_values = st.lists(
+        expressions(),
+        min_size=len(keyword_args),
+        max_size=len(keyword_args)
+    )
+    keyword_args = [
+        x + '=' + e
+        for x, e in
+        zip(keyword_args, draw(st_values))
+    ]
+    star_args = ['*' + x for x in draw(st_star_args)]
+    double_star_args = ['**' + x for x in draw(st_double_star_args)]
+
+    arguments = positional_args + keyword_args + star_args + double_star_args
+    draw(st.randoms()).shuffle(arguments)
+    arguments = ','.join(arguments)
+
+    function_call = 'fn({arguments})'.format(arguments=arguments)
+    try:
+        # TODO: Figure out the exact rules for ordering of positional, keyword,
+        # star args, double star args and in which versions the various
+        # types of arguments are supported so we don't need to check that the
+        # expression compiles like this.
+        compile(function_call, '<string>', 'single')
+    except:
+        assume(False)
+    return function_call
+
+
+def test_function_no_args():
+    validate_uncompyle("fn()")
+
+@pytest.mark.skipif(PYTHON_VERSION < 2.7,
+                    reason="need at least Python 2.7")
+def isolated_function_calls(which):
+    """
+    Returns a strategy for generating function calls, but isolated to
+    particular types of arguments, for example only positional arguments.
+
+    This can help reason about debugging errors in specific types of function
+    calls.
+
+    :param which: One of 'keyword', 'positional', 'star', 'double_star'
+
+    :return: Strategy for generating an function call isolated to specific
+             argument types.
+    """
+    kwargs = dict(
+        max_keyword_args=0,
+        max_positional_args=0,
+        max_star_args=0,
+        max_double_star_args=0,
+    )
+    kwargs['_'.join(('min', which, 'args'))] = 1
+    kwargs['_'.join(('max', which, 'args'))] = 5 if 'star' not in which else 1
+    return function_calls(**kwargs)
+
+
+with settings(max_examples=25):
+
+    @pytest.mark.skipif(PYTHON_VERSION < 2.7,
+                        reason="need at least Python 2.7")
+    @given(isolated_function_calls('positional'))
+    @example("fn(0)")
+    def test_function_positional_only(expr):
+        validate_uncompyle(expr)
+
+    @pytest.mark.skipif(PYTHON_VERSION < 2.7,
+                        reason="need at least Python 2.7")
+    @given(isolated_function_calls('keyword'))
+    @example("fn(a=0)")
+    def test_function_call_keyword_only(expr):
+        validate_uncompyle(expr)
+
+    @pytest.mark.skipif(PYTHON_VERSION < 2.7,
+                        reason="need at least Python 2.7")
+    @given(isolated_function_calls('star'))
+    @example("fn(*items)")
+    def test_function_call_star_only(expr):
+        validate_uncompyle(expr)
+
+    @pytest.mark.skipif(PYTHON_VERSION < 2.7,
+                        reason="need at least Python 2.7")
+    @given(isolated_function_calls('double_star'))
+    @example("fn(**{})")
+    def test_function_call_double_star_only(expr):
+        validate_uncompyle(expr)
+
+
+@pytest.mark.xfail()
+def test_BUILD_CONST_KEY_MAP_BUILD_MAP_UNPACK_WITH_CALL_BUILD_TUPLE_CALL_FUNCTION_EX():
+    validate_uncompyle("fn(w=0,m=0,**v)")
+
+
+@pytest.mark.xfail()
+def test_BUILD_MAP_BUILD_MAP_UNPACK_WITH_CALL_BUILD_TUPLE_CALL_FUNCTION_EX():
+    validate_uncompyle("fn(a=0,**g)")
+
+
+@pytest.mark.xfail()
+def test_CALL_FUNCTION_EX():
+    validate_uncompyle("fn(*g,**j)")
+
+
+@pytest.mark.xfail()
+def test_BUILD_MAP_CALL_FUNCTION_EX():
+    validate_uncompyle("fn(*z,u=0)")
+
+
+@pytest.mark.xfail()
+def test_BUILD_TUPLE_CALL_FUNCTION_EX():
+    validate_uncompyle("fn(**a)")
+
+
+@pytest.mark.xfail()
+def test_BUILD_MAP_BUILD_TUPLE_BUILD_TUPLE_UNPACK_WITH_CALL_CALL_FUNCTION_EX():
+    validate_uncompyle("fn(b,b,b=0,*a)")
+
+
+@pytest.mark.xfail()
+def test_BUILD_TUPLE_BUILD_TUPLE_UNPACK_WITH_CALL_CALL_FUNCTION_EX():
+    validate_uncompyle("fn(*c,v)")
+
+
+@pytest.mark.xfail()
+def test_BUILD_CONST_KEY_MAP_CALL_FUNCTION_EX():
+    validate_uncompyle("fn(i=0,y=0,*p)")
+
+
+@pytest.mark.skip(reason='skipping property based test until all individual tests are passing')
+@given(function_calls())
+def test_function_call(function_call):
+    validate_uncompyle(function_call)

pytest/test_grammar.py

@@ -20,13 +20,17 @@ def test_grammar():
     # We have custom rules that create the below
     expect_lhs = set(['pos_arg', 'get_iter', 'attribute'])
 
-    unused_rhs = set(['list', 'mkfunc', 'dict',
+    unused_rhs = set(['list', 'mkfunc',
                       'mklambda',
                       'unpack',])
     expect_right_recursive = set([('designList',
                                    ('store', 'DUP_TOP', 'designList'))])
-    expect_lhs.add('kvlist')
-    expect_lhs.add('kv3')
+
+    if PYTHON_VERSION > 2.6:
+        expect_lhs.add('kvlist')
+        expect_lhs.add('kv3')
+        unused_rhs.add('dict')
+
     if PYTHON3:
         expect_lhs.add('load_genexpr')
 
@@ -46,7 +50,7 @@ def test_grammar():
                 unused_rhs.add('call')
                 if PYTHON_VERSION == 3.5:
                     expect_right_recursive.add((('l_stmts',
-                                                 ('lastl_stmt', 'COME_FROM', 'l_stmts'))))
+                                                 ('lastl_stmt', 'come_froms', 'l_stmts'))))
                     pass
                 pass
             else:
@@ -85,6 +89,8 @@ def test_grammar():
             """.split())
     if 2.6 <= PYTHON_VERSION <= 2.7:
         opcode_set = set(s.opc.opname).union(ignore_set)
+        if PYTHON_VERSION == 2.6:
+            opcode_set.add("THEN")
         check_tokens(tokens, opcode_set)
     elif PYTHON_VERSION == 3.4:
         ignore_set.add('LOAD_CLASSNAME')

pytest/test_pysource.py

@@ -8,8 +8,12 @@ from uncompyle6.semantics.consts import (
 
 if PYTHON3:
     from io import StringIO
+    def iteritems(d):
+        return d.items()
 else:
     from StringIO import StringIO
+    def iteritems(d):
+        return d.iteritems()
 
 from uncompyle6.semantics.pysource import SourceWalker as SourceWalker
 
@@ -26,7 +30,7 @@ def test_template_engine():
     # FIXME: and so on...
 
 from uncompyle6.semantics.consts import (
-    TABLE_R, TABLE_DIRECT,
+    TABLE_DIRECT, TABLE_R,
     )
 
 from uncompyle6.semantics.fragments import (
@@ -40,7 +44,7 @@ def test_tables():
             (TABLE_DIRECT, 'TABLE_DIRECT', False),
             (TABLE_R, 'TABLE_R', False),
             (TABLE_DIRECT_FRAGMENT, 'TABLE_DIRECT_FRAGMENT', True)):
-        for k, entry in t.iteritems():
+        for k, entry in iteritems(t):
             if k in skip_for_now:
                 continue
             fmt = entry[0]

pytest/test_single_compile.py

@@ -1,19 +1,21 @@
+import pytest
 from uncompyle6 import PYTHON_VERSION, deparse_code
 
-if PYTHON_VERSION >= 2.5:
-    def test_single_mode():
-        single_expressions = (
-            'i = 1',
-            'i and (j or k)',
-            'i += 1',
-            'i = j % 4',
-            'i = {}',
-            'i = []',
-            'for i in range(10):\n    i\n',
-            'for i in range(10):\n    for j in range(10):\n        i + j\n',
-            'try:\n    i\nexcept Exception:\n    j\nelse:\n    k\n'
-        )
+@pytest.mark.skip(PYTHON_VERSION < 2.7,
+                  reason="need at least Python 2.7")
+def test_single_mode():
+    single_expressions = (
+        'i = 1',
+        'i and (j or k)',
+        'i += 1',
+        'i = j % 4',
+        'i = {}',
+        'i = []',
+        'for i in range(10):\n    i\n',
+        'for i in range(10):\n    for j in range(10):\n        i + j\n',
+        'try:\n    i\nexcept Exception:\n    j\nelse:\n    k\n'
+    )
 
-        for expr in single_expressions:
-            code = compile(expr + '\n', '<string>', 'single')
-            assert deparse_code(PYTHON_VERSION, code, compile_mode='single').text == expr + '\n'
+    for expr in single_expressions:
+        code = compile(expr + '\n', '<string>', 'single')
+        assert deparse_code(PYTHON_VERSION, code, compile_mode='single').text == expr + '\n'

pytest/testdata/if-2.7.right

@@ -9,4 +9,4 @@
           12  JUMP_FORWARD          0  'to 15'
         15_0  COME_FROM            12  '12'
           15  LOAD_CONST            0  None
-          18  RETURN_VALUE
+          18  RETURN_VALUE     

pytest/testdata/ifelse-2.7.right

@@ -12,4 +12,4 @@
           18  STORE_NAME            2  'd'
         21_0  COME_FROM            12  '12'
           21  LOAD_CONST            2  None
-          24  RETURN_VALUE
+          24  RETURN_VALUE     

pytest/validate.py

@@ -1,25 +1,24 @@
+# future
+from __future__ import print_function
 # std
 import os
 import difflib
 import subprocess
 import tempfile
-
-from StringIO import StringIO
+import functools
+# compatability
+import six
 # uncompyle6 / xdis
 from uncompyle6 import PYTHON_VERSION, IS_PYPY, deparse_code
 # TODO : I think we can get xdis to support the dis api (python 3 version) by doing something like this there
 from xdis.bytecode import Bytecode
 from xdis.main import get_opcode
 opc = get_opcode(PYTHON_VERSION, IS_PYPY)
+Bytecode = functools.partial(Bytecode, opc=opc)
 
-try:
-    import functools
-    Bytecode = functools.partial(Bytecode, opc=opc)
-    def _dis_to_text(co):
-        return Bytecode(co).dis()
-except:
-    pass
 
+def _dis_to_text(co):
+    return Bytecode(co).dis()
 
 
 def print_diff(original, uncompyled):
@@ -43,11 +42,8 @@ def print_diff(original, uncompyled):
         print('\nTo display diff highlighting run:\n    pip install BeautifulSoup4')
         diff = difflib.HtmlDiff().make_table(*args)
 
-    f = tempfile.NamedTemporaryFile(delete=False)
-    try:
+    with tempfile.NamedTemporaryFile(delete=False) as f:
         f.write(str(diff).encode('utf-8'))
-    finally:
-        f.close()
 
     try:
         print()
@@ -64,7 +60,8 @@ def print_diff(original, uncompyled):
         print('\nFor side by side diff install elinks')
         diff = difflib.Differ().compare(original_lines, uncompyled_lines)
         print('\n'.join(diff))
-    os.unlink(f.name)
+    finally:
+        os.unlink(f.name)
 
 
 def are_instructions_equal(i1, i2):
@@ -126,9 +123,8 @@ def validate_uncompyle(text, mode='exec'):
     original_text = text
 
     deparsed = deparse_code(PYTHON_VERSION, original_code,
-
                             compile_mode=mode,
-                            out=StringIO(),
+                            out=six.StringIO(),
                             is_pypy=IS_PYPY)
     uncompyled_text = deparsed.text
     uncompyled_code = compile(uncompyled_text, '<string>', 'exec')

requirements-dev.txt

@@ -1,3 +1,3 @@
-pytest>=3.0.0
+pytest>=3.0.0,<=3.0.1
 flake8
-hypothesis<=3.8.3
+hypothesis<=3.0.0

setup.py

@@ -1,16 +1,16 @@
 #!/usr/bin/env python
-"""Setup script for the 'uncompyle6' distribution."""
-
 import sys
 
+"""Setup script for the 'uncompyle6' distribution."""
+
 SYS_VERSION = sys.version_info[0:2]
-if not ((2, 4) <= SYS_VERSION  <= (2, 7)):
-    mess = "Python Release 2.4 .. 2.7 are supported in this code branch."
-    if ((3, 2) <= SYS_VERSION  <= (3, 7)):
-        mess += ("\nFor your Python, version %s, use the master code/branch." %
+if not ((2, 6) <= SYS_VERSION  <= (3, 7)) or ((3, 0) <= SYS_VERSION <= (3, 0)):
+    mess = "Python Release 2.6 .. 3.7 excluding 3.0 are supported in this code branch."
+    if ((2, 4) <= SYS_VERSION <= (2, 7)):
+        mess += ("\nFor your Python, version %s, use the python-2.4 code/branch." %
                  sys.version[0:3])
-    else:
-        mess += ("\nThis package is not supported before Python 2.4. Your Python version is %s."
+    elif SYS_VERSION < (2, 4) or (3, 0) <= SYS_VERSION:
+        mess += ("\nThis package is not supported for Python version %s."
                  % sys.version[0:3])
     print(mess)
     raise Exception(mess)

test-unit/test_grammar.py

@@ -1,55 +0,0 @@
-import re
-import unittest
-from uncompyle6 import PYTHON_VERSION, IS_PYPY # , PYTHON_VERSION
-from uncompyle6.parser import get_python_parser, python_parser
-
-class TestGrammar(unittest.TestCase):
-    def test_grammar(self):
-
-        def check_tokens(tokens, opcode_set):
-            remain_tokens = set(tokens) - opcode_set
-            remain_tokens = set([re.sub('_\d+$','', t) for t in remain_tokens])
-            remain_tokens = set([re.sub('_CONT$','', t) for t in remain_tokens])
-            remain_tokens = set(remain_tokens) - opcode_set
-            self.assertEqual(remain_tokens, set([]),
-                    "Remaining tokens %s\n====\n%s" % (remain_tokens, p.dump_grammar()))
-
-        p = get_python_parser(PYTHON_VERSION, is_pypy=IS_PYPY)
-        (lhs, rhs, tokens,
-         right_recursive, dup_rhs) = p.check_sets()
-        expect_lhs = set(['expr1024', 'pos_arg'])
-        unused_rhs = set(['list', 'call', 'mkfunc',
-                          'mklambda',
-                          'unpack',])
-
-        expect_right_recursive = frozenset([('designList',
-                                             ('store', 'DUP_TOP', 'designList'))])
-        expect_lhs.add('kwarg')
-
-        self.assertEqual(expect_lhs, set(lhs))
-        self.assertEqual(unused_rhs, set(rhs))
-        self.assertEqual(expect_right_recursive, right_recursive)
-
-        expect_dup_rhs = frozenset([('COME_FROM',), ('CONTINUE',), ('JUMP_ABSOLUTE',),
-                                    ('LOAD_CONST',),
-                                    ('JUMP_BACK',), ('JUMP_FORWARD',)])
-
-        reduced_dup_rhs = {}
-        for k in dup_rhs:
-            if k not in expect_dup_rhs:
-                reduced_dup_rhs[k] = dup_rhs[k]
-                pass
-            pass
-        for k in reduced_dup_rhs:
-            print(k, reduced_dup_rhs[k])
-        # assert not reduced_dup_rhs, reduced_dup_rhs
-
-    def test_dup_rule(self):
-        import inspect
-        python_parser(PYTHON_VERSION, inspect.currentframe().f_code,
-                      is_pypy=IS_PYPY,
-                      parser_debug={
-                          'dups': True, 'transition': False, 'reduce': False,
-                          'rules': False, 'errorstack': None, 'context': True})
-if __name__ == '__main__':
-    unittest.main()

test/Makefile

@@ -28,7 +28,7 @@ check:
 	$(MAKE) check-$(PYTHON_VERSION)
 
 #: Run working tests from Python 2.6 or 2.7
-check-2.4 check-2.5 check-2.6 check-2.7: check-bytecode-2 check-bytecode-3 check-bytecode-1  check-native-short
+check-2.6 check-2.7: check-bytecode-2 check-bytecode-3 check-bytecode-1 check-native-short
 
 #: Run working tests from Python 3.0
 check-3.0: check-bytecode
@@ -37,6 +37,7 @@ check-3.0: check-bytecode
 #: Run working tests from Python 3.1
 check-3.1: check-bytecode
 	$(PYTHON) test_pythonlib.py --bytecode-3.1 --weak-verify $(COMPILE)
+	$(PYTHON) test_pythonlib.py --bytecode-3.1-run --verify-run
 
 #: Run working tests from Python 3.2
 check-3.2: check-bytecode
@@ -201,6 +202,7 @@ check-bytecode-3.0:
 #: Check deparsing Python 3.1
 check-bytecode-3.1:
 	$(PYTHON) test_pythonlib.py --bytecode-3.1 --weak-verify
+	$(PYTHON) test_pythonlib.py --bytecode-3.1-run --verify-run
 
 #: Check deparsing Python 3.2
 check-bytecode-3.2:
@@ -232,10 +234,6 @@ check-native-short:
 	$(PYTHON) test_pythonlib.py --bytecode-$(PYTHON_VERSION) --weak-verify $(COMPILE)
 	$(PYTHON) test_pythonlib.py --bytecode-$(PYTHON_VERSION)-run --verify-run $(COMPILE)
 
-#: Run longer Python 2.6's lib files known to be okay
-check-2.4-ok:
-	$(PYTHON) test_pythonlib.py --ok-2.4 --verify $(COMPILE)
-
 #: Run longer Python 2.6's lib files known to be okay
 check-2.6-ok:
 	$(PYTHON) test_pythonlib.py --ok-2.6 --weak-verify $(COMPILE)

test/dis-compare.py

@@ -1,8 +1,9 @@
 #!/usr/bin/env python
 # Mode: -*- python -*-
 #
-# Copyright (c) 2015, 2017 by Rocky Bernstein <rb@dustyfeet.com>
+# Copyright (c) 2015 by Rocky Bernstein <rb@dustyfeet.com>
 #
+from __future__ import print_function
 
 
 import dis, os.path

test/grammar-cover/grammar.sh

@@ -30,7 +30,7 @@ while [[ -n $1 ]]  ; do
         source ./admin-tools/setup-python-2.4.sh
     fi
     GRAMMAR_TXT=$tmpdir/grammar-${SHORT_VERSION}.txt
-    pyenv local ${LONG_VERSION}
+    (cd ../.. && pyenv local ${LONG_VERSION})
     cd ./test
     if [[ -r $COVER_FILE ]]; then
 	rm $COVER_FILE

test/run-and-email.sh

@@ -14,7 +14,7 @@ function displaytime {
     printf '%d seconds\n' $S
 }
 
-PYVERSION=${PYVERSION:-"3.5.5 2.7.14 3.4.8 2.6.9"}
+PYVERSION=${PYVERSION:-"3.5.5 2.7.14 3.2.6 3.3.7 3.4.8 2.6.9 3.6.4"}
 # PYVERSION=${PYVERSION:-"3.5.5"}
 
 USER=${USER:-rocky}
@@ -28,6 +28,10 @@ for VERSION in $PYVERSION ; do
 
     if [[ $VERSION == '3.5.5' ]] ; then
 	MAX_TESTS=224
+    elif [[ $VERSION == '3.2.6' ]] ; then
+	MAX_TESTS=700
+    elif [[ $VERSION == '3.6.4' ]] ; then
+	MAX_TESTS=400
     else
 	MAX_TESTS=800
     fi
@@ -42,7 +46,7 @@ for VERSION in $PYVERSION ; do
       rc=$?
 
       echo Python Version $(pyenv local) >> $LOGFILE
-      echo "" >>LOGFILE
+      echo "" >>$LOGFILE
 
       typeset -i ALL_FILES_ENDTIME=$(date +%s)
       (( time_diff =  ALL_FILES_ENDTIME - ALL_FILES_STARTTIME))

test/simple-uncompyle-code-test.py

@@ -1,5 +1,7 @@
 #!/usr/bin/env python
 
+from __future__ import print_function
+
 from uncompyle6 import uncompyle
 import sys, inspect
 

test/simple_source/bug25/02_yield_bug.py

@@ -1,7 +0,0 @@
-# From 2.4 test_array.py
-# In Python 2.4 and earlier "yield" is not valid and instead
-# we must use "yield None". Bug was not adding "None"
-
-def yield_bug():
-    yield None
-    return

test/simple_source/bug25/04_try_else_confuse.py

@@ -1,19 +0,0 @@
-# From 2.4 test_sax.py
-# Bug was distinguishing try from try/else
-
-def verify_empty_attrs():
-    gvqk = 3
-    try:
-        gvk = 1/0
-    except ZeroDivisionError:
-        gvk = 1
-
-    try:
-        gvqk = 0
-    except KeyError:
-        gvqk = 1
-
-    # If try/else was used above the return will be 4
-    return gvk + gvqk
-
-assert 1 == verify_empty_attrs()

test/simple_source/bug31/05_abc_test.py

@@ -0,0 +1,18 @@
+# Extracted from Python 3.5 test_abc.py
+# Bug is class having only a single kwarg
+# subclass.
+import abc
+import unittest
+from inspect import isabstract
+
+def test_abstractmethod_integration(self):
+    for abstractthing in [abc.abstractmethod]:
+        class C(metaclass=abc.ABCMeta):
+            @abstractthing
+            def foo(self): pass  # abstract
+            def bar(self): pass  # concrete
+        assert C.__abstractmethods__, {"foo"}
+        assert isabstract(C)
+        pass
+
+test_abstractmethod_integration(None)

test/simple_source/bug33/02_named_and_kwargs.py

@@ -1,5 +1,6 @@
 # Python 3.6 subprocess.py bug
 # Bug is getting params correct: timeout before **kwargs
+import subprocess
 def call(*popenargs, timeout=None, **kwargs):
     return
 
@@ -14,6 +15,9 @@ def subprocess_shell(self, protocol_factory, cmd, *, stdin=subprocess.PIPE,
 # From 3.4 asyncio/locks.py
 # Bug was handling" "value=1, *"
 
+class Semaphore:
+    pass
+
 class BoundedSemaphore(Semaphore):
     def __init__(self, value=1, *, loop=None):
         super().__init__(value, loop=loop)

test/simple_source/bug35/04_call_function.py

@@ -1,5 +1,6 @@
 # From sql/schema.py and 3.5 _strptime.py
-# Note that kwargs comes before "positional" args
+# Bug was code not knowing which Python versions
+# have kwargs coming before positional args in code.
 
 # RUNNABLE!
 
@@ -58,3 +59,23 @@ def __call__(self, *args, **kwds):
 # From 3.6.4 shutil
 def unpack_archive(func, filename, dict, format_info, extract_dir=None):
     func(filename, extract_dir, **dict(format_info[2]))
+
+# From 3.5.5 test_xrdrlib.py
+import xdrlib
+def assertRaisesConversion(self, *args):
+   self.assertRaises(xdrlib.ConversionError, *args)
+
+# From 3.2.6 _pyio.py
+class BlockingIOError(IOError):
+    def __init__(self, errno, strerror, characters_written=5):
+        super().__init__(errno, strerror)
+
+# From urllib/parse.py
+# Bug was using a subclass made from a call (to namedtuple)
+from collections import namedtuple
+
+class ResultMixin(object):
+    pass
+
+class SplitResult(namedtuple('SplitResult', 'scheme netloc path query fragment'), ResultMixin):
+    pass

test/simple_source/bug35/07_build_map_unpack.py

@@ -0,0 +1,10 @@
+# Adapted from Python 3.6 trace.py
+# Bug was in handling BUID_TUPLE_UNPACK created via
+# *opts.arguments
+import argparse
+parser = argparse.ArgumentParser()
+parser.add_argument('filename', nargs='?')
+parser.add_argument('arguments', nargs=argparse.REMAINDER)
+opts = parser.parse_args(["foo", "a", "b"])
+argv = opts.filename, *opts.arguments
+assert argv == ('foo', 'a', 'b')

test/stdlib/run-and-email.sh

@@ -2,7 +2,7 @@
 USER=${USER:-rocky}
 EMAIL=${EMAIL:-rb@dustyfeet.com}
 SUBJECT_PREFIX="stdlib unit testing for"
-for VERSION in 3.4.8 3.5.5 3.6.4 ; do
+for VERSION in 2.6.9 2.7.14 3.4.8 3.5.5 3.6.4 ; do
     typeset -i rc=0
     LOGFILE=/tmp/runtests-$VERSION-$$.log
     if ! pyenv local $VERSION ; then

test/stdlib/runtests.sh

@@ -35,10 +35,10 @@ case $PYVERSION in
     2.5)
 	SKIP_TESTS=(
 	    [test_contextlib.py]=1 # Syntax error - look at
-	    [test_dis.py]=1        # We change line numbers - duh!
-	    [test_grammar.py]=1    # Too many stmts. Handle large stmts
-	    [test_grp.py]=1        # Long test - might work Control flow?
-	    [test_pdb.py]=1        # Line-number specific
+	    [test_dis.py]=1   # We change line numbers - duh!
+	    [test_grammar.py]=1  # Too many stmts. Handle large stmts
+	    [test_grp.py]=1      # Long test - might work Control flow?
+	    [test_pdb.py]=1      # Line-number specific
 	    [test_pwd.py]=1 # Long test - might work? Control flow?
 	    [test_queue.py]=1 # Control flow?
 	    [test_re.py]=1 # Probably Control flow?
@@ -101,8 +101,10 @@ case $PYVERSION in
     3.5)
 	SKIP_TESTS=(
 	    [test_decorators.py]=1  # Control flow wrt "if elif"
+	    [test_quopri.py]=1      # Fails in crontab environment?
 	)
 	;;
+
     3.6)
 	SKIP_TESTS=(
 	    [test_contains.py]=1    # Code "while False: yield None" is optimized away in compilation

test/test_pyenvlib.py

@@ -20,7 +20,8 @@ Step 2: Run the test:
 	  test_pyenvlib --mylib --verify # decompile verify 'mylib'
 """
 
-from uncompyle6 import main, PYTHON3
+from __future__ import print_function
+
 import os, time, re, shutil, sys
 from fnmatch import fnmatch
 

test/test_pythonlib.py

@@ -27,6 +27,8 @@ Step 2: Run the test:
   test_pythonlib.py --mylib --verify # decompile verify 'mylib'
 """
 
+from __future__ import print_function
+
 import getopt, os, py_compile, sys, shutil, tempfile, time
 
 from uncompyle6 import PYTHON_VERSION
@@ -128,10 +130,8 @@ def do_tests(src_dir, obj_patterns, target_dir, opts):
     if opts['do_compile']:
         compiled_version = opts['compiled_version']
         if compiled_version and PYTHON_VERSION != compiled_version:
-            sys.stderr.write("Not compiling: "
-                             "desired Python version is %s "
-                             "but we are running %s" %
-                             (compiled_version, PYTHON_VERSION))
+            print("Not compiling: desired Python version is %s but we are running %s" %
+                  (compiled_version, PYTHON_VERSION), file=sys.stderr)
         else:
             for root, dirs, basenames in os.walk(src_dir):
                 file_matches(files, root, basenames, PY)
@@ -149,8 +149,8 @@ def do_tests(src_dir, obj_patterns, target_dir, opts):
         file_matches(files, dirname, basenames, obj_patterns)
 
     if not files:
-        sys.stderr.write("Didn't come up with any files to test! "
-                         "Try with --compile?")
+        print("Didn't come up with any files to test! Try with --compile?",
+              file=sys.stderr)
         exit(1)
 
     os.chdir(cwd)
@@ -164,9 +164,9 @@ def do_tests(src_dir, obj_patterns, target_dir, opts):
         except ValueError:
             pass
 
-    print time.ctime()
-    print 'Source directory: ', src_dir
-    print 'Output directory: ', target_dir
+    print(time.ctime())
+    print('Source directory: ', src_dir)
+    print('Output directory: ', target_dir)
     try:
         _, _, failed_files, failed_verify = \
           main(src_dir, target_dir, files, [],
@@ -242,13 +242,14 @@ if __name__ == '__main__':
         if os.path.isdir(src_dir):
             checked_dirs.append([src_dir, pattern, target_dir])
         else:
-            sys.stderr.write("Can't find directory %s. Skipping" % src_dir)
+            print("Can't find directory %s. Skipping" % src_dir,
+                  file=sys.stderr)
             continue
         last_compile_version = compiled_version
         pass
 
     if not checked_dirs:
-        sys.stderr.write("No directories found to check\n")
+        print("No directories found to check", file=sys.stderr)
         sys.exit(1)
 
     test_opts['compiled_version'] = last_compile_version

uncompyle6/bin/pydisassemble.py

@@ -3,6 +3,7 @@
 #
 # Copyright (c) 2015-2016, 2018 by Rocky Bernstein <rb@dustyfeet.com>
 #
+from __future__ import print_function
 import sys, os, getopt
 
 from uncompyle6.disas import disassemble_file
@@ -12,8 +13,8 @@ program, ext = os.path.splitext(os.path.basename(__file__))
 
 __doc__ = """
 Usage:
-  %s [OPTIONS]... FILE
-  %s [--help | -h | -V | --version]
+  {0} [OPTIONS]... FILE
+  {0} [--help | -h | -V | --version]
 
 Disassemble FILE with the instruction mangling that is done to
 assist uncompyle6 in parsing the instruction stream. For example
@@ -22,9 +23,9 @@ BUILD_LIST have arguement counts appended to the instruction name, and
 COME_FROM instructions are inserted into the instruction stream.
 
 Examples:
-  %s foo.pyc
-  %s foo.py    # same thing as above but find the file
-  %s foo.pyc bar.pyc  # disassemble foo.pyc and bar.pyc
+  {0} foo.pyc
+  {0} foo.py    # same thing as above but find the file
+  {0} foo.pyc bar.pyc  # disassemble foo.pyc and bar.pyc
 
 See also `pydisasm' from the `xdis' package.
 
@@ -32,7 +33,7 @@ Options:
   -V | --version     show version and stop
   -h | --help        show this message
 
-""" % ((program,) * 5)
+""".format(program)
 
 PATTERNS = ('*.pyc', '*.pyo')
 
@@ -41,15 +42,15 @@ def main():
 Type -h for for full help.""" % program
 
     if len(sys.argv) == 1:
-        sys.stderr.write("No file(s) given\n")
-        sys.stderr.write(Usage_short)
+        print("No file(s) given", file=sys.stderr)
+        print(Usage_short, file=sys.stderr)
         sys.exit(1)
 
     try:
         opts, files = getopt.getopt(sys.argv[1:], 'hVU',
                                     ['help', 'version', 'uncompyle6'])
-    except getopt.GetoptError(e):
-        sys.stderr.write('%s: %s' % (os.path.basename(sys.argv[0]), e))
+    except getopt.GetoptError as e:
+        print('%s: %s' % (os.path.basename(sys.argv[0]), e),  file=sys.stderr)
         sys.exit(-1)
 
     for opt, val in opts:
@@ -61,14 +62,15 @@ Type -h for for full help.""" % program
             sys.exit(0)
         else:
             print(opt)
-            sys.stderr.write(Usage_short)
+            print(Usage_short, file=sys.stderr)
             sys.exit(1)
 
     for file in files:
         if os.path.exists(files[0]):
             disassemble_file(file, sys.stdout)
         else:
-            sys.stderr.write("Can't read %s - skipping\n" % files[0])
+            print("Can't read %s - skipping" % files[0],
+                  file=sys.stderr)
             pass
         pass
     return

uncompyle6/bin/uncompile.py

@@ -4,6 +4,7 @@
 # Copyright (c) 2015-2017 by Rocky Bernstein
 # Copyright (c) 2000-2002 by hartmut Goebel <h.goebel@crazy-compilers.com>
 #
+from __future__ import print_function
 import sys, os, getopt, time
 
 program = 'uncompyle6'
@@ -67,11 +68,11 @@ def usage():
 
 
 def main_bin():
-    if not (sys.version_info[0:2] in ((2, 4), (2, 5), (2, 6), (2, 7),
-                                      (3, 2), (3, 3),
+    if not (sys.version_info[0:2] in ((2, 6), (2, 7),
+                                      (3, 1), (3, 2), (3, 3),
                                       (3, 4), (3, 5), (3, 6))):
-        sys.stderr.write('Error: %s requires Python 2.4 2.5 2.6, 2.7, '
-                         '3.2, 3.3, 3.4, 3.5, or 3.6' % program)
+        print('Error: %s requires Python 2.6-2.7, or 3.1-3.6' % program,
+              file=sys.stderr)
         sys.exit(-1)
 
     do_verify = recurse_dirs = False
@@ -87,8 +88,8 @@ def main_bin():
                                     'help asm grammar linemaps recurse timestamp tree '
                                     'fragments verify verify-run version weak-verify '
                                     'showgrammar'.split(' '))
-    except getopt.GetoptError(e):
-        sys.stderr.write('%s: %s\n' % (os.path.basename(sys.argv[0]), e))
+    except getopt.GetoptError as e:
+        print('%s: %s' % (os.path.basename(sys.argv[0]), e),  file=sys.stderr)
         sys.exit(-1)
 
     options = {}
@@ -128,7 +129,7 @@ def main_bin():
         elif opt in ('--recurse', '-r'):
             recurse_dirs = True
         else:
-            sys.stderr.write(opt)
+            print(opt, file=sys.stderr)
             usage()
 
     # expand directory if specified
@@ -153,7 +154,7 @@ def main_bin():
         files = [f[sb_len:] for f in files]
 
     if not files:
-        sys.stderr.write("No files given\n")
+        print("No files given", file=sys.stderr)
         usage()
 
     if outfile == '-':

uncompyle6/disas.py

@@ -29,6 +29,8 @@ Second, we need structured instruction information for the
 want to run on earlier Python versions.
 """
 
+from __future__ import print_function
+
 import sys
 from collections import deque
 
@@ -47,9 +49,10 @@ def disco(version, co, out=None, is_pypy=False):
 
     # store final output stream for case of error
     real_out = out or sys.stdout
-    real_out.write('# Python %s\n' % version)
+    print('# Python %s' % version, file=real_out)
     if co.co_filename:
-        real_out.write('# Embedded file name: %s\n' % co.co_filename)
+        print('# Embedded file name: %s' % co.co_filename,
+              file=real_out)
 
     scanner = get_scanner(version, is_pypy=is_pypy)
 
@@ -61,15 +64,16 @@ def disco_loop(disasm, queue, real_out):
     while len(queue) > 0:
         co = queue.popleft()
         if co.co_name != '<module>':
-            real_out.write('\n# %s line %d of %s\n' %
-                      (co.co_name, co.co_firstlineno, co.co_filename))
+            print('\n# %s line %d of %s' %
+                      (co.co_name, co.co_firstlineno, co.co_filename),
+                      file=real_out)
         tokens, customize = disasm(co)
         for t in tokens:
             if iscode(t.pattr):
                 queue.append(t.pattr)
             elif iscode(t.attr):
                 queue.append(t.attr)
-            real_out.write(t)
+            print(t, file=real_out)
             pass
         pass
 

uncompyle6/linenumbers.py

@@ -25,7 +25,7 @@ def line_number_mapping(pyc_filename, src_filename):
      source_size) = load_module(pyc_filename)
     try:
         code2 = load_file(src_filename)
-    except SyntaxError, e:
+    except SyntaxError as e:
         return str(e)
 
     queue = deque([code1, code2])

uncompyle6/main.py

@@ -12,10 +12,10 @@
 #
 #  You should have received a copy of the GNU General Public License
 #  along with this program.  If not, see <http://www.gnu.org/licenses/>.
+from __future__ import print_function
+import datetime, os, subprocess, sys, tempfile
 
-import datetime, os, subprocess, sys
-
-from uncompyle6 import verify, IS_PYPY
+from uncompyle6 import verify, IS_PYPY, PYTHON_VERSION
 from xdis.code import iscode
 from uncompyle6.disas import check_object_path
 from uncompyle6.semantics import pysource
@@ -38,7 +38,11 @@ def _get_outstream(outfile):
         os.makedirs(dir)
     except OSError:
         pass
-    return open(outfile, 'wb')
+    if PYTHON_VERSION < 3.0:
+        mode = 'wb'
+    else:
+        mode = 'w'
+    return open(outfile, mode)
 
 def decompile(
         bytecode_version, co, out=None, showasm=None, showast=False,
@@ -59,35 +63,20 @@ def decompile(
 
     assert iscode(co)
 
-    if is_pypy:
-        co_pypy_str = 'PyPy '
-    else:
-        co_pypy_str = ''
-
-    if IS_PYPY:
-        run_pypy_str = 'PyPy '
-    else:
-        run_pypy_str = ''
-
-    if magic_int:
-        m = str(magic_int)
-    else:
-        m = ""
-
+    co_pypy_str = 'PyPy ' if is_pypy else ''
+    run_pypy_str = 'PyPy ' if IS_PYPY else ''
     sys_version_lines = sys.version.split('\n')
     write('# uncompyle6 version %s\n'
           '# %sPython bytecode %s%s\n# Decompiled from: %sPython %s' %
           (VERSION, co_pypy_str, bytecode_version,
-               " (%s)" % m, run_pypy_str,
-           '\n# '.join(sys_version_lines)))
+               " (%s)" % str(magic_int) if magic_int else "",
+          run_pypy_str, '\n# '.join(sys_version_lines)))
     if co.co_filename:
         write('# Embedded file name: %s' % co.co_filename,)
     if timestamp:
-        write('# Compiled at: %s' %
-              datetime.datetime.fromtimestamp(timestamp))
+        write('# Compiled at: %s' % datetime.datetime.fromtimestamp(timestamp))
     if source_size:
-        real_out.write('# Size of source mod 2**32: %d bytes\n' %
-                       source_size)
+        write('# Size of source mod 2**32: %d bytes' % source_size)
 
     debug_opts = {
         'asm': showasm,
@@ -120,7 +109,7 @@ def decompile(
                                   is_pypy=is_pypy)
             pass
         return deparsed
-    except pysource.SourceWalkerError, e:
+    except pysource.SourceWalkerError as e:
         # deparsing failed
         raise pysource.SourceWalkerError(str(e))
 
@@ -200,10 +189,17 @@ def main(in_base, out_base, files, codes, outfile=None,
                     prefix = prefix[:-len('.py')]
 
                 # Unbuffer output if possible
-                if sys.stdout.isatty():
-                    buffering = -1
+                buffering = -1 if sys.stdout.isatty() else 0
+                if PYTHON_VERSION >= 3.5:
+                    t = tempfile.NamedTemporaryFile(mode='w+b',
+                                                    buffering=buffering,
+                                                    suffix='.py',
+                                                    prefix=prefix)
                 else:
-                    buffering = 0
+                    t = tempfile.NamedTemporaryFile(mode='w+b',
+                                                    suffix='.py',
+                                                    prefix=prefix)
+                current_outfile = t.name
                 sys.stdout = os.fdopen(sys.stdout.fileno(), 'w', buffering)
                 tee = subprocess.Popen(["tee", current_outfile],
                                        stdin=subprocess.PIPE)
@@ -242,9 +238,9 @@ def main(in_base, out_base, files, codes, outfile=None,
                     pass
                 pass
             tot_files += 1
-        except (ValueError, SyntaxError, ParserError, pysource.SourceWalkerError):
+        except (ValueError, SyntaxError, ParserError, pysource.SourceWalkerError) as e:
             sys.stdout.write("\n")
-            sys.stderr.write("# file %s\n" % (infile))
+            sys.stderr.write("\n# file %s\n# %s\n" % (infile, e))
             failed_files += 1
             tot_files += 1
         except KeyboardInterrupt:
@@ -282,16 +278,13 @@ def main(in_base, out_base, files, codes, outfile=None,
                         else:
                             okay_files += 1
                             pass
-                    except verify.VerifyCmpError, e:
+                    except verify.VerifyCmpError as e:
                         print(e)
                         verify_failed_files += 1
                         os.rename(current_outfile, current_outfile + '_unverified')
                         sys.stderr.write("### Error Verifying %s\n" % filename)
                         sys.stderr.write(str(e) + "\n")
                         if not outfile:
-                            sys.stderr.write("### Error Verifiying %s" %
-                                            filename)
-                            sys.stderr.write(e)
                             if raise_on_error:
                                 raise
                             pass
@@ -301,15 +294,14 @@ def main(in_base, out_base, files, codes, outfile=None,
                     okay_files += 1
                 pass
             elif do_verify:
-                sys.stderr.write("\n### uncompile successful, "
-                                 "but no file to compare against")
+                sys.stderr.write("\n### uncompile successful, but no file to compare against\n")
                 pass
             else:
                 okay_files += 1
                 if not current_outfile:
                     mess = '\n# okay decompiling'
                     # mem_usage = __memUsage()
-                    print mess, infile
+                    print(mess, infile)
         if current_outfile:
             sys.stdout.write("%s\r" %
                              status_msg(do_verify, tot_files, okay_files, failed_files,

uncompyle6/parser.py

@@ -19,6 +19,8 @@
 Common uncompyle6 parser routines.
 """
 
+from __future__ import print_function
+
 import sys
 
 from xdis.code import iscode
@@ -179,7 +181,7 @@ class PythonParser(GenericASTBuilder):
                     indent = '   '
                 else:
                     indent = '-> '
-                print "%s%s" % (indent, instructions[i])
+                print("%s%s" % (indent, instructions[i]))
             raise ParserError(err_token, err_token.offset)
         else:
             raise ParserError(None, -1)
@@ -773,4 +775,4 @@ if __name__ == '__main__':
         ast = python_parser(PYTHON_VERSION, co, showasm=True, is_pypy=IS_PYPY)
         print(ast)
         return
-    # parse_test(parse_test.__code__)
+    parse_test(parse_test.__code__)

uncompyle6/parsers/parse2.py

@@ -25,6 +25,8 @@ If we succeed in creating a parse tree, then we have a Python program
 that a later phase can turn into a sequence of ASCII text.
 """
 
+from __future__ import print_function
+
 from uncompyle6.parser import PythonParser, PythonParserSingle, nop_func
 from uncompyle6.parsers.astnode import AST
 from spark_parser import DEFAULT_DEBUG as PARSER_DEFAULT_DEBUG

uncompyle6/parsers/parse24.py

@@ -82,6 +82,15 @@ class Python24Parser(Python25Parser):
             l = len(tokens)
             if 0 <= l < len(tokens):
                 return not int(tokens[first].pattr) == tokens[last].offset
+        elif lhs == 'try_except':
+            if last == len(tokens):
+                last -= 1
+            if tokens[last] != 'COME_FROM' and tokens[last-1] == 'COME_FROM':
+                last -= 1
+            return (tokens[last] == 'COME_FROM'
+                    and tokens[last-1] == 'END_FINALLY'
+                    and tokens[last-2] == 'POP_TOP'
+                    and tokens[last-3].kind != 'JUMP_FORWARD')
 
         return False
 

uncompyle6/parsers/parse25.py

@@ -19,9 +19,6 @@ class Python25Parser(Python26Parser):
 
         return_if_stmt ::= ret_expr  RETURN_END_IF JUMP_BACK
 
-        # We have no jumps to jumps, so no "come_froms" but a single "COME_FROM"
-        ifelsestmt ::= testexpr c_stmts_opt jf_cf_pop else_suite COME_FROM
-
         # Python 2.6 uses ROT_TWO instead of the STORE_xxx
         # withas is allowed as a "from future" in 2.5
         # 2.6 and 2.7 do something slightly different
@@ -39,8 +36,6 @@ class Python25Parser(Python26Parser):
         # loop. FIXME: should "come_froms" below be a single COME_FROM?
         tryelsestmt    ::= SETUP_EXCEPT suite_stmts_opt POP_BLOCK
                            except_handler else_suite come_froms
-        tryelsestmtl   ::= SETUP_EXCEPT suite_stmts_opt POP_BLOCK
-                            except_handler else_suitel
 
         # Python 2.6 omits the LOAD_FAST DELETE_FAST below
         # withas is allowed as a "from future" in 2.5
@@ -59,9 +54,6 @@ class Python25Parser(Python26Parser):
     def customize_grammar_rules(self, tokens, customize):
         # Remove grammar rules inherited from Python 2.6 or Python 2
         self.remove_rules("""
-        # No jump to jumps in 2.4 so we have a single "COME_FROM", not "come_froms"
-        ifelsestmt    ::= testexpr c_stmts_opt jf_cf_pop else_suite come_froms
-
         setupwith  ::= DUP_TOP LOAD_ATTR ROT_TWO LOAD_ATTR CALL_FUNCTION_0 POP_TOP
         withstmt   ::= expr setupwith SETUP_FINALLY suite_stmts_opt
                        POP_BLOCK LOAD_CONST COME_FROM WITH_CLEANUP END_FINALLY

uncompyle6/parsers/parse27.py

@@ -226,5 +226,4 @@ if __name__ == '__main__':
                              for t in remain_tokens])
         remain_tokens = set(remain_tokens) - opcode_set
         print(remain_tokens)
-        p.check_grammar()
-        p.dump_grammar()
+        # p.dump_grammar()

uncompyle6/parsers/parse3.py

@@ -30,7 +30,6 @@ from uncompyle6.parser import PythonParser, PythonParserSingle, nop_func
 from uncompyle6.parsers.astnode import AST
 from spark_parser import DEFAULT_DEBUG as PARSER_DEFAULT_DEBUG
 from xdis import PYTHON3
-from itertools import islice,chain,repeat
 
 class Python3Parser(PythonParser):
 
@@ -117,9 +116,14 @@ class Python3Parser(PythonParser):
 
         classdef ::= build_class store
 
+        # FIXME: we need to add these because don't detect this properly
+        # in custom rules. Specifically if one of the exprs is CALL_FUNCTION
+        # then we'll mistake that for the final CALL_FUNCTION.
+        # We can fix by triggering on the CALL_FUNCTION op
         # Python3 introduced LOAD_BUILD_CLASS
         # Other definitions are in a custom rule
         build_class ::= LOAD_BUILD_CLASS mkfunc expr call CALL_FUNCTION_3
+        build_class ::= LOAD_BUILD_CLASS mkfunc expr call expr CALL_FUNCTION_4
 
         stmt ::= classdefdeco
         classdefdeco ::= classdefdeco1 store
@@ -424,7 +428,7 @@ class Python3Parser(PythonParser):
                         LOAD_CONST CALL_FUNCTION_n
         build_class ::= LOAD_BUILD_CLASS mkfunc
                         expr
-                        call_function
+                        call
                         CALL_FUNCTION_3
          '''
         # FIXME: I bet this can be simplified
@@ -627,7 +631,7 @@ class Python3Parser(PythonParser):
                 self.addRule(rule, nop_func)
                 rule = 'expr ::= build_list_unpack'
                 self.addRule(rule, nop_func)
-            elif opname_base == 'BUILD_MAP':
+            elif opname_base in ('BUILD_MAP', 'BUILD_MAP_UNPACK'):
                 kvlist_n = "kvlist_%s" % token.attr
                 if opname == 'BUILD_MAP_n':
                     # PyPy sometimes has no count. Sigh.
@@ -646,23 +650,21 @@ class Python3Parser(PythonParser):
                         # FIXME: Use the attr
                         # so this doesn't run into exponential parsing time.
                         if opname.startswith('BUILD_MAP_UNPACK'):
-                            # FIXME: start here
-                            # rule = "%s ::= %s %s" % (kvlist_n, 'expr ' * (token.attr*2), opname)
-                            rule = kvlist_n + ' ::= ' + 'expr ' * (token.attr*2)
                             self.add_unique_rule(rule, opname, token.attr, customize)
                             rule = 'dict_entry ::= ' + 'expr ' * (token.attr*2)
                             self.add_unique_rule(rule, opname, token.attr, customize)
-                            rule = 'dict ::= %s' % ('dict_entry ' * token.attr)
-                            self.addRule(rule, nop_func)
 
+                            # FIXME: start here. The LHS should be unmap_dict, not dict.
                             # FIXME: really we need a combination of dict_entry-like things.
                             # It just so happens the most common case is not to mix
                             # dictionary comphensions with dictionary, elements
                             if self.seen_LOAD_DICTCOMP:
                                 rule = 'dict ::= %s%s' % ('dict_comp ' * token.attr, opname)
                                 self.addRule(rule, nop_func)
-
-                            rule = 'unmap_dict ::= %s%s' % (('dict ' * token.attr), opname)
+                            rule = """
+                             expr       ::= unmap_dict
+                             unmap_dict ::= %s%s
+                             """ % ('expr ' * token.attr, opname)
                         else:
                             rule = "%s ::= %s %s" % (kvlist_n, 'expr ' * (token.attr*2), opname)
                             self.add_unique_rule(rule, opname, token.attr, customize)
@@ -680,7 +682,9 @@ class Python3Parser(PythonParser):
                 v = token.attr
                 rule = ('starred ::= %s %s' % ('expr ' * v, opname))
                 self.addRule(rule, nop_func)
-            elif opname_base in ('BUILD_LIST', 'BUILD_SET', 'BUILD_TUPLE'):
+
+            elif opname_base in ('BUILD_LIST', 'BUILD_SET', 'BUILD_TUPLE',
+                                 'BUILD_TUPLE_UNPACK'):
                 v = token.attr
 
                 is_LOAD_CLOSURE = False
@@ -1160,7 +1164,7 @@ class Python3Parser(PythonParser):
 
             if tokens[cfl-1] != 'JUMP_BACK':
                 cfl_offset = tokens[cfl-1].offset
-                insn = chain((i for i in self.insts if cfl_offset == i.offset), repeat(None)).next()
+                insn = next(i for i in self.insts if cfl_offset == i.offset)
                 if insn and  insn.is_jump_target:
                     return True
 

uncompyle6/parsers/parse30.py

@@ -2,6 +2,7 @@
 """
 spark grammar differences over Python 3.1 for Python 3.0.
 """
+from __future__ import print_function
 
 from uncompyle6.parser import PythonParserSingle
 from uncompyle6.parsers.parse31 import Python31Parser

uncompyle6/parsers/parse31.py

@@ -2,6 +2,7 @@
 """
 spark grammar differences over Python 3.2 for Python 3.1.
 """
+from __future__ import print_function
 
 from uncompyle6.parser import PythonParserSingle
 from uncompyle6.parsers.parse32 import Python32Parser

uncompyle6/parsers/parse32.py

@@ -2,6 +2,8 @@
 """
 spark grammar differences over Python 3 for Python 3.2.
 """
+from __future__ import print_function
+
 from uncompyle6.parser import PythonParserSingle
 from uncompyle6.parsers.parse3 import Python3Parser
 

uncompyle6/parsers/parse33.py

@@ -2,6 +2,7 @@
 """
 spark grammar differences over Python 3.2 for Python 3.3.
 """
+from __future__ import print_function
 
 from uncompyle6.parser import PythonParserSingle
 from uncompyle6.parsers.parse32 import Python32Parser