Merge branch 'master' into python-2.4

which are created via a call to create a subclass.

Should be more general though.

.gitignore

@@ -21,3 +21,4 @@
 ChangeLog
 __pycache__
 build
+nohup.out

.travis.yml

@@ -3,13 +3,7 @@ language: python
 sudo: false
 
 python:
-  - '3.5'
-  - '2.7'
-  - '2.6'
-  - '3.3'
-  - '3.4'
-  - '3.2'
-  - '3.6'
+  - '2.7' # this is a cheat here because travis doesn't do 2.4-2.6
 
 install:
 - pip install -e .

Makefile

@@ -27,19 +27,20 @@ check:
 check-short: pytest
 	$(MAKE) -C test check-short
 
+# Note for 2.6 use <=3.0.1 see requirements-dev.txt
 #: Tests for Python 2.7, 3.3 and 3.4
-check-2.7 check-3.3 check-3.4: pytest
+check-2.6 check-2.7 check-3.3 check-3.4 check-3.5: pytest
 	$(MAKE) -C test $@
 
 #: Tests for Python 3.2 and 3.5 - pytest doesn't work here
 # Or rather 3.5 doesn't work not on Travis
-check-3.0 check-3.1 check-3.2 check-3.5 check-3.6:
+check-3.0 check-3.1 check-3.2 check-3.6:
 	$(MAKE) -C test $@
 
 check-3.7: pytest
 
-#:Tests for Python 2.6 (doesn't have pytest)
-check-2.6:
+#:Tests for Python 2.4-2.5 (don't have pytest)
+check-2.4 check-2.5:
 	$(MAKE) -C test $@
 
 #:PyPy 2.6.1 PyPy 5.0.1, or PyPy 5.8.0-beta0

NEWS

@@ -1,3 +1,11 @@
+uncompyle6 3.1.2 2018-04-08 Eastern Orthodox Easter
+
+- Python 3.x subclass and call parsing fixes
+- Allow/note running on Python 3.1
+- improve 3.5+ BUILD_MAP_UNPACK
+- DRY instruction building code between 2.x and 3.x
+- expand testing
+
 uncompyle6 3.1.1 2018-04-01 Easter April Fool's
 
 Jesus on Friday's New York Times puzzle: "I'm stuck on 2A"

__pkginfo__.py

@@ -35,6 +35,8 @@ classifiers =  ['Development Status :: 5 - Production/Stable',
                 'Programming Language :: Python :: 2.5',
                 'Programming Language :: Python :: 2.6',
                 'Programming Language :: Python :: 2.7',
+                'Programming Language :: Python :: 3.1',
+                'Programming Language :: Python :: 3.2',
                 'Programming Language :: Python :: 3.3',
                 'Programming Language :: Python :: 3.4',
                 'Programming Language :: Python :: 3.5',
@@ -54,8 +56,7 @@ entry_points       = {
     ]}
 ftp_url            = None
 install_requires   = ['spark-parser >= 1.8.5, < 1.9.0',
-                      'xdis >= 3.7.0, < 3.8.0', 'six']
-
+                      'xdis >= 3.7.0, < 3.8.0']
 license            = 'GPL3'
 mailing_list       = 'python-debugger@googlegroups.com'
 modname            = 'uncompyle6'

admin-tools/how-to-make-a-release.txt

@@ -0,0 +1,46 @@
+git pull
+
+Change version in uncompyle6/version.py
+source uncompyle6/version.py
+echo $VERSION
+git commit -m"Get ready for release $VERSION" .
+
+Update ChangeLog:
+  make ChangeLog
+
+Update NEWS from ChangeLog
+make check
+
+git commit --amend .
+
+git push
+
+Make sure pyenv is running
+# Pyenv
+
+source admin-tools/check-newer-versions.sh
+
+
+# Switch to python-2.4 and build that first...
+source admin-tools/setup-python-2.4
+
+rm ChangeLog
+git merge master
+
+Update NEWS from master branch
+
+git commit -m"Get ready for release $VERSION" .
+
+source admin-tools/check-older-versions.sh
+source admin-tools/check-newer-versions.sh
+
+make-dist-older.sh
+
+git tag release-python-2.4-$VERSION
+
+./make-dist-newer.sh
+
+git tag release-$VERSION
+
+
+twine upload dist/uncompyle6-${VERSION}*

admin-tools/pyenv-newer-versions

@@ -5,4 +5,4 @@ if [[ $0 == ${BASH_SOURCE[0]} ]] ; then
     echo "This script should be *sourced* rather than run directly through bash"
     exit 1
 fi
-export PYVERSIONS='3.5.5 3.6.4 2.6.9 3.3.7 2.7.14 3.4.8'
+export PYVERSIONS='3.5.5 3.6.4 2.6.9 3.3.7 2.7.14 3.2.6 3.1.5 3.4.8'

admin-tools/update-sources.sh

@@ -0,0 +1,3 @@
+#!/bin/bash
+cd $(dirname ${BASH_SOURCE[0]})/..
+git pull

circle.yml

@@ -11,4 +11,4 @@ dependencies:
 test:
     override:
         - python ./setup.py develop && make check-2.7
-        - cd ./test/stdlib && pyenv local 2.7.10 && bash ./runtests.sh 'test_[p-z]*.py'
+        - cd ./test/stdlib && pyenv local 2.7.10 && bash ./runtests.sh 'test_[f-i]*.py'

pytest/test_deparse.py

@@ -30,7 +30,7 @@ def list_comp():
     [y for y in range(3)]
 
 def get_parsed_for_fn(fn):
-    code = fn.__code__ if PYTHON3 else fn.func_code
+    code = fn.func_code
     return deparse(code, version=PYTHON_VERSION)
 
 def check_expect(expect, parsed, fn_name):

pytest/test_docstring.py

@@ -10,7 +10,7 @@ else:
     maxint = sys.maxint
 from uncompyle6.semantics.helper import print_docstring
 
-class PrintFake():
+class PrintFake:
     def __init__(self):
         self.pending_newlines = 0
         self.f = StringIO()

pytest/test_fjt.py

@@ -1,8 +1,6 @@
 #!/usr/bin/env python
 from uncompyle6 import PYTHON_VERSION, IS_PYPY
 from uncompyle6.scanner import get_scanner
-from xdis.bytecode import Bytecode
-from array import array
 def bug(state, slotstate):
     if state:
         if slotstate is not None:
@@ -22,18 +20,10 @@ def bug_loop(disassemble, tb=None):
     disassemble(tb)
 
 def test_if_in_for():
-    code = bug.__code__
+    code = bug.func_code
     scan = get_scanner(PYTHON_VERSION)
-    print(PYTHON_VERSION)
     if 2.7 <= PYTHON_VERSION <= 3.0 and not IS_PYPY:
-        n = scan.setup_code(code)
-        bytecode = Bytecode(code, scan.opc)
-        scan.build_lines_data(code, n)
-        scan.insts = list(bytecode)
-        scan.offset2inst_index = {}
-        for i, inst in enumerate(scan.insts):
-            scan.offset2inst_index[inst.offset] = i
-        scan.build_prev_op(n)
+        scan.build_instructions(code)
         fjt = scan.find_jump_targets(False)
 
         ## FIXME: the data below is wrong.
@@ -48,14 +38,7 @@ def test_if_in_for():
         #    {'start': 62, 'end': 63, 'type': 'for-else'}]
 
         code = bug_loop.__code__
-        n = scan.setup_code(code)
-        bytecode = Bytecode(code, scan.opc)
-        scan.build_lines_data(code, n)
-        scan.insts = list(bytecode)
-        scan.build_prev_op(n)
-        scan.offset2inst_index = {}
-        for i, inst in enumerate(scan.insts):
-            scan.offset2inst_index[inst.offset] = i
+        scan.build_instructions(code)
         fjt = scan.find_jump_targets(False)
         assert{64: [42], 67: [42, 42], 42: [16, 41], 19: [6]} == fjt
         assert scan.structs == [
@@ -69,14 +52,7 @@ def test_if_in_for():
             {'start': 48, 'end': 67, 'type': 'while-loop'}]
 
     elif 3.2 < PYTHON_VERSION <= 3.4:
-        bytecode = Bytecode(code, scan.opc)
-        scan.code = array('B', code.co_code)
-        scan.lines = scan.build_lines_data(code)
-        scan.build_prev_op()
-        scan.insts = list(bytecode)
-        scan.offset2inst_index = {}
-        for i, inst in enumerate(scan.insts):
-            scan.offset2inst_index[inst.offset] = i
+        scan.build_instructions(code)
         fjt  = scan.find_jump_targets(False)
         assert {69: [66], 63: [18]} == fjt
         assert scan.structs == \
@@ -86,5 +62,6 @@ def test_if_in_for():
            {'end': 59, 'type': 'for-loop', 'start': 31},
            {'end': 63, 'type': 'for-else', 'start': 62}]
     else:
-        assert True, "FIXME: should note fixed"
+        print("FIXME: should fix for %s" % PYTHON_VERSION)
+        assert True
     return

pytest/test_fstring.py

@@ -1,150 +0,0 @@
-# std
-import os
-# test
-import pytest
-import hypothesis
-from hypothesis import strategies as st
-# uncompyle6
-from uncompyle6 import PYTHON_VERSION, deparse_code
-
-
-@st.composite
-def expressions(draw):
-    # todo : would be nice to generate expressions using hypothesis however
-    # this is pretty involved so for now just use a corpus of expressions
-    # from which to select.
-    return draw(st.sampled_from((
-        'abc',
-        'len(items)',
-        'x + 1',
-        'lineno',
-        'container',
-        'self.attribute',
-        'self.method()',
-        # These expressions are failing, I think these are control
-        # flow problems rather than problems with FORMAT_VALUE,
-        # however I need to confirm this...
-        #'sorted(items, key=lambda x: x.name)',
-        #'func(*args, **kwargs)',
-        #'text or default',
-        #'43 if life_the_universe and everything else None'
-    )))
-
-
-@st.composite
-def format_specifiers(draw):
-    """
-    Generate a valid format specifier using the rules:
-
-    format_spec ::=  [[fill]align][sign][#][0][width][,][.precision][type]
-    fill        ::=  <any character>
-    align       ::=  "<" | ">" | "=" | "^"
-    sign        ::=  "+" | "-" | " "
-    width       ::=  integer
-    precision   ::=  integer
-    type        ::=  "b" | "c" | "d" | "e" | "E" | "f" | "F" | "g" | "G" | "n" | "o" | "s" | "x" | "X" | "%"
-
-    See https://docs.python.org/2/library/string.html
-
-    :param draw: Let hypothesis draw from other strategies.
-
-    :return: An example format_specifier.
-    """
-    alphabet_strategy = st.characters(min_codepoint=ord('a'), max_codepoint=ord('z'))
-    fill = draw(st.one_of(alphabet_strategy, st.none()))
-    align = draw(st.sampled_from(list('<>=^')))
-    fill_align = (fill + align or '') if fill else ''
-
-    type_ = draw(st.sampled_from('bcdeEfFgGnosxX%'))
-    can_have_sign = type_ in 'deEfFgGnoxX%'
-    can_have_comma = type_ in 'deEfFgG%'
-    can_have_precision = type_ in 'fFgG'
-    can_have_pound = type_ in 'boxX%'
-    can_have_zero = type_ in 'oxX'
-
-    sign = draw(st.sampled_from(list('+- ') + [''])) if can_have_sign else ''
-    pound = draw(st.sampled_from(('#', '',))) if can_have_pound else ''
-    zero = draw(st.sampled_from(('0', '',))) if can_have_zero else ''
-
-    int_strategy = st.integers(min_value=1, max_value=1000)
-
-    width = draw(st.one_of(int_strategy, st.none()))
-    width = str(width) if width is not None else ''
-
-    comma = draw(st.sampled_from((',', '',))) if can_have_comma else ''
-    if can_have_precision:
-        precision = draw(st.one_of(int_strategy, st.none()))
-        precision = '.' + str(precision) if precision else ''
-    else:
-        precision = ''
-
-    return ''.join((fill_align, sign, pound, zero, width, comma, precision, type_,))
-
-
-@st.composite
-def fstrings(draw):
-    """
-    Generate a valid f-string.
-    See https://www.python.org/dev/peps/pep-0498/#specification
-
-    :param draw: Let hypothsis draw from other strategies.
-
-    :return: A valid f-string.
-    """
-    character_strategy = st.characters(
-        blacklist_characters='\r\n\'\\s{}',
-        min_codepoint=1,
-        max_codepoint=1000,
-    )
-    is_raw = draw(st.booleans())
-    integer_strategy = st.integers(min_value=0, max_value=3)
-    expression_count = draw(integer_strategy)
-    content = []
-    for _ in range(expression_count):
-        expression = draw(expressions())
-        conversion = draw(st.sampled_from(('', '!s', '!r', '!a',)))
-        has_specifier = draw(st.booleans())
-        specifier = ':' + draw(format_specifiers()) if has_specifier else ''
-        content.append('{{{}{}}}'.format(expression, conversion, specifier))
-        content.append(draw(st.text(character_strategy)))
-    content = ''.join(content)
-    return "f{}'{}'".format('r' if is_raw else '', content)
-
-
-@pytest.mark.skipif(PYTHON_VERSION < 3.6, reason='need at least python 3.6')
-@hypothesis.given(format_specifiers())
-def test_format_specifiers(format_specifier):
-    """Verify that format_specifiers generates valid specifiers"""
-    try:
-        exec('"{:' + format_specifier + '}".format(0)')
-    except ValueError as e:
-        if 'Unknown format code' not in str(e):
-            raise
-
-
-def run_test(text):
-    hypothesis.assume(len(text))
-    hypothesis.assume("f'{" in text)
-    expr = text + '\n'
-    code = compile(expr, '<string>', 'single')
-    deparsed = deparse_code(PYTHON_VERSION, code, compile_mode='single')
-    recompiled = compile(deparsed.text, '<string>', 'single')
-    if recompiled != code:
-        assert 'dis(' + deparsed.text.strip('\n') + ')' == 'dis(' + expr.strip('\n') + ')'
-
-
-@pytest.mark.skipif(PYTHON_VERSION < 3.6, reason='need at least python 3.6')
-@hypothesis.given(fstrings())
-def test_uncompyle_fstring(fstring):
-    """Verify uncompyling fstring bytecode"""
-    run_test(fstring)
-
-
-@pytest.mark.skipif(PYTHON_VERSION < 3.6, reason='need at least python 3.6')
-@pytest.mark.parametrize('fstring', [
-    "f'{abc}{abc!s}'",
-    "f'{abc}0'",
-])
-def test_uncompyle_direct(fstring):
-    """useful for debugging"""
-    run_test(fstring)

pytest/test_function_call.py

@@ -1,175 +0,0 @@
-# std
-import string
-# 3rd party
-from hypothesis import given, assume, example, settings, strategies as st
-import pytest
-# uncompyle
-from validate import validate_uncompyle
-from test_fstring import expressions
-
-
-alpha = st.sampled_from(string.ascii_lowercase)
-numbers = st.sampled_from(string.digits)
-alphanum = st.sampled_from(string.ascii_lowercase + string.digits)
-
-
-@st.composite
-def function_calls(draw,
-                   min_keyword_args=0, max_keyword_args=5,
-                   min_positional_args=0, max_positional_args=5,
-                   min_star_args=0, max_star_args=1,
-                   min_double_star_args=0, max_double_star_args=1):
-    """
-    Strategy factory for generating function calls.
-
-    :param draw: Callable which draws examples from other strategies.
-
-    :return: The function call text.
-    """
-    st_positional_args = st.lists(
-        alpha,
-        min_size=min_positional_args,
-        max_size=max_positional_args
-    )
-    st_keyword_args = st.lists(
-        alpha,
-        min_size=min_keyword_args,
-        max_size=max_keyword_args
-    )
-    st_star_args = st.lists(
-        alpha,
-        min_size=min_star_args,
-        max_size=max_star_args
-    )
-    st_double_star_args = st.lists(
-        alpha,
-        min_size=min_double_star_args,
-        max_size=max_double_star_args
-    )
-
-    positional_args = draw(st_positional_args)
-    keyword_args = draw(st_keyword_args)
-    st_values = st.lists(
-        expressions(),
-        min_size=len(keyword_args),
-        max_size=len(keyword_args)
-    )
-    keyword_args = [
-        x + '=' + e
-        for x, e in
-        zip(keyword_args, draw(st_values))
-    ]
-    star_args = ['*' + x for x in draw(st_star_args)]
-    double_star_args = ['**' + x for x in draw(st_double_star_args)]
-
-    arguments = positional_args + keyword_args + star_args + double_star_args
-    draw(st.randoms()).shuffle(arguments)
-    arguments = ','.join(arguments)
-
-    function_call = 'fn({arguments})'.format(arguments=arguments)
-    try:
-        # TODO: Figure out the exact rules for ordering of positional, keyword,
-        # star args, double star args and in which versions the various
-        # types of arguments are supported so we don't need to check that the
-        # expression compiles like this.
-        compile(function_call, '<string>', 'single')
-    except:
-        assume(False)
-    return function_call
-
-
-def test_function_no_args():
-    validate_uncompyle("fn()")
-
-
-def isolated_function_calls(which):
-    """
-    Returns a strategy for generating function calls, but isolated to 
-    particular types of arguments, for example only positional arguments.
-
-    This can help reason about debugging errors in specific types of function
-    calls.
-
-    :param which: One of 'keyword', 'positional', 'star', 'double_star'
-
-    :return: Strategy for generating an function call isolated to specific
-             argument types.
-    """
-    kwargs = dict(
-        max_keyword_args=0,
-        max_positional_args=0,
-        max_star_args=0,
-        max_double_star_args=0,
-    )
-    kwargs['_'.join(('min', which, 'args'))] = 1
-    kwargs['_'.join(('max', which, 'args'))] = 5 if 'star' not in which else 1
-    return function_calls(**kwargs)
-
-
-with settings(max_examples=25):
-
-    @given(isolated_function_calls('positional'))
-    @example("fn(0)")
-    def test_function_positional_only(expr):
-        validate_uncompyle(expr)
-
-    @given(isolated_function_calls('keyword'))
-    @example("fn(a=0)")
-    def test_function_call_keyword_only(expr):
-        validate_uncompyle(expr)
-
-    @given(isolated_function_calls('star'))
-    @example("fn(*items)")
-    def test_function_call_star_only(expr):
-        validate_uncompyle(expr)
-
-    @given(isolated_function_calls('double_star'))
-    @example("fn(**{})")
-    def test_function_call_double_star_only(expr):
-        validate_uncompyle(expr)
-
-
-@pytest.mark.xfail()
-def test_BUILD_CONST_KEY_MAP_BUILD_MAP_UNPACK_WITH_CALL_BUILD_TUPLE_CALL_FUNCTION_EX():
-    validate_uncompyle("fn(w=0,m=0,**v)")
-
-
-@pytest.mark.xfail()
-def test_BUILD_MAP_BUILD_MAP_UNPACK_WITH_CALL_BUILD_TUPLE_CALL_FUNCTION_EX():
-    validate_uncompyle("fn(a=0,**g)")
-
-
-@pytest.mark.xfail()
-def test_CALL_FUNCTION_EX():
-    validate_uncompyle("fn(*g,**j)")
-
-
-@pytest.mark.xfail()
-def test_BUILD_MAP_CALL_FUNCTION_EX():
-    validate_uncompyle("fn(*z,u=0)")
-
-
-@pytest.mark.xfail()
-def test_BUILD_TUPLE_CALL_FUNCTION_EX():
-    validate_uncompyle("fn(**a)")
-
-
-@pytest.mark.xfail()
-def test_BUILD_MAP_BUILD_TUPLE_BUILD_TUPLE_UNPACK_WITH_CALL_CALL_FUNCTION_EX():
-    validate_uncompyle("fn(b,b,b=0,*a)")
-
-
-@pytest.mark.xfail()
-def test_BUILD_TUPLE_BUILD_TUPLE_UNPACK_WITH_CALL_CALL_FUNCTION_EX():
-    validate_uncompyle("fn(*c,v)")
-
-
-@pytest.mark.xfail()
-def test_BUILD_CONST_KEY_MAP_CALL_FUNCTION_EX():
-    validate_uncompyle("fn(i=0,y=0,*p)")
-
-
-@pytest.mark.skip(reason='skipping property based test until all individual tests are passing')
-@given(function_calls())
-def test_function_call(function_call):
-    validate_uncompyle(function_call)

pytest/test_grammar.py

@@ -20,13 +20,17 @@ def test_grammar():
     # We have custom rules that create the below
     expect_lhs = set(['pos_arg', 'get_iter', 'attribute'])
 
-    unused_rhs = set(['list', 'mkfunc', 'dict',
+    unused_rhs = set(['list', 'mkfunc',
                       'mklambda',
                       'unpack',])
     expect_right_recursive = set([('designList',
                                    ('store', 'DUP_TOP', 'designList'))])
-    expect_lhs.add('kvlist')
-    expect_lhs.add('kv3')
+
+    if PYTHON_VERSION > 2.6:
+        expect_lhs.add('kvlist')
+        expect_lhs.add('kv3')
+        unused_rhs.add('dict')
+
     if PYTHON3:
         expect_lhs.add('load_genexpr')
 
@@ -46,7 +50,7 @@ def test_grammar():
                 unused_rhs.add('call')
                 if PYTHON_VERSION == 3.5:
                     expect_right_recursive.add((('l_stmts',
-                                                 ('lastl_stmt', 'COME_FROM', 'l_stmts'))))
+                                                 ('lastl_stmt', 'come_froms', 'l_stmts'))))
                     pass
                 pass
             else:
@@ -85,6 +89,8 @@ def test_grammar():
             """.split())
     if 2.6 <= PYTHON_VERSION <= 2.7:
         opcode_set = set(s.opc.opname).union(ignore_set)
+        if PYTHON_VERSION == 2.6:
+            opcode_set.add("THEN")
         check_tokens(tokens, opcode_set)
     elif PYTHON_VERSION == 3.4:
         ignore_set.add('LOAD_CLASSNAME')

pytest/test_pysource.py

@@ -8,12 +8,8 @@ from uncompyle6.semantics.consts import (
 
 if PYTHON3:
     from io import StringIO
-    def iteritems(d):
-        return d.items()
 else:
     from StringIO import StringIO
-    def iteritems(d):
-        return d.iteritems()
 
 from uncompyle6.semantics.pysource import SourceWalker as SourceWalker
 
@@ -30,7 +26,7 @@ def test_template_engine():
     # FIXME: and so on...
 
 from uncompyle6.semantics.consts import (
-    TABLE_DIRECT, TABLE_R,
+    TABLE_R, TABLE_DIRECT,
     )
 
 from uncompyle6.semantics.fragments import (
@@ -44,7 +40,7 @@ def test_tables():
             (TABLE_DIRECT, 'TABLE_DIRECT', False),
             (TABLE_R, 'TABLE_R', False),
             (TABLE_DIRECT_FRAGMENT, 'TABLE_DIRECT_FRAGMENT', True)):
-        for k, entry in iteritems(t):
+        for k, entry in t.iteritems():
             if k in skip_for_now:
                 continue
             fmt = entry[0]

pytest/test_single_compile.py

@@ -1,19 +1,19 @@
-import pytest
-from uncompyle6 import PYTHON_VERSION, PYTHON3, deparse_code
+from uncompyle6 import PYTHON_VERSION, deparse_code
 
-def test_single_mode():
-    single_expressions = (
-        'i = 1',
-        'i and (j or k)',
-        'i += 1',
-        'i = j % 4',
-        'i = {}',
-        'i = []',
-        'for i in range(10):\n    i\n',
-        'for i in range(10):\n    for j in range(10):\n        i + j\n',
-        'try:\n    i\nexcept Exception:\n    j\nelse:\n    k\n'
-    )
+if PYTHON_VERSION >= 2.6:
+    def test_single_mode():
+        single_expressions = (
+            'i = 1',
+            'i and (j or k)',
+            'i += 1',
+            'i = j % 4',
+            'i = {}',
+            'i = []',
+            'for i in range(10):\n    i\n',
+            'for i in range(10):\n    for j in range(10):\n        i + j\n',
+            'try:\n    i\nexcept Exception:\n    j\nelse:\n    k\n'
+        )
 
-    for expr in single_expressions:
-        code = compile(expr + '\n', '<string>', 'single')
-        assert deparse_code(PYTHON_VERSION, code, compile_mode='single').text == expr + '\n'
+        for expr in single_expressions:
+            code = compile(expr + '\n', '<string>', 'single')
+            assert deparse_code(PYTHON_VERSION, code, compile_mode='single').text == expr + '\n'

pytest/testdata/if-2.7.right

@@ -9,4 +9,4 @@
           12  JUMP_FORWARD          0  'to 15'
         15_0  COME_FROM            12  '12'
           15  LOAD_CONST            0  None
-          18  RETURN_VALUE     
+          18  RETURN_VALUE

pytest/testdata/ifelse-2.7.right

@@ -12,4 +12,4 @@
           18  STORE_NAME            2  'd'
         21_0  COME_FROM            12  '12'
           21  LOAD_CONST            2  None
-          24  RETURN_VALUE     
+          24  RETURN_VALUE

pytest/validate.py

@@ -1,24 +1,25 @@
-# future
-from __future__ import print_function
 # std
 import os
 import difflib
 import subprocess
 import tempfile
-import functools
-# compatability
-import six
+
+from StringIO import StringIO
 # uncompyle6 / xdis
 from uncompyle6 import PYTHON_VERSION, IS_PYPY, deparse_code
 # TODO : I think we can get xdis to support the dis api (python 3 version) by doing something like this there
 from xdis.bytecode import Bytecode
 from xdis.main import get_opcode
 opc = get_opcode(PYTHON_VERSION, IS_PYPY)
-Bytecode = functools.partial(Bytecode, opc=opc)
 
+try:
+    import functools
+    Bytecode = functools.partial(Bytecode, opc=opc)
+    def _dis_to_text(co):
+        return Bytecode(co).dis()
+except:
+    pass
 
-def _dis_to_text(co):
-    return Bytecode(co).dis()
 
 
 def print_diff(original, uncompyled):
@@ -42,8 +43,11 @@ def print_diff(original, uncompyled):
         print('\nTo display diff highlighting run:\n    pip install BeautifulSoup4')
         diff = difflib.HtmlDiff().make_table(*args)
 
-    with tempfile.NamedTemporaryFile(delete=False) as f:
+    f = tempfile.NamedTemporaryFile(delete=False)
+    try:
         f.write(str(diff).encode('utf-8'))
+    finally:
+        f.close()
 
     try:
         print()
@@ -60,8 +64,7 @@ def print_diff(original, uncompyled):
         print('\nFor side by side diff install elinks')
         diff = difflib.Differ().compare(original_lines, uncompyled_lines)
         print('\n'.join(diff))
-    finally:
-        os.unlink(f.name)
+    os.unlink(f.name)
 
 
 def are_instructions_equal(i1, i2):
@@ -123,8 +126,9 @@ def validate_uncompyle(text, mode='exec'):
     original_text = text
 
     deparsed = deparse_code(PYTHON_VERSION, original_code,
+
                             compile_mode=mode,
-                            out=six.StringIO(),
+                            out=StringIO(),
                             is_pypy=IS_PYPY)
     uncompyled_text = deparsed.text
     uncompyled_code = compile(uncompyled_text, '<string>', 'exec')

requirements-dev.txt

@@ -1,3 +1,3 @@
-pytest>=3.0.0
+pytest>=3.0.0,<=3.0.1
 flake8
-hypothesis<=3.8.3
+hypothesis<=3.0.0

setup.py

@@ -1,16 +1,16 @@
 #!/usr/bin/env python
-import sys
-
 """Setup script for the 'uncompyle6' distribution."""
 
+import sys
+
 SYS_VERSION = sys.version_info[0:2]
-if not ((2, 6) <= SYS_VERSION  <= (3, 7)) or ((3, 0) <= SYS_VERSION <= (3, 1)):
-    mess = "Python Release 2.6 .. 3.7 excluding 3.0 and 3.1 are supported in this code branch."
-    if ((2, 4) <= SYS_VERSION <= (2, 7)):
-        mess += ("\nFor your Python, version %s, use the python-2.4 code/branch." %
+if not ((2, 4) <= SYS_VERSION  <= (2, 7)):
+    mess = "Python Release 2.4 .. 2.7 are supported in this code branch."
+    if ((3, 2) <= SYS_VERSION  <= (3, 7)):
+        mess += ("\nFor your Python, version %s, use the master code/branch." %
                  sys.version[0:3])
-    elif SYS_VERSION < (2, 4) or ((3, 0) <= SYS_VERSION <= (3, 1)):
-        mess += ("\nThis package is not supported for Python version %s."
+    else:
+        mess += ("\nThis package is not supported before Python 2.4. Your Python version is %s."
                  % sys.version[0:3])
     print(mess)
     raise Exception(mess)

test-unit/test_grammar.py

@@ -0,0 +1,55 @@
+import re
+import unittest
+from uncompyle6 import PYTHON_VERSION, IS_PYPY # , PYTHON_VERSION
+from uncompyle6.parser import get_python_parser, python_parser
+
+class TestGrammar(unittest.TestCase):
+    def test_grammar(self):
+
+        def check_tokens(tokens, opcode_set):
+            remain_tokens = set(tokens) - opcode_set
+            remain_tokens = set([re.sub('_\d+$','', t) for t in remain_tokens])
+            remain_tokens = set([re.sub('_CONT$','', t) for t in remain_tokens])
+            remain_tokens = set(remain_tokens) - opcode_set
+            self.assertEqual(remain_tokens, set([]),
+                    "Remaining tokens %s\n====\n%s" % (remain_tokens, p.dump_grammar()))
+
+        p = get_python_parser(PYTHON_VERSION, is_pypy=IS_PYPY)
+        (lhs, rhs, tokens,
+         right_recursive, dup_rhs) = p.check_sets()
+        expect_lhs = set(['expr1024', 'pos_arg'])
+        unused_rhs = set(['list', 'call', 'mkfunc',
+                          'mklambda',
+                          'unpack',])
+
+        expect_right_recursive = frozenset([('designList',
+                                             ('store', 'DUP_TOP', 'designList'))])
+        expect_lhs.add('kwarg')
+
+        self.assertEqual(expect_lhs, set(lhs))
+        self.assertEqual(unused_rhs, set(rhs))
+        self.assertEqual(expect_right_recursive, right_recursive)
+
+        expect_dup_rhs = frozenset([('COME_FROM',), ('CONTINUE',), ('JUMP_ABSOLUTE',),
+                                    ('LOAD_CONST',),
+                                    ('JUMP_BACK',), ('JUMP_FORWARD',)])
+
+        reduced_dup_rhs = {}
+        for k in dup_rhs:
+            if k not in expect_dup_rhs:
+                reduced_dup_rhs[k] = dup_rhs[k]
+                pass
+            pass
+        for k in reduced_dup_rhs:
+            print(k, reduced_dup_rhs[k])
+        # assert not reduced_dup_rhs, reduced_dup_rhs
+
+    def test_dup_rule(self):
+        import inspect
+        python_parser(PYTHON_VERSION, inspect.currentframe().f_code,
+                      is_pypy=IS_PYPY,
+                      parser_debug={
+                          'dups': True, 'transition': False, 'reduce': False,
+                          'rules': False, 'errorstack': None, 'context': True})
+if __name__ == '__main__':
+    unittest.main()

test/Makefile

@@ -28,7 +28,7 @@ check:
 	$(MAKE) check-$(PYTHON_VERSION)
 
 #: Run working tests from Python 2.6 or 2.7
-check-2.6 check-2.7: check-bytecode-2 check-bytecode-3 check-bytecode-1 check-native-short
+check-2.4 check-2.5 check-2.6 check-2.7: check-bytecode-2 check-bytecode-3 check-bytecode-1  check-native-short
 
 #: Run working tests from Python 3.0
 check-3.0: check-bytecode
@@ -37,6 +37,7 @@ check-3.0: check-bytecode
 #: Run working tests from Python 3.1
 check-3.1: check-bytecode
 	$(PYTHON) test_pythonlib.py --bytecode-3.1 --weak-verify $(COMPILE)
+	$(PYTHON) test_pythonlib.py --bytecode-3.1-run --verify-run
 
 #: Run working tests from Python 3.2
 check-3.2: check-bytecode
@@ -201,6 +202,7 @@ check-bytecode-3.0:
 #: Check deparsing Python 3.1
 check-bytecode-3.1:
 	$(PYTHON) test_pythonlib.py --bytecode-3.1 --weak-verify
+	$(PYTHON) test_pythonlib.py --bytecode-3.1-run --verify-run
 
 #: Check deparsing Python 3.2
 check-bytecode-3.2:
@@ -232,6 +234,10 @@ check-native-short:
 	$(PYTHON) test_pythonlib.py --bytecode-$(PYTHON_VERSION) --weak-verify $(COMPILE)
 	$(PYTHON) test_pythonlib.py --bytecode-$(PYTHON_VERSION)-run --verify-run $(COMPILE)
 
+#: Run longer Python 2.6's lib files known to be okay
+check-2.4-ok:
+	$(PYTHON) test_pythonlib.py --ok-2.4 --verify $(COMPILE)
+
 #: Run longer Python 2.6's lib files known to be okay
 check-2.6-ok:
 	$(PYTHON) test_pythonlib.py --ok-2.6 --weak-verify $(COMPILE)

test/dis-compare.py

@@ -1,9 +1,8 @@
 #!/usr/bin/env python
 # Mode: -*- python -*-
 #
-# Copyright (c) 2015 by Rocky Bernstein <rb@dustyfeet.com>
+# Copyright (c) 2015, 2017 by Rocky Bernstein <rb@dustyfeet.com>
 #
-from __future__ import print_function
 
 
 import dis, os.path

test/grammar-cover/grammar.sh

@@ -30,7 +30,7 @@ while [[ -n $1 ]]  ; do
         source ./admin-tools/setup-python-2.4.sh
     fi
     GRAMMAR_TXT=$tmpdir/grammar-${SHORT_VERSION}.txt
-    pyenv local ${LONG_VERSION}
+    (cd ../.. && pyenv local ${LONG_VERSION})
     cd ./test
     if [[ -r $COVER_FILE ]]; then
 	rm $COVER_FILE

test/run-and-email.sh

@@ -14,7 +14,7 @@ function displaytime {
     printf '%d seconds\n' $S
 }
 
-PYVERSION=${PYVERSION:-"3.5.5 2.7.14 3.4.8 2.6.9"}
+PYVERSION=${PYVERSION:-"3.5.5 2.7.14 3.2.6 3.3.7 3.4.8 2.6.9 3.6.4"}
 # PYVERSION=${PYVERSION:-"3.5.5"}
 
 USER=${USER:-rocky}
@@ -28,6 +28,10 @@ for VERSION in $PYVERSION ; do
 
     if [[ $VERSION == '3.5.5' ]] ; then
 	MAX_TESTS=224
+    elif [[ $VERSION == '3.2.6' ]] ; then
+	MAX_TESTS=700
+    elif [[ $VERSION == '3.6.4' ]] ; then
+	MAX_TESTS=400
     else
 	MAX_TESTS=800
     fi
@@ -42,7 +46,7 @@ for VERSION in $PYVERSION ; do
       rc=$?
 
       echo Python Version $(pyenv local) >> $LOGFILE
-      echo "" >>LOGFILE
+      echo "" >>$LOGFILE
 
       typeset -i ALL_FILES_ENDTIME=$(date +%s)
       (( time_diff =  ALL_FILES_ENDTIME - ALL_FILES_STARTTIME))

test/simple-uncompyle-code-test.py

@@ -1,7 +1,5 @@
 #!/usr/bin/env python
 
-from __future__ import print_function
-
 from uncompyle6 import uncompyle
 import sys, inspect
 

test/simple_source/bug25/02_yield_bug.py

@@ -0,0 +1,7 @@
+# From 2.4 test_array.py
+# In Python 2.4 and earlier "yield" is not valid and instead
+# we must use "yield None". Bug was not adding "None"
+
+def yield_bug():
+    yield None
+    return

test/simple_source/bug25/04_try_else_confuse.py

@@ -0,0 +1,19 @@
+# From 2.4 test_sax.py
+# Bug was distinguishing try from try/else
+
+def verify_empty_attrs():
+    gvqk = 3
+    try:
+        gvk = 1/0
+    except ZeroDivisionError:
+        gvk = 1
+
+    try:
+        gvqk = 0
+    except KeyError:
+        gvqk = 1
+
+    # If try/else was used above the return will be 4
+    return gvk + gvqk
+
+assert 1 == verify_empty_attrs()

test/simple_source/bug31/05_abc_test.py

@@ -0,0 +1,18 @@
+# Extracted from Python 3.5 test_abc.py
+# Bug is class having only a single kwarg
+# subclass.
+import abc
+import unittest
+from inspect import isabstract
+
+def test_abstractmethod_integration(self):
+    for abstractthing in [abc.abstractmethod]:
+        class C(metaclass=abc.ABCMeta):
+            @abstractthing
+            def foo(self): pass  # abstract
+            def bar(self): pass  # concrete
+        assert C.__abstractmethods__, {"foo"}
+        assert isabstract(C)
+        pass
+
+test_abstractmethod_integration(None)

test/simple_source/bug33/02_named_and_kwargs.py

@@ -1,5 +1,6 @@
 # Python 3.6 subprocess.py bug
 # Bug is getting params correct: timeout before **kwargs
+import subprocess
 def call(*popenargs, timeout=None, **kwargs):
     return
 
@@ -14,6 +15,9 @@ def subprocess_shell(self, protocol_factory, cmd, *, stdin=subprocess.PIPE,
 # From 3.4 asyncio/locks.py
 # Bug was handling" "value=1, *"
 
+class Semaphore:
+    pass
+
 class BoundedSemaphore(Semaphore):
     def __init__(self, value=1, *, loop=None):
         super().__init__(value, loop=loop)

test/simple_source/bug35/04_call_function.py

@@ -1,5 +1,6 @@
 # From sql/schema.py and 3.5 _strptime.py
-# Note that kwargs comes before "positional" args
+# Bug was code not knowing which Python versions
+# have kwargs coming before positional args in code.
 
 # RUNNABLE!
 
@@ -58,3 +59,23 @@ def __call__(self, *args, **kwds):
 # From 3.6.4 shutil
 def unpack_archive(func, filename, dict, format_info, extract_dir=None):
     func(filename, extract_dir, **dict(format_info[2]))
+
+# From 3.5.5 test_xrdrlib.py
+import xdrlib
+def assertRaisesConversion(self, *args):
+   self.assertRaises(xdrlib.ConversionError, *args)
+
+# From 3.2.6 _pyio.py
+class BlockingIOError(IOError):
+    def __init__(self, errno, strerror, characters_written=5):
+        super().__init__(errno, strerror)
+
+# From urllib/parse.py
+# Bug was using a subclass made from a call (to namedtuple)
+from collections import namedtuple
+
+class ResultMixin(object):
+    pass
+
+class SplitResult(namedtuple('SplitResult', 'scheme netloc path query fragment'), ResultMixin):
+    pass

test/simple_source/bug35/07_build_map_unpack.py

@@ -0,0 +1,10 @@
+# Adapted from Python 3.6 trace.py
+# Bug was in handling BUID_TUPLE_UNPACK created via
+# *opts.arguments
+import argparse
+parser = argparse.ArgumentParser()
+parser.add_argument('filename', nargs='?')
+parser.add_argument('arguments', nargs=argparse.REMAINDER)
+opts = parser.parse_args(["foo", "a", "b"])
+argv = opts.filename, *opts.arguments
+assert argv == ('foo', 'a', 'b')

test/stdlib/run-and-email.sh

@@ -2,7 +2,7 @@
 USER=${USER:-rocky}
 EMAIL=${EMAIL:-rb@dustyfeet.com}
 SUBJECT_PREFIX="stdlib unit testing for"
-for VERSION in 3.4.8 3.5.5 3.6.4 ; do
+for VERSION in 2.6.9 2.7.14 3.4.8 3.5.5 3.6.4 ; do
     typeset -i rc=0
     LOGFILE=/tmp/runtests-$VERSION-$$.log
     if ! pyenv local $VERSION ; then

test/stdlib/runtests.sh

@@ -35,10 +35,10 @@ case $PYVERSION in
     2.5)
 	SKIP_TESTS=(
 	    [test_contextlib.py]=1 # Syntax error - look at
-	    [test_dis.py]=1   # We change line numbers - duh!
-	    [test_grammar.py]=1  # Too many stmts. Handle large stmts
-	    [test_grp.py]=1      # Long test - might work Control flow?
-	    [test_pdb.py]=1      # Line-number specific
+	    [test_dis.py]=1        # We change line numbers - duh!
+	    [test_grammar.py]=1    # Too many stmts. Handle large stmts
+	    [test_grp.py]=1        # Long test - might work Control flow?
+	    [test_pdb.py]=1        # Line-number specific
 	    [test_pwd.py]=1 # Long test - might work? Control flow?
 	    [test_queue.py]=1 # Control flow?
 	    [test_re.py]=1 # Probably Control flow?
@@ -101,8 +101,10 @@ case $PYVERSION in
     3.5)
 	SKIP_TESTS=(
 	    [test_decorators.py]=1  # Control flow wrt "if elif"
+	    [test_quopri.py]=1      # Fails in crontab environment?
 	)
 	;;
+
     3.6)
 	SKIP_TESTS=(
 	    [test_contains.py]=1    # Code "while False: yield None" is optimized away in compilation

test/test_pyenvlib.py

@@ -20,8 +20,7 @@ Step 2: Run the test:
 	  test_pyenvlib --mylib --verify # decompile verify 'mylib'
 """
 
-from __future__ import print_function
-
+from uncompyle6 import main, PYTHON3
 import os, time, re, shutil, sys
 from fnmatch import fnmatch
 

test/test_pythonlib.py

@@ -27,8 +27,6 @@ Step 2: Run the test:
   test_pythonlib.py --mylib --verify # decompile verify 'mylib'
 """
 
-from __future__ import print_function
-
 import getopt, os, py_compile, sys, shutil, tempfile, time
 
 from uncompyle6 import PYTHON_VERSION
@@ -130,8 +128,10 @@ def do_tests(src_dir, obj_patterns, target_dir, opts):
     if opts['do_compile']:
         compiled_version = opts['compiled_version']
         if compiled_version and PYTHON_VERSION != compiled_version:
-            print("Not compiling: desired Python version is %s but we are running %s" %
-                  (compiled_version, PYTHON_VERSION), file=sys.stderr)
+            sys.stderr.write("Not compiling: "
+                             "desired Python version is %s "
+                             "but we are running %s" %
+                             (compiled_version, PYTHON_VERSION))
         else:
             for root, dirs, basenames in os.walk(src_dir):
                 file_matches(files, root, basenames, PY)
@@ -149,8 +149,8 @@ def do_tests(src_dir, obj_patterns, target_dir, opts):
         file_matches(files, dirname, basenames, obj_patterns)
 
     if not files:
-        print("Didn't come up with any files to test! Try with --compile?",
-              file=sys.stderr)
+        sys.stderr.write("Didn't come up with any files to test! "
+                         "Try with --compile?")
         exit(1)
 
     os.chdir(cwd)
@@ -164,9 +164,9 @@ def do_tests(src_dir, obj_patterns, target_dir, opts):
         except ValueError:
             pass
 
-    print(time.ctime())
-    print('Source directory: ', src_dir)
-    print('Output directory: ', target_dir)
+    print time.ctime()
+    print 'Source directory: ', src_dir
+    print 'Output directory: ', target_dir
     try:
         _, _, failed_files, failed_verify = \
           main(src_dir, target_dir, files, [],
@@ -242,14 +242,13 @@ if __name__ == '__main__':
         if os.path.isdir(src_dir):
             checked_dirs.append([src_dir, pattern, target_dir])
         else:
-            print("Can't find directory %s. Skipping" % src_dir,
-                  file=sys.stderr)
+            sys.stderr.write("Can't find directory %s. Skipping" % src_dir)
             continue
         last_compile_version = compiled_version
         pass
 
     if not checked_dirs:
-        print("No directories found to check", file=sys.stderr)
+        sys.stderr.write("No directories found to check\n")
         sys.exit(1)
 
     test_opts['compiled_version'] = last_compile_version

uncompyle6/bin/pydisassemble.py

@@ -3,7 +3,6 @@
 #
 # Copyright (c) 2015-2016, 2018 by Rocky Bernstein <rb@dustyfeet.com>
 #
-from __future__ import print_function
 import sys, os, getopt
 
 from uncompyle6.disas import disassemble_file
@@ -13,8 +12,8 @@ program, ext = os.path.splitext(os.path.basename(__file__))
 
 __doc__ = """
 Usage:
-  {0} [OPTIONS]... FILE
-  {0} [--help | -h | -V | --version]
+  %s [OPTIONS]... FILE
+  %s [--help | -h | -V | --version]
 
 Disassemble FILE with the instruction mangling that is done to
 assist uncompyle6 in parsing the instruction stream. For example
@@ -23,9 +22,9 @@ BUILD_LIST have arguement counts appended to the instruction name, and
 COME_FROM instructions are inserted into the instruction stream.
 
 Examples:
-  {0} foo.pyc
-  {0} foo.py    # same thing as above but find the file
-  {0} foo.pyc bar.pyc  # disassemble foo.pyc and bar.pyc
+  %s foo.pyc
+  %s foo.py    # same thing as above but find the file
+  %s foo.pyc bar.pyc  # disassemble foo.pyc and bar.pyc
 
 See also `pydisasm' from the `xdis' package.
 
@@ -33,7 +32,7 @@ Options:
   -V | --version     show version and stop
   -h | --help        show this message
 
-""".format(program)
+""" % ((program,) * 5)
 
 PATTERNS = ('*.pyc', '*.pyo')
 
@@ -42,15 +41,15 @@ def main():
 Type -h for for full help.""" % program
 
     if len(sys.argv) == 1:
-        print("No file(s) given", file=sys.stderr)
-        print(Usage_short, file=sys.stderr)
+        sys.stderr.write("No file(s) given\n")
+        sys.stderr.write(Usage_short)
         sys.exit(1)
 
     try:
         opts, files = getopt.getopt(sys.argv[1:], 'hVU',
                                     ['help', 'version', 'uncompyle6'])
-    except getopt.GetoptError as e:
-        print('%s: %s' % (os.path.basename(sys.argv[0]), e),  file=sys.stderr)
+    except getopt.GetoptError(e):
+        sys.stderr.write('%s: %s' % (os.path.basename(sys.argv[0]), e))
         sys.exit(-1)
 
     for opt, val in opts:
@@ -62,15 +61,14 @@ Type -h for for full help.""" % program
             sys.exit(0)
         else:
             print(opt)
-            print(Usage_short, file=sys.stderr)
+            sys.stderr.write(Usage_short)
             sys.exit(1)
 
     for file in files:
         if os.path.exists(files[0]):
             disassemble_file(file, sys.stdout)
         else:
-            print("Can't read %s - skipping" % files[0],
-                  file=sys.stderr)
+            sys.stderr.write("Can't read %s - skipping\n" % files[0])
             pass
         pass
     return

uncompyle6/bin/uncompile.py

@@ -4,7 +4,6 @@
 # Copyright (c) 2015-2017 by Rocky Bernstein
 # Copyright (c) 2000-2002 by hartmut Goebel <h.goebel@crazy-compilers.com>
 #
-from __future__ import print_function
 import sys, os, getopt, time
 
 program = 'uncompyle6'
@@ -68,11 +67,11 @@ def usage():
 
 
 def main_bin():
-    if not (sys.version_info[0:2] in ((2, 6), (2, 7),
-                                      (3, 1), (3, 2), (3, 3),
+    if not (sys.version_info[0:2] in ((2, 4), (2, 5), (2, 6), (2, 7),
+                                      (3, 2), (3, 3),
                                       (3, 4), (3, 5), (3, 6))):
-        print('Error: %s requires Python 2.6-2.7, or 3.1-3.6' % program,
-              file=sys.stderr)
+        sys.stderr.write('Error: %s requires Python 2.4 2.5 2.6, 2.7, '
+                         '3.2, 3.3, 3.4, 3.5, or 3.6' % program)
         sys.exit(-1)
 
     do_verify = recurse_dirs = False
@@ -88,8 +87,8 @@ def main_bin():
                                     'help asm grammar linemaps recurse timestamp tree '
                                     'fragments verify verify-run version weak-verify '
                                     'showgrammar'.split(' '))
-    except getopt.GetoptError as e:
-        print('%s: %s' % (os.path.basename(sys.argv[0]), e),  file=sys.stderr)
+    except getopt.GetoptError(e):
+        sys.stderr.write('%s: %s\n' % (os.path.basename(sys.argv[0]), e))
         sys.exit(-1)
 
     options = {}
@@ -129,7 +128,7 @@ def main_bin():
         elif opt in ('--recurse', '-r'):
             recurse_dirs = True
         else:
-            print(opt, file=sys.stderr)
+            sys.stderr.write(opt)
             usage()
 
     # expand directory if specified
@@ -154,7 +153,7 @@ def main_bin():
         files = [f[sb_len:] for f in files]
 
     if not files:
-        print("No files given", file=sys.stderr)
+        sys.stderr.write("No files given\n")
         usage()
 
     if outfile == '-':

uncompyle6/disas.py

@@ -29,8 +29,6 @@ Second, we need structured instruction information for the
 want to run on earlier Python versions.
 """
 
-from __future__ import print_function
-
 import sys
 from collections import deque
 
@@ -49,10 +47,9 @@ def disco(version, co, out=None, is_pypy=False):
 
     # store final output stream for case of error
     real_out = out or sys.stdout
-    print('# Python %s' % version, file=real_out)
+    real_out.write('# Python %s\n' % version)
     if co.co_filename:
-        print('# Embedded file name: %s' % co.co_filename,
-              file=real_out)
+        real_out.write('# Embedded file name: %s\n' % co.co_filename)
 
     scanner = get_scanner(version, is_pypy=is_pypy)
 
@@ -64,16 +61,15 @@ def disco_loop(disasm, queue, real_out):
     while len(queue) > 0:
         co = queue.popleft()
         if co.co_name != '<module>':
-            print('\n# %s line %d of %s' %
-                      (co.co_name, co.co_firstlineno, co.co_filename),
-                      file=real_out)
+            real_out.write('\n# %s line %d of %s\n' %
+                      (co.co_name, co.co_firstlineno, co.co_filename))
         tokens, customize = disasm(co)
         for t in tokens:
             if iscode(t.pattr):
                 queue.append(t.pattr)
             elif iscode(t.attr):
                 queue.append(t.attr)
-            print(t, file=real_out)
+            real_out.write(t)
             pass
         pass
 

uncompyle6/linenumbers.py

@@ -25,7 +25,7 @@ def line_number_mapping(pyc_filename, src_filename):
      source_size) = load_module(pyc_filename)
     try:
         code2 = load_file(src_filename)
-    except SyntaxError as e:
+    except SyntaxError, e:
         return str(e)
 
     queue = deque([code1, code2])

uncompyle6/main.py

@@ -12,10 +12,10 @@
 #
 #  You should have received a copy of the GNU General Public License
 #  along with this program.  If not, see <http://www.gnu.org/licenses/>.
-from __future__ import print_function
-import datetime, os, subprocess, sys, tempfile
 
-from uncompyle6 import verify, IS_PYPY, PYTHON_VERSION
+import datetime, os, subprocess, sys
+
+from uncompyle6 import verify, IS_PYPY
 from xdis.code import iscode
 from uncompyle6.disas import check_object_path
 from uncompyle6.semantics import pysource
@@ -38,11 +38,7 @@ def _get_outstream(outfile):
         os.makedirs(dir)
     except OSError:
         pass
-    if PYTHON_VERSION < 3.0:
-        mode = 'wb'
-    else:
-        mode = 'w'
-    return open(outfile, mode)
+    return open(outfile, 'wb')
 
 def decompile(
         bytecode_version, co, out=None, showasm=None, showast=False,
@@ -63,20 +59,35 @@ def decompile(
 
     assert iscode(co)
 
-    co_pypy_str = 'PyPy ' if is_pypy else ''
-    run_pypy_str = 'PyPy ' if IS_PYPY else ''
+    if is_pypy:
+        co_pypy_str = 'PyPy '
+    else:
+        co_pypy_str = ''
+
+    if IS_PYPY:
+        run_pypy_str = 'PyPy '
+    else:
+        run_pypy_str = ''
+
+    if magic_int:
+        m = str(magic_int)
+    else:
+        m = ""
+
     sys_version_lines = sys.version.split('\n')
     write('# uncompyle6 version %s\n'
           '# %sPython bytecode %s%s\n# Decompiled from: %sPython %s' %
           (VERSION, co_pypy_str, bytecode_version,
-               " (%s)" % str(magic_int) if magic_int else "",
-          run_pypy_str, '\n# '.join(sys_version_lines)))
+               " (%s)" % m, run_pypy_str,
+           '\n# '.join(sys_version_lines)))
     if co.co_filename:
         write('# Embedded file name: %s' % co.co_filename,)
     if timestamp:
-        write('# Compiled at: %s' % datetime.datetime.fromtimestamp(timestamp))
+        write('# Compiled at: %s' %
+              datetime.datetime.fromtimestamp(timestamp))
     if source_size:
-        write('# Size of source mod 2**32: %d bytes' % source_size)
+        real_out.write('# Size of source mod 2**32: %d bytes\n' %
+                       source_size)
 
     debug_opts = {
         'asm': showasm,
@@ -109,7 +120,7 @@ def decompile(
                                   is_pypy=is_pypy)
             pass
         return deparsed
-    except pysource.SourceWalkerError as e:
+    except pysource.SourceWalkerError, e:
         # deparsing failed
         raise pysource.SourceWalkerError(str(e))
 
@@ -189,17 +200,10 @@ def main(in_base, out_base, files, codes, outfile=None,
                     prefix = prefix[:-len('.py')]
 
                 # Unbuffer output if possible
-                buffering = -1 if sys.stdout.isatty() else 0
-                if PYTHON_VERSION >= 3.5:
-                    t = tempfile.NamedTemporaryFile(mode='w+b',
-                                                    buffering=buffering,
-                                                    suffix='.py',
-                                                    prefix=prefix)
+                if sys.stdout.isatty():
+                    buffering = -1
                 else:
-                    t = tempfile.NamedTemporaryFile(mode='w+b',
-                                                    suffix='.py',
-                                                    prefix=prefix)
-                current_outfile = t.name
+                    buffering = 0
                 sys.stdout = os.fdopen(sys.stdout.fileno(), 'w', buffering)
                 tee = subprocess.Popen(["tee", current_outfile],
                                        stdin=subprocess.PIPE)
@@ -238,9 +242,9 @@ def main(in_base, out_base, files, codes, outfile=None,
                     pass
                 pass
             tot_files += 1
-        except (ValueError, SyntaxError, ParserError, pysource.SourceWalkerError) as e:
+        except (ValueError, SyntaxError, ParserError, pysource.SourceWalkerError):
             sys.stdout.write("\n")
-            sys.stderr.write("\n# file %s\n# %s\n" % (infile, e))
+            sys.stderr.write("# file %s\n" % (infile))
             failed_files += 1
             tot_files += 1
         except KeyboardInterrupt:
@@ -278,13 +282,16 @@ def main(in_base, out_base, files, codes, outfile=None,
                         else:
                             okay_files += 1
                             pass
-                    except verify.VerifyCmpError as e:
+                    except verify.VerifyCmpError, e:
                         print(e)
                         verify_failed_files += 1
                         os.rename(current_outfile, current_outfile + '_unverified')
                         sys.stderr.write("### Error Verifying %s\n" % filename)
                         sys.stderr.write(str(e) + "\n")
                         if not outfile:
+                            sys.stderr.write("### Error Verifiying %s" %
+                                            filename)
+                            sys.stderr.write(e)
                             if raise_on_error:
                                 raise
                             pass
@@ -294,14 +301,15 @@ def main(in_base, out_base, files, codes, outfile=None,
                     okay_files += 1
                 pass
             elif do_verify:
-                sys.stderr.write("\n### uncompile successful, but no file to compare against\n")
+                sys.stderr.write("\n### uncompile successful, "
+                                 "but no file to compare against")
                 pass
             else:
                 okay_files += 1
                 if not current_outfile:
                     mess = '\n# okay decompiling'
                     # mem_usage = __memUsage()
-                    print(mess, infile)
+                    print mess, infile
         if current_outfile:
             sys.stdout.write("%s\r" %
                              status_msg(do_verify, tot_files, okay_files, failed_files,

uncompyle6/parser.py

@@ -19,8 +19,6 @@
 Common uncompyle6 parser routines.
 """
 
-from __future__ import print_function
-
 import sys
 
 from xdis.code import iscode
@@ -181,7 +179,7 @@ class PythonParser(GenericASTBuilder):
                     indent = '   '
                 else:
                     indent = '-> '
-                print("%s%s" % (indent, instructions[i]))
+                print "%s%s" % (indent, instructions[i])
             raise ParserError(err_token, err_token.offset)
         else:
             raise ParserError(None, -1)
@@ -775,4 +773,4 @@ if __name__ == '__main__':
         ast = python_parser(PYTHON_VERSION, co, showasm=True, is_pypy=IS_PYPY)
         print(ast)
         return
-    parse_test(parse_test.__code__)
+    # parse_test(parse_test.__code__)

uncompyle6/parsers/parse2.py

@@ -25,8 +25,6 @@ If we succeed in creating a parse tree, then we have a Python program
 that a later phase can turn into a sequence of ASCII text.
 """
 
-from __future__ import print_function
-
 from uncompyle6.parser import PythonParser, PythonParserSingle, nop_func
 from uncompyle6.parsers.astnode import AST
 from spark_parser import DEFAULT_DEBUG as PARSER_DEFAULT_DEBUG

uncompyle6/parsers/parse24.py

@@ -82,15 +82,6 @@ class Python24Parser(Python25Parser):
             l = len(tokens)
             if 0 <= l < len(tokens):
                 return not int(tokens[first].pattr) == tokens[last].offset
-        elif lhs == 'try_except':
-            if last == len(tokens):
-                last -= 1
-            if tokens[last] != 'COME_FROM' and tokens[last-1] == 'COME_FROM':
-                last -= 1
-            return (tokens[last] == 'COME_FROM'
-                    and tokens[last-1] == 'END_FINALLY'
-                    and tokens[last-2] == 'POP_TOP'
-                    and tokens[last-3].kind != 'JUMP_FORWARD')
 
         return False
 

uncompyle6/parsers/parse25.py

@@ -19,6 +19,9 @@ class Python25Parser(Python26Parser):
 
         return_if_stmt ::= ret_expr  RETURN_END_IF JUMP_BACK
 
+        # We have no jumps to jumps, so no "come_froms" but a single "COME_FROM"
+        ifelsestmt ::= testexpr c_stmts_opt jf_cf_pop else_suite COME_FROM
+
         # Python 2.6 uses ROT_TWO instead of the STORE_xxx
         # withas is allowed as a "from future" in 2.5
         # 2.6 and 2.7 do something slightly different
@@ -36,6 +39,8 @@ class Python25Parser(Python26Parser):
         # loop. FIXME: should "come_froms" below be a single COME_FROM?
         tryelsestmt    ::= SETUP_EXCEPT suite_stmts_opt POP_BLOCK
                            except_handler else_suite come_froms
+        tryelsestmtl   ::= SETUP_EXCEPT suite_stmts_opt POP_BLOCK
+                            except_handler else_suitel
 
         # Python 2.6 omits the LOAD_FAST DELETE_FAST below
         # withas is allowed as a "from future" in 2.5
@@ -54,6 +59,9 @@ class Python25Parser(Python26Parser):
     def customize_grammar_rules(self, tokens, customize):
         # Remove grammar rules inherited from Python 2.6 or Python 2
         self.remove_rules("""
+        # No jump to jumps in 2.4 so we have a single "COME_FROM", not "come_froms"
+        ifelsestmt    ::= testexpr c_stmts_opt jf_cf_pop else_suite come_froms
+
         setupwith  ::= DUP_TOP LOAD_ATTR ROT_TWO LOAD_ATTR CALL_FUNCTION_0 POP_TOP
         withstmt   ::= expr setupwith SETUP_FINALLY suite_stmts_opt
                        POP_BLOCK LOAD_CONST COME_FROM WITH_CLEANUP END_FINALLY

uncompyle6/parsers/parse27.py

@@ -226,4 +226,5 @@ if __name__ == '__main__':
                              for t in remain_tokens])
         remain_tokens = set(remain_tokens) - opcode_set
         print(remain_tokens)
-        # p.dump_grammar()
+        p.check_grammar()
+        p.dump_grammar()

uncompyle6/parsers/parse3.py

@@ -30,6 +30,7 @@ from uncompyle6.parser import PythonParser, PythonParserSingle, nop_func
 from uncompyle6.parsers.astnode import AST
 from spark_parser import DEFAULT_DEBUG as PARSER_DEFAULT_DEBUG
 from xdis import PYTHON3
+from itertools import islice,chain,repeat
 
 class Python3Parser(PythonParser):
 
@@ -116,9 +117,14 @@ class Python3Parser(PythonParser):
 
         classdef ::= build_class store
 
+        # FIXME: we need to add these because don't detect this properly
+        # in custom rules. Specifically if one of the exprs is CALL_FUNCTION
+        # then we'll mistake that for the final CALL_FUNCTION.
+        # We can fix by triggering on the CALL_FUNCTION op
         # Python3 introduced LOAD_BUILD_CLASS
         # Other definitions are in a custom rule
         build_class ::= LOAD_BUILD_CLASS mkfunc expr call CALL_FUNCTION_3
+        build_class ::= LOAD_BUILD_CLASS mkfunc expr call expr CALL_FUNCTION_4
 
         stmt ::= classdefdeco
         classdefdeco ::= classdefdeco1 store
@@ -423,7 +429,7 @@ class Python3Parser(PythonParser):
                         LOAD_CONST CALL_FUNCTION_n
         build_class ::= LOAD_BUILD_CLASS mkfunc
                         expr
-                        call_function
+                        call
                         CALL_FUNCTION_3
          '''
         # FIXME: I bet this can be simplified
@@ -626,7 +632,7 @@ class Python3Parser(PythonParser):
                 self.addRule(rule, nop_func)
                 rule = 'expr ::= build_list_unpack'
                 self.addRule(rule, nop_func)
-            elif opname_base == 'BUILD_MAP':
+            elif opname_base in ('BUILD_MAP', 'BUILD_MAP_UNPACK'):
                 kvlist_n = "kvlist_%s" % token.attr
                 if opname == 'BUILD_MAP_n':
                     # PyPy sometimes has no count. Sigh.
@@ -645,23 +651,21 @@ class Python3Parser(PythonParser):
                         # FIXME: Use the attr
                         # so this doesn't run into exponential parsing time.
                         if opname.startswith('BUILD_MAP_UNPACK'):
-                            # FIXME: start here
-                            # rule = "%s ::= %s %s" % (kvlist_n, 'expr ' * (token.attr*2), opname)
-                            rule = kvlist_n + ' ::= ' + 'expr ' * (token.attr*2)
                             self.add_unique_rule(rule, opname, token.attr, customize)
                             rule = 'dict_entry ::= ' + 'expr ' * (token.attr*2)
                             self.add_unique_rule(rule, opname, token.attr, customize)
-                            rule = 'dict ::= %s' % ('dict_entry ' * token.attr)
-                            self.addRule(rule, nop_func)
 
+                            # FIXME: start here. The LHS should be unmap_dict, not dict.
                             # FIXME: really we need a combination of dict_entry-like things.
                             # It just so happens the most common case is not to mix
                             # dictionary comphensions with dictionary, elements
                             if self.seen_LOAD_DICTCOMP:
                                 rule = 'dict ::= %s%s' % ('dict_comp ' * token.attr, opname)
                                 self.addRule(rule, nop_func)
-
-                            rule = 'unmap_dict ::= %s%s' % (('dict ' * token.attr), opname)
+                            rule = """
+                             expr       ::= unmap_dict
+                             unmap_dict ::= %s%s
+                             """ % ('expr ' * token.attr, opname)
                         else:
                             rule = "%s ::= %s %s" % (kvlist_n, 'expr ' * (token.attr*2), opname)
                             self.add_unique_rule(rule, opname, token.attr, customize)
@@ -679,7 +683,9 @@ class Python3Parser(PythonParser):
                 v = token.attr
                 rule = ('starred ::= %s %s' % ('expr ' * v, opname))
                 self.addRule(rule, nop_func)
-            elif opname_base in ('BUILD_LIST', 'BUILD_SET', 'BUILD_TUPLE'):
+
+            elif opname_base in ('BUILD_LIST', 'BUILD_SET', 'BUILD_TUPLE',
+                                 'BUILD_TUPLE_UNPACK'):
                 v = token.attr
 
                 is_LOAD_CLOSURE = False
@@ -1159,7 +1165,7 @@ class Python3Parser(PythonParser):
 
             if tokens[cfl-1] != 'JUMP_BACK':
                 cfl_offset = tokens[cfl-1].offset
-                insn = next(i for i in self.insts if cfl_offset == i.offset)
+                insn = chain((i for i in self.insts if cfl_offset == i.offset), repeat(None)).next()
                 if insn and  insn.is_jump_target:
                     return True
 

uncompyle6/parsers/parse30.py

@@ -2,7 +2,6 @@
 """
 spark grammar differences over Python 3.1 for Python 3.0.
 """
-from __future__ import print_function
 
 from uncompyle6.parser import PythonParserSingle
 from uncompyle6.parsers.parse31 import Python31Parser

uncompyle6/parsers/parse31.py

@@ -2,7 +2,6 @@
 """
 spark grammar differences over Python 3.2 for Python 3.1.
 """
-from __future__ import print_function
 
 from uncompyle6.parser import PythonParserSingle
 from uncompyle6.parsers.parse32 import Python32Parser

uncompyle6/parsers/parse32.py

@@ -2,8 +2,6 @@
 """
 spark grammar differences over Python 3 for Python 3.2.
 """
-from __future__ import print_function
-
 from uncompyle6.parser import PythonParserSingle
 from uncompyle6.parsers.parse3 import Python3Parser
 

uncompyle6/parsers/parse33.py

@@ -2,7 +2,6 @@
 """
 spark grammar differences over Python 3.2 for Python 3.3.
 """
-from __future__ import print_function
 
 from uncompyle6.parser import PythonParserSingle
 from uncompyle6.parsers.parse32 import Python32Parser