Merge branch 'master' into python-2.4

bugs still remain, just reduced.

.gitignore

@@ -1,11 +1,15 @@
-*.pyo
 *.pyc
+*.pyo
 *_dis
 *~
 /.cache
 /.eggs
+/.hypothesis
+/.idea
+/.pytest_cache
 /.python-version
 /.tox
+/.venv*
 /README
 /__pkginfo__.pyc
 /dist
@@ -14,9 +18,6 @@
 /tmp
 /uncompyle6.egg-info
 /unpyc
+ChangeLog
 __pycache__
 build
-/.venv*
-/.idea
-/.hypothesis
-ChangeLog

.travis.yml

@@ -3,13 +3,7 @@ language: python
 sudo: false
 
 python:
-  - '3.5'
-  - '2.7.12'
-  - '2.6'
-  - '3.3'
-  - '3.4'
-  - '3.2'
-  - '3.6'
+  - '2.7' # this is a cheat here because travis doesn't do 2.4-2.6
 
 install:
 - pip install -e .

HOW-TO-REPORT-A-BUG.md

@@ -57,6 +57,19 @@ disassembler called `pydisasm`.
 
 ### Semantic equivalence vs. exact source code
 
+Consider how Python compiles something like "(x*y) + 5". Early on
+Python creates an "abstract syntax tree" (AST) for this. And this is
+"abstract" in the sense that unimportant, redundant or unnecceary
+items have been removed. Here, this means that any notion that you
+wrote "x+y" in parenthesis is lost, since in this context they are
+unneeded. Also lost is the fact that the multiplication didn't have
+spaces around it while the addition did. It should not come as a
+surprise then that the bytecode which is derived from the AST also has
+no notion of such possible variation. Generally this kind of thing
+isn't noticed since the Python community has laid out a very rigid set
+of formatting guidelines; and it has largely beaten the community into
+compliance.
+
 Almost all versions of Python can perform some sort of code
 improvement that can't be undone. In earlier versions of Python it is
 rare; in later Python versions, it is more common.
@@ -66,7 +79,7 @@ If the code emitted is semantically equivalent, then this isn't a bug.
 
 For example the code might be
 
-```
+```python
 if a:
   if b:
      x = 1
@@ -74,7 +87,7 @@ if a:
 
 and we might produce:
 
-```
+```python
 if a and b:
   x = 1
 ```
@@ -87,24 +100,35 @@ else:
 
 ```
 
-may come out as `elif`.
+may come out as `elif` or vice versa.
 
 
 As mentioned in the README, It is possible that Python changes what
 you write to be more efficient. For example, for:
 
 
-```
+```python
 if True:
   x = 5
 ```
 
 Python will generate code like:
 
-```
+```python
 x = 5
 ```
 
+Even more extreme, if your code is:
+
+```python
+if False:
+   x = 1
+   y = 2
+   # ...
+```
+
+Python will eliminate the entire "if" statement.
+
 So just because the text isn't the same, does not
 necessarily mean there's a bug.
 

Makefile

@@ -39,7 +39,7 @@ check-3.0 check-3.1 check-3.2 check-3.5 check-3.6:
 check-3.7: pytest
 
 #:Tests for Python 2.6 (doesn't have pytest)
-check-2.6:
+check-2.4 check-2.5 check-2.6:
 	$(MAKE) -C test $@
 
 #:PyPy 2.6.1 PyPy 5.0.1, or PyPy 5.8.0-beta0

NEWS

@@ -1,3 +1,34 @@
+uncompyle6 3.1.1 2018-04-01 Easter April Fool's
+
+Jesus on Friday's New York Times puzzle: "I'm stuck on 2A"
+
+- fill out 3.5+ BUILD_MAP_UNPACK (more work is needed)
+- fill out 3.4+ CALL_FUNCTION_... (more work is needed)
+- fill out 3.5 MAKE_FUNCTION  (more work is needed)
+- reduce 3.5, 3.6 control-flow bugs
+- reduce ambiguity in rules that lead to long (exponential?) parses
+- limit/isolate some 2.6/2.7,3.x grammar rules
+- more runtime testing of decompiled code
+- more removal of parenthesis around calls via setting precidence
+
+uncompyle6 3.1.0 2018-03-21 Equinox
+
+- Add code_deparse_with_offset() fragment function.
+- Correct paramenter call fragment deparse_code()
+- Lots of 3.6, 3.x, and 2.7 bug fixes
+  About 5% of 3.6 fail parsing now. But
+  semantics still needs much to be desired.
+
+uncompyle6 3.0.1 2018-02-17
+
+- All Python 2.6.9 standard library files weakly verify
+- Many 3.6 fixes. 84% of the first 200 standard library files weakly compile.
+  One more big push is needed to get the remaining to compile
+- Many decompilation fixes for other Python versions
+- Add more to the test framework
+- And more add tests target previous existing bugs more completely
+- sync recent license changes in metadata
+
 uncompyle6 3.0.0 2018-02-17
 
 - deparse_code() and lookalikes from the various semantic actions are
@@ -10,7 +41,7 @@ uncompyle6 3.0.0 2018-02-17
   of jumps, and messes up the peephole-like analysis that is
   done for control flow since we don't have something better in place.
 - Code has been reorganized to be more instruction nametuple based where it
-  has been more byecode array based. There was and still is code that had
+  has been more bytecode array based. There was and still is code that had
   had magic numbers to advance instructions or to pick out operands.
 - Bug fixes in numerous other Python versions
 - Instruction display improved
@@ -28,7 +59,7 @@ uncompyle6 2.16.0 2018-02-17
 - Better 2.7 end_if and COME_FROM determination
 - Fix up 3.6+ CALL_FUNCTION_EX
 - Misc pydisasm fixes
-- Wierd comprehension bug seen via new loctraceback
+- Weird comprehension bug seen via new loctraceback
 - Fix Python 3.5+ CALL_FUNCTION_VAR and BUILD_LIST_UNPACK in call; with this
   we can can handle 3.5+ f(a, b, *c, *d, *e) now
 
@@ -81,7 +112,7 @@ Decompilation bug fixes, mostly 3.6 and pre 2.7
 - limit pypy customization to pypy
 - Add addr fields in COME_FROMS
 - Allow use of full instructions in parser reduction routines
-- Reduce grammar in Pythion 3 by specialization more to specific
+- Reduce grammar in Python 3 by specialization more to specific
   Python versions
 - Match Python AST names more closely when possible
 
@@ -99,7 +130,7 @@ uncompyle6 2.14.0 2017-11-26 johnnybamazing
   and remove used grammar rules
 - Fix a number of bytecode decompile problems
   (many more remain)
-- Add stdlib/runtests.sh for even more rigourous testing
+- Add stdlib/runtests.sh for even more rigorous testing
 
 uncompyle6 2.13.3 2017-11-13
 
@@ -123,7 +154,7 @@ Overall: better 3.6 decompiling and some much needed code refactoring and cleanu
   added to assist here. Ignoring errors may be okay because the fragment parser often just needs,
   well, *fragments*.
 - Distinguish RETURN_VALUE from RETURN_END_IF in exception bodies better in 3.6
-- bug in 3.x language changes: import queue va import Queue
+- bug in 3.x language changes: import queue via import Queue
 - reinstate some bytecode tests since decompiling has gotten better
 - Revise how to report a bug
 
@@ -158,8 +189,8 @@ uncompyle6 2.11.4 2017-08-15
 
 * scanner and parser now allow 3-part version string lookups,
   e.g. 2.7.1 We allow a float here, but if passed a string like '2.7'. or
-* unpin 3.5.1. xdis 3.5.4 has been releasd and fixes the problems we had. Use that.
-* some routnes here moved to xdis. Use the xdis version
+* unpin 3.5.1. xdis 3.5.4 has been release and fixes the problems we had. Use that.
+* some routines here moved to xdis. Use the xdis version
 * README.rst: Link typo Name is trepan2 now not trepan
 * xdis-forced change adjust for COMPARE_OP "is-not" in
   semanatic routines. We need "is not".
@@ -255,9 +286,9 @@ uncompyle6 2.9.8 2016-12-16
 - fix bug in --verify option
 - DRY (a little) control-flow detection
 - fix syntax in tuples with one element
-- if AST rule inheritence in Python 2.5
+- if AST rule inheritance in Python 2.5
 - NAME_MODULE removal for Python <= 2.4
-- verifycall fixes for Python <= 2.4
+- verify call fixes for Python <= 2.4
 - more Python lint
 
 uncompyle6 2.9.7 2016-12-16
@@ -293,7 +324,7 @@ uncompyle6 2.9.6 2016-11-20
   in the results.
 - better control flow debugging output
 - Python 2 and 3 detect structure code is more similar
-- Handle Docstrings with embedded tiple quotes (""")
+- Handle Docstrings with embedded triple quotes (""")
 
 uncompyle6 2.9.5 2016-11-13
 

__pkginfo__.py

@@ -35,8 +35,6 @@ classifiers =  ['Development Status :: 5 - Production/Stable',
                 'Programming Language :: Python :: 2.5',
                 'Programming Language :: Python :: 2.6',
                 'Programming Language :: Python :: 2.7',
-                'Programming Language :: Python :: 3.1',
-                'Programming Language :: Python :: 3.2',
                 'Programming Language :: Python :: 3.3',
                 'Programming Language :: Python :: 3.4',
                 'Programming Language :: Python :: 3.5',
@@ -56,9 +54,8 @@ entry_points       = {
     ]}
 ftp_url            = None
 install_requires   = ['spark-parser >= 1.8.5, < 1.9.0',
-                      'xdis >= 3.6.9, < 3.7.0', 'six']
-
-license            = 'MIT'
+                      'xdis >= 3.7.0, < 3.8.0']
+license            = 'GPL3'
 mailing_list       = 'python-debugger@googlegroups.com'
 modname            = 'uncompyle6'
 py_modules         = None

admin-tools/how-to-make-a-release.txt

@@ -0,0 +1,46 @@
+git pull
+
+Change version in uncompyle6/version.py
+source uncompyle6/version.py
+echo $VERSION
+git commit -m"Get ready for release $VERSION" .
+
+Update ChangeLog:
+  make ChangeLog
+
+Update NEWS from ChangeLog
+make check
+
+git commit --amend .
+
+git push
+
+Make sure pyenv is running
+# Pyenv
+
+source admin-tools/check-newer-versions.sh
+
+
+# Switch to python-2.4 and build that first...
+source admin-tools/setup-python-2.4
+
+rm ChangeLog
+git merge master
+
+Update NEWS from master branch
+
+git commit -m"Get ready for release $VERSION" .
+
+source admin-tools/check-older-versions.sh
+source admin-tools/check-newer-versions.sh
+
+make-dist-older.sh
+
+git tag release-python-2.4-$VERSION
+
+./make-dist-newer.sh
+
+git tag release-$VERSION
+
+
+twine upload dist/uncompyle6-${VERSION}*

admin-tools/pyenv-newer-versions

@@ -5,4 +5,4 @@ if [[ $0 == ${BASH_SOURCE[0]} ]] ; then
     echo "This script should be *sourced* rather than run directly through bash"
     exit 1
 fi
-export PYVERSIONS='3.5.3 3.6.3 2.6.9 3.3.6 2.7.14 3.4.2'
+export PYVERSIONS='3.5.5 3.6.4 2.6.9 3.3.7 2.7.14 3.4.8'

admin-tools/run-pyenvlib-test-all.sh

@@ -0,0 +1,28 @@
+#!/bin/bash
+# Runs test_pyenvlib.test on all versions of Python master.
+function finish {
+  cd $owd
+}
+
+# FIXME put some of the below in a common routine
+owd=$(pwd)
+trap finish EXIT
+
+cd $(dirname ${BASH_SOURCE[0]})
+if ! source ./pyenv-newer-versions ; then
+    exit $?
+fi
+if ! source ./setup-master.sh ; then
+    exit $?
+fi
+cd ../test
+for version in $PYVERSIONS; do
+    if ! pyenv local $version ; then
+	exit $?
+    fi
+    echo "====== Running test_pyenvlib.py on $version ====="
+    if ! python ./test_pyenvlib.py --weak-verify --max 800 --${version} ; then
+	exit $?
+    fi
+    echo "------ Done test_pyenvlib.py on $version -----"
+done

admin-tools/setup-master.sh

@@ -1,5 +1,5 @@
 #!/bin/bash
-PYTHON_VERSION=3.6.3
+PYTHON_VERSION=3.6.4
 
 # FIXME put some of the below in a common routine
 function finish {

admin-tools/update-sources.sh

@@ -0,0 +1,3 @@
+#!/bin/bash
+cd $(dirname ${BASH_SOURCE[0]})/..
+git pull

circle.yml

@@ -11,4 +11,4 @@ dependencies:
 test:
     override:
         - python ./setup.py develop && make check-2.7
-        - cd ./test/stdlib && pyenv local 2.7.10 && bash ./runtests.sh 'test_[p-z]*.py'
+        - cd ./test/stdlib && pyenv local 2.7.10 && bash ./runtests.sh 'test_[f-i]*.py'

pytest/test_deparse.py

@@ -30,7 +30,7 @@ def list_comp():
     [y for y in range(3)]
 
 def get_parsed_for_fn(fn):
-    code = fn.__code__ if PYTHON3 else fn.func_code
+    code = fn.func_code
     return deparse(code, version=PYTHON_VERSION)
 
 def check_expect(expect, parsed, fn_name):

pytest/test_docstring.py

@@ -10,7 +10,7 @@ else:
     maxint = sys.maxint
 from uncompyle6.semantics.helper import print_docstring
 
-class PrintFake():
+class PrintFake:
     def __init__(self):
         self.pending_newlines = 0
         self.f = StringIO()

pytest/test_fjt.py

@@ -22,9 +22,8 @@ def bug_loop(disassemble, tb=None):
     disassemble(tb)
 
 def test_if_in_for():
-    code = bug.__code__
+    code = bug.func_code
     scan = get_scanner(PYTHON_VERSION)
-    print(PYTHON_VERSION)
     if 2.7 <= PYTHON_VERSION <= 3.0 and not IS_PYPY:
         n = scan.setup_code(code)
         bytecode = Bytecode(code, scan.opc)

pytest/test_fstring.py

@@ -1,150 +0,0 @@
-# std
-import os
-# test
-import pytest
-import hypothesis
-from hypothesis import strategies as st
-# uncompyle6
-from uncompyle6 import PYTHON_VERSION, deparse_code
-
-
-@st.composite
-def expressions(draw):
-    # todo : would be nice to generate expressions using hypothesis however
-    # this is pretty involved so for now just use a corpus of expressions
-    # from which to select.
-    return draw(st.sampled_from((
-        'abc',
-        'len(items)',
-        'x + 1',
-        'lineno',
-        'container',
-        'self.attribute',
-        'self.method()',
-        # These expressions are failing, I think these are control
-        # flow problems rather than problems with FORMAT_VALUE,
-        # however I need to confirm this...
-        #'sorted(items, key=lambda x: x.name)',
-        #'func(*args, **kwargs)',
-        #'text or default',
-        #'43 if life_the_universe and everything else None'
-    )))
-
-
-@st.composite
-def format_specifiers(draw):
-    """
-    Generate a valid format specifier using the rules:
-
-    format_spec ::=  [[fill]align][sign][#][0][width][,][.precision][type]
-    fill        ::=  <any character>
-    align       ::=  "<" | ">" | "=" | "^"
-    sign        ::=  "+" | "-" | " "
-    width       ::=  integer
-    precision   ::=  integer
-    type        ::=  "b" | "c" | "d" | "e" | "E" | "f" | "F" | "g" | "G" | "n" | "o" | "s" | "x" | "X" | "%"
-
-    See https://docs.python.org/2/library/string.html
-
-    :param draw: Let hypothesis draw from other strategies.
-
-    :return: An example format_specifier.
-    """
-    alphabet_strategy = st.characters(min_codepoint=ord('a'), max_codepoint=ord('z'))
-    fill = draw(st.one_of(alphabet_strategy, st.none()))
-    align = draw(st.sampled_from(list('<>=^')))
-    fill_align = (fill + align or '') if fill else ''
-
-    type_ = draw(st.sampled_from('bcdeEfFgGnosxX%'))
-    can_have_sign = type_ in 'deEfFgGnoxX%'
-    can_have_comma = type_ in 'deEfFgG%'
-    can_have_precision = type_ in 'fFgG'
-    can_have_pound = type_ in 'boxX%'
-    can_have_zero = type_ in 'oxX'
-
-    sign = draw(st.sampled_from(list('+- ') + [''])) if can_have_sign else ''
-    pound = draw(st.sampled_from(('#', '',))) if can_have_pound else ''
-    zero = draw(st.sampled_from(('0', '',))) if can_have_zero else ''
-
-    int_strategy = st.integers(min_value=1, max_value=1000)
-
-    width = draw(st.one_of(int_strategy, st.none()))
-    width = str(width) if width is not None else ''
-
-    comma = draw(st.sampled_from((',', '',))) if can_have_comma else ''
-    if can_have_precision:
-        precision = draw(st.one_of(int_strategy, st.none()))
-        precision = '.' + str(precision) if precision else ''
-    else:
-        precision = ''
-
-    return ''.join((fill_align, sign, pound, zero, width, comma, precision, type_,))
-
-
-@st.composite
-def fstrings(draw):
-    """
-    Generate a valid f-string.
-    See https://www.python.org/dev/peps/pep-0498/#specification
-
-    :param draw: Let hypothsis draw from other strategies.
-
-    :return: A valid f-string.
-    """
-    character_strategy = st.characters(
-        blacklist_characters='\r\n\'\\s{}',
-        min_codepoint=1,
-        max_codepoint=1000,
-    )
-    is_raw = draw(st.booleans())
-    integer_strategy = st.integers(min_value=0, max_value=3)
-    expression_count = draw(integer_strategy)
-    content = []
-    for _ in range(expression_count):
-        expression = draw(expressions())
-        conversion = draw(st.sampled_from(('', '!s', '!r', '!a',)))
-        has_specifier = draw(st.booleans())
-        specifier = ':' + draw(format_specifiers()) if has_specifier else ''
-        content.append('{{{}{}}}'.format(expression, conversion, specifier))
-        content.append(draw(st.text(character_strategy)))
-    content = ''.join(content)
-    return "f{}'{}'".format('r' if is_raw else '', content)
-
-
-@pytest.mark.skipif(PYTHON_VERSION < 3.6, reason='need at least python 3.6')
-@hypothesis.given(format_specifiers())
-def test_format_specifiers(format_specifier):
-    """Verify that format_specifiers generates valid specifiers"""
-    try:
-        exec('"{:' + format_specifier + '}".format(0)')
-    except ValueError as e:
-        if 'Unknown format code' not in str(e):
-            raise
-
-
-def run_test(text):
-    hypothesis.assume(len(text))
-    hypothesis.assume("f'{" in text)
-    expr = text + '\n'
-    code = compile(expr, '<string>', 'single')
-    deparsed = deparse_code(PYTHON_VERSION, code, compile_mode='single')
-    recompiled = compile(deparsed.text, '<string>', 'single')
-    if recompiled != code:
-        assert 'dis(' + deparsed.text.strip('\n') + ')' == 'dis(' + expr.strip('\n') + ')'
-
-
-@pytest.mark.skipif(PYTHON_VERSION < 3.6, reason='need at least python 3.6')
-@hypothesis.given(fstrings())
-def test_uncompyle_fstring(fstring):
-    """Verify uncompyling fstring bytecode"""
-    run_test(fstring)
-
-
-@pytest.mark.skipif(PYTHON_VERSION < 3.6, reason='need at least python 3.6')
-@pytest.mark.parametrize('fstring', [
-    "f'{abc}{abc!s}'",
-    "f'{abc}0'",
-])
-def test_uncompyle_direct(fstring):
-    """useful for debugging"""
-    run_test(fstring)

pytest/test_function_call.py

@@ -1,175 +0,0 @@
-# std
-import string
-# 3rd party
-from hypothesis import given, assume, example, settings, strategies as st
-import pytest
-# uncompyle
-from validate import validate_uncompyle
-from test_fstring import expressions
-
-
-alpha = st.sampled_from(string.ascii_lowercase)
-numbers = st.sampled_from(string.digits)
-alphanum = st.sampled_from(string.ascii_lowercase + string.digits)
-
-
-@st.composite
-def function_calls(draw,
-                   min_keyword_args=0, max_keyword_args=5,
-                   min_positional_args=0, max_positional_args=5,
-                   min_star_args=0, max_star_args=1,
-                   min_double_star_args=0, max_double_star_args=1):
-    """
-    Strategy factory for generating function calls.
-
-    :param draw: Callable which draws examples from other strategies.
-
-    :return: The function call text.
-    """
-    st_positional_args = st.lists(
-        alpha,
-        min_size=min_positional_args,
-        max_size=max_positional_args
-    )
-    st_keyword_args = st.lists(
-        alpha,
-        min_size=min_keyword_args,
-        max_size=max_keyword_args
-    )
-    st_star_args = st.lists(
-        alpha,
-        min_size=min_star_args,
-        max_size=max_star_args
-    )
-    st_double_star_args = st.lists(
-        alpha,
-        min_size=min_double_star_args,
-        max_size=max_double_star_args
-    )
-
-    positional_args = draw(st_positional_args)
-    keyword_args = draw(st_keyword_args)
-    st_values = st.lists(
-        expressions(),
-        min_size=len(keyword_args),
-        max_size=len(keyword_args)
-    )
-    keyword_args = [
-        x + '=' + e
-        for x, e in
-        zip(keyword_args, draw(st_values))
-    ]
-    star_args = ['*' + x for x in draw(st_star_args)]
-    double_star_args = ['**' + x for x in draw(st_double_star_args)]
-
-    arguments = positional_args + keyword_args + star_args + double_star_args
-    draw(st.randoms()).shuffle(arguments)
-    arguments = ','.join(arguments)
-
-    function_call = 'fn({arguments})'.format(arguments=arguments)
-    try:
-        # TODO: Figure out the exact rules for ordering of positional, keyword,
-        # star args, double star args and in which versions the various
-        # types of arguments are supported so we don't need to check that the
-        # expression compiles like this.
-        compile(function_call, '<string>', 'single')
-    except:
-        assume(False)
-    return function_call
-
-
-def test_function_no_args():
-    validate_uncompyle("fn()")
-
-
-def isolated_function_calls(which):
-    """
-    Returns a strategy for generating function calls, but isolated to 
-    particular types of arguments, for example only positional arguments.
-
-    This can help reason about debugging errors in specific types of function
-    calls.
-
-    :param which: One of 'keyword', 'positional', 'star', 'double_star'
-
-    :return: Strategy for generating an function call isolated to specific
-             argument types.
-    """
-    kwargs = dict(
-        max_keyword_args=0,
-        max_positional_args=0,
-        max_star_args=0,
-        max_double_star_args=0,
-    )
-    kwargs['_'.join(('min', which, 'args'))] = 1
-    kwargs['_'.join(('max', which, 'args'))] = 5 if 'star' not in which else 1
-    return function_calls(**kwargs)
-
-
-with settings(max_examples=25):
-
-    @given(isolated_function_calls('positional'))
-    @example("fn(0)")
-    def test_function_positional_only(expr):
-        validate_uncompyle(expr)
-
-    @given(isolated_function_calls('keyword'))
-    @example("fn(a=0)")
-    def test_function_call_keyword_only(expr):
-        validate_uncompyle(expr)
-
-    @given(isolated_function_calls('star'))
-    @example("fn(*items)")
-    def test_function_call_star_only(expr):
-        validate_uncompyle(expr)
-
-    @given(isolated_function_calls('double_star'))
-    @example("fn(**{})")
-    def test_function_call_double_star_only(expr):
-        validate_uncompyle(expr)
-
-
-@pytest.mark.xfail()
-def test_BUILD_CONST_KEY_MAP_BUILD_MAP_UNPACK_WITH_CALL_BUILD_TUPLE_CALL_FUNCTION_EX():
-    validate_uncompyle("fn(w=0,m=0,**v)")
-
-
-@pytest.mark.xfail()
-def test_BUILD_MAP_BUILD_MAP_UNPACK_WITH_CALL_BUILD_TUPLE_CALL_FUNCTION_EX():
-    validate_uncompyle("fn(a=0,**g)")
-
-
-@pytest.mark.xfail()
-def test_CALL_FUNCTION_EX():
-    validate_uncompyle("fn(*g,**j)")
-
-
-@pytest.mark.xfail()
-def test_BUILD_MAP_CALL_FUNCTION_EX():
-    validate_uncompyle("fn(*z,u=0)")
-
-
-@pytest.mark.xfail()
-def test_BUILD_TUPLE_CALL_FUNCTION_EX():
-    validate_uncompyle("fn(**a)")
-
-
-@pytest.mark.xfail()
-def test_BUILD_MAP_BUILD_TUPLE_BUILD_TUPLE_UNPACK_WITH_CALL_CALL_FUNCTION_EX():
-    validate_uncompyle("fn(b,b,b=0,*a)")
-
-
-@pytest.mark.xfail()
-def test_BUILD_TUPLE_BUILD_TUPLE_UNPACK_WITH_CALL_CALL_FUNCTION_EX():
-    validate_uncompyle("fn(*c,v)")
-
-
-@pytest.mark.xfail()
-def test_BUILD_CONST_KEY_MAP_CALL_FUNCTION_EX():
-    validate_uncompyle("fn(i=0,y=0,*p)")
-
-
-@pytest.mark.skip(reason='skipping property based test until all individual tests are passing')
-@given(function_calls())
-def test_function_call(function_call):
-    validate_uncompyle(function_call)

pytest/test_grammar.py

@@ -18,27 +18,28 @@ def test_grammar():
      right_recursive, dup_rhs) = p.check_sets()
 
     # We have custom rules that create the below
-    expect_lhs = set(['expr1024', 'pos_arg', 'get_iter', 'attribute'])
+    expect_lhs = set(['pos_arg', 'get_iter', 'attribute'])
 
-    unused_rhs = set(['list', 'mkfunc',
+    unused_rhs = set(['list', 'mkfunc', 'dict',
                       'mklambda',
                       'unpack',])
     expect_right_recursive = set([('designList',
                                    ('store', 'DUP_TOP', 'designList'))])
+    expect_lhs.add('kvlist')
+    expect_lhs.add('kv3')
     if PYTHON3:
         expect_lhs.add('load_genexpr')
-        expect_lhs.add('kvlist')
-        expect_lhs.add('kv3')
 
         unused_rhs = unused_rhs.union(set("""
-        except_pop_except generator_exp classdefdeco2
-        dict
+        except_pop_except generator_exp
         """.split()))
         if PYTHON_VERSION >= 3.0:
             expect_lhs.add("annotate_arg")
             expect_lhs.add("annotate_tuple")
             unused_rhs.add("mkfunc_annotate")
             unused_rhs.add('call')
+            unused_rhs.add("dict_comp")
+            unused_rhs.add("classdefdeco1")
             if PYTHON_VERSION < 3.6:
                 # 3.6 has at least one non-custom call rule
                 # the others don't
@@ -51,7 +52,6 @@ def test_grammar():
             else:
                 expect_right_recursive.add((('l_stmts',
                                              ('lastl_stmt', 'COME_FROM', 'l_stmts'))))
-                # expect_lhs.add('kwargs1')
                 pass
             pass
         pass

pytest/test_pysource.py

@@ -8,12 +8,8 @@ from uncompyle6.semantics.consts import (
 
 if PYTHON3:
     from io import StringIO
-    def iteritems(d):
-        return d.items()
 else:
     from StringIO import StringIO
-    def iteritems(d):
-        return d.iteritems()
 
 from uncompyle6.semantics.pysource import SourceWalker as SourceWalker
 
@@ -30,7 +26,7 @@ def test_template_engine():
     # FIXME: and so on...
 
 from uncompyle6.semantics.consts import (
-    TABLE_DIRECT, TABLE_R,
+    TABLE_R, TABLE_DIRECT,
     )
 
 from uncompyle6.semantics.fragments import (
@@ -44,7 +40,7 @@ def test_tables():
             (TABLE_DIRECT, 'TABLE_DIRECT', False),
             (TABLE_R, 'TABLE_R', False),
             (TABLE_DIRECT_FRAGMENT, 'TABLE_DIRECT_FRAGMENT', True)):
-        for k, entry in iteritems(t):
+        for k, entry in t.iteritems():
             if k in skip_for_now:
                 continue
             fmt = entry[0]

pytest/test_single_compile.py

@@ -1,19 +1,19 @@
-import pytest
-from uncompyle6 import PYTHON_VERSION, PYTHON3, deparse_code
+from uncompyle6 import PYTHON_VERSION, deparse_code
 
-def test_single_mode():
-    single_expressions = (
-        'i = 1',
-        'i and (j or k)',
-        'i += 1',
-        'i = j % 4',
-        'i = {}',
-        'i = []',
-        'for i in range(10):\n    i\n',
-        'for i in range(10):\n    for j in range(10):\n        i + j\n',
-        'try:\n    i\nexcept Exception:\n    j\nelse:\n    k\n'
-    )
+if PYTHON_VERSION >= 2.5:
+    def test_single_mode():
+        single_expressions = (
+            'i = 1',
+            'i and (j or k)',
+            'i += 1',
+            'i = j % 4',
+            'i = {}',
+            'i = []',
+            'for i in range(10):\n    i\n',
+            'for i in range(10):\n    for j in range(10):\n        i + j\n',
+            'try:\n    i\nexcept Exception:\n    j\nelse:\n    k\n'
+        )
 
-    for expr in single_expressions:
-        code = compile(expr + '\n', '<string>', 'single')
-        assert deparse_code(PYTHON_VERSION, code, compile_mode='single').text == expr + '\n'
+        for expr in single_expressions:
+            code = compile(expr + '\n', '<string>', 'single')
+            assert deparse_code(PYTHON_VERSION, code, compile_mode='single').text == expr + '\n'

pytest/testdata/if-2.7.right

@@ -9,4 +9,4 @@
           12  JUMP_FORWARD          0  'to 15'
         15_0  COME_FROM            12  '12'
           15  LOAD_CONST            0  None
-          18  RETURN_VALUE     
+          18  RETURN_VALUE

pytest/testdata/ifelse-2.7.right

@@ -12,4 +12,4 @@
           18  STORE_NAME            2  'd'
         21_0  COME_FROM            12  '12'
           21  LOAD_CONST            2  None
-          24  RETURN_VALUE     
+          24  RETURN_VALUE

pytest/validate.py

@@ -1,24 +1,25 @@
-# future
-from __future__ import print_function
 # std
 import os
 import difflib
 import subprocess
 import tempfile
-import functools
-# compatability
-import six
+
+from StringIO import StringIO
 # uncompyle6 / xdis
 from uncompyle6 import PYTHON_VERSION, IS_PYPY, deparse_code
 # TODO : I think we can get xdis to support the dis api (python 3 version) by doing something like this there
 from xdis.bytecode import Bytecode
 from xdis.main import get_opcode
 opc = get_opcode(PYTHON_VERSION, IS_PYPY)
-Bytecode = functools.partial(Bytecode, opc=opc)
 
+try:
+    import functools
+    Bytecode = functools.partial(Bytecode, opc=opc)
+    def _dis_to_text(co):
+        return Bytecode(co).dis()
+except:
+    pass
 
-def _dis_to_text(co):
-    return Bytecode(co).dis()
 
 
 def print_diff(original, uncompyled):
@@ -42,8 +43,11 @@ def print_diff(original, uncompyled):
         print('\nTo display diff highlighting run:\n    pip install BeautifulSoup4')
         diff = difflib.HtmlDiff().make_table(*args)
 
-    with tempfile.NamedTemporaryFile(delete=False) as f:
+    f = tempfile.NamedTemporaryFile(delete=False)
+    try:
         f.write(str(diff).encode('utf-8'))
+    finally:
+        f.close()
 
     try:
         print()
@@ -60,8 +64,7 @@ def print_diff(original, uncompyled):
         print('\nFor side by side diff install elinks')
         diff = difflib.Differ().compare(original_lines, uncompyled_lines)
         print('\n'.join(diff))
-    finally:
-        os.unlink(f.name)
+    os.unlink(f.name)
 
 
 def are_instructions_equal(i1, i2):
@@ -123,8 +126,9 @@ def validate_uncompyle(text, mode='exec'):
     original_text = text
 
     deparsed = deparse_code(PYTHON_VERSION, original_code,
+
                             compile_mode=mode,
-                            out=six.StringIO(),
+                            out=StringIO(),
                             is_pypy=IS_PYPY)
     uncompyled_text = deparsed.text
     uncompyled_code = compile(uncompyled_text, '<string>', 'exec')

setup.py

@@ -1,16 +1,16 @@
 #!/usr/bin/env python
-import sys
-
 """Setup script for the 'uncompyle6' distribution."""
 
+import sys
+
 SYS_VERSION = sys.version_info[0:2]
-if not ((2, 6) <= SYS_VERSION  <= (3, 7)) or ((3, 0) <= SYS_VERSION <= (3, 1)):
-    mess = "Python Release 2.6 .. 3.7 excluding 3.0 and 3.1 are supported in this code branch."
-    if ((2, 4) <= SYS_VERSION <= (2, 7)):
-        mess += ("\nFor your Python, version %s, use the python-2.4 code/branch." %
+if not ((2, 4) <= SYS_VERSION  <= (2, 7)):
+    mess = "Python Release 2.4 .. 2.7 are supported in this code branch."
+    if ((3, 2) <= SYS_VERSION  <= (3, 7)):
+        mess += ("\nFor your Python, version %s, use the master code/branch." %
                  sys.version[0:3])
-    elif SYS_VERSION < (2, 4) or ((3, 0) <= SYS_VERSION <= (3, 1)):
-        mess += ("\nThis package is not supported for Python version %s."
+    else:
+        mess += ("\nThis package is not supported before Python 2.4. Your Python version is %s."
                  % sys.version[0:3])
     print(mess)
     raise Exception(mess)

test-unit/test_grammar.py

@@ -0,0 +1,55 @@
+import re
+import unittest
+from uncompyle6 import PYTHON_VERSION, IS_PYPY # , PYTHON_VERSION
+from uncompyle6.parser import get_python_parser, python_parser
+
+class TestGrammar(unittest.TestCase):
+    def test_grammar(self):
+
+        def check_tokens(tokens, opcode_set):
+            remain_tokens = set(tokens) - opcode_set
+            remain_tokens = set([re.sub('_\d+$','', t) for t in remain_tokens])
+            remain_tokens = set([re.sub('_CONT$','', t) for t in remain_tokens])
+            remain_tokens = set(remain_tokens) - opcode_set
+            self.assertEqual(remain_tokens, set([]),
+                    "Remaining tokens %s\n====\n%s" % (remain_tokens, p.dump_grammar()))
+
+        p = get_python_parser(PYTHON_VERSION, is_pypy=IS_PYPY)
+        (lhs, rhs, tokens,
+         right_recursive, dup_rhs) = p.check_sets()
+        expect_lhs = set(['expr1024', 'pos_arg'])
+        unused_rhs = set(['list', 'call', 'mkfunc',
+                          'mklambda',
+                          'unpack',])
+
+        expect_right_recursive = frozenset([('designList',
+                                             ('store', 'DUP_TOP', 'designList'))])
+        expect_lhs.add('kwarg')
+
+        self.assertEqual(expect_lhs, set(lhs))
+        self.assertEqual(unused_rhs, set(rhs))
+        self.assertEqual(expect_right_recursive, right_recursive)
+
+        expect_dup_rhs = frozenset([('COME_FROM',), ('CONTINUE',), ('JUMP_ABSOLUTE',),
+                                    ('LOAD_CONST',),
+                                    ('JUMP_BACK',), ('JUMP_FORWARD',)])
+
+        reduced_dup_rhs = {}
+        for k in dup_rhs:
+            if k not in expect_dup_rhs:
+                reduced_dup_rhs[k] = dup_rhs[k]
+                pass
+            pass
+        for k in reduced_dup_rhs:
+            print(k, reduced_dup_rhs[k])
+        # assert not reduced_dup_rhs, reduced_dup_rhs
+
+    def test_dup_rule(self):
+        import inspect
+        python_parser(PYTHON_VERSION, inspect.currentframe().f_code,
+                      is_pypy=IS_PYPY,
+                      parser_debug={
+                          'dups': True, 'transition': False, 'reduce': False,
+                          'rules': False, 'errorstack': None, 'context': True})
+if __name__ == '__main__':
+    unittest.main()

test/.gitignore

@@ -0,0 +1 @@
+/nohup.out

test/Makefile

@@ -3,9 +3,9 @@ PHONY=check clean dist distclean test test-unit test-functional rmChangeLog clea
       check-bytecode-2.2 check-byteocde-2.3 check-bytecode-2.4 \
       check-short check-2.6 check-2.7 check-3.0 check-3.1 check-3.2 check-3.3 \
       check-3.4 check-3.5 check-5.6 5.6 5.8 \
-      grammar-coverage-2.5 grammar-coverage-2.6 grammarcoverage-2.7 \
-      grammar-coverage-3.1 grammar-coverage-3.2 grammarcoverage-3.3 \
-      grammar-coverage-3.4 grammar-coverage-3.5 grammarcoverage-3.6
+      grammar-coverage-2.5 grammar-coverage-2.6 grammar-coverage-2.7 \
+      grammar-coverage-3.1 grammar-coverage-3.2 grammar-coverage-3.3 \
+      grammar-coverage-3.4 grammar-coverage-3.5 grammar-coverage-3.6
 
 
 GIT2CL ?= git2cl
@@ -28,7 +28,7 @@ check:
 	$(MAKE) check-$(PYTHON_VERSION)
 
 #: Run working tests from Python 2.6 or 2.7
-check-2.6 check-2.7: check-bytecode-2 check-bytecode-3 check-bytecode-1 check-native-short
+check-2.4 check-2.5 check-2.6 check-2.7: check-bytecode-2 check-bytecode-3 check-bytecode-1  check-native-short
 
 #: Run working tests from Python 3.0
 check-3.0: check-bytecode
@@ -41,18 +41,22 @@ check-3.1: check-bytecode
 #: Run working tests from Python 3.2
 check-3.2: check-bytecode
 	$(PYTHON) test_pythonlib.py --bytecode-3.2 --weak-verify $(COMPILE)
+	$(PYTHON) test_pythonlib.py --bytecode-3.2-run --verify-run
 
 #: Run working tests from Python 3.3
 check-3.3: check-bytecode
 	$(PYTHON) test_pythonlib.py --bytecode-3.3 --weak-verify $(COMPILE)
+	$(PYTHON) test_pythonlib.py --bytecode-3.3-run --verify-run
 
 #: Run working tests from Python 3.4
 check-3.4: check-bytecode check-3.4-ok check-2.7-ok
 	$(PYTHON) test_pythonlib.py --bytecode-3.4 --weak-verify $(COMPILE)
+	$(PYTHON) test_pythonlib.py --bytecode-3.4-run --verify-run
 
 #: Run working tests from Python 3.5
 check-3.5: check-bytecode
 	$(PYTHON) test_pythonlib.py --bytecode-3.5 --weak-verify $(COMPILE)
+	$(PYTHON) test_pythonlib.py --bytecode-3.5-run --verify-run
 
 #: Run working tests from Python 3.6
 check-3.6: check-bytecode
@@ -113,23 +117,29 @@ check-bytecode-2.4:
 check-bytecode-2.5:
 	$(PYTHON) test_pythonlib.py --bytecode-2.5
 
+#: Get grammar coverage for Python 2.4
+grammar-coverage-2.4:
+	-rm $(COVER_DIR)/spark-grammar-24.cover
+	SPARK_PARSER_COVERAGE=$(COVER_DIR)/spark-grammar-2.4.cover $(PYTHON) test_pythonlib.py --bytecode-2.4
+	SPARK_PARSER_COVERAGE=$(COVER_DIR)/spark-grammar-2.4.cover $(PYTHON) test_pyenvlib.py --2.4.6 --max= 800
+
 #: Get grammar coverage for Python 2.5
 grammar-coverage-2.5:
-	-rm $(COVER_DIR)/spark-grammar-25.cover
-	SPARK_PARSER_COVERAGE=$(COVER_DIR)/spark-grammar-25.cover $(PYTHON) test_pythonlib.py --bytecode-2.5
-	SPARK_PARSER_COVERAGE=$(COVER_DIR)/spark-grammar-25.cover $(PYTHON) test_pyenvlib.py --2.5.6
+	-rm $(COVER_DIR)/spark-grammar-2.5.cover || true
+	SPARK_PARSER_COVERAGE=$(COVER_DIR)/spark-grammar-2.5.cover $(PYTHON) test_pythonlib.py --bytecode-2.5
+	SPARK_PARSER_COVERAGE=$(COVER_DIR)/spark-grammar-2.5.cover $(PYTHON) test_pyenvlib.py --2.5.6 --max=800
 
 #: Get grammar coverage for Python 2.6
 grammar-coverage-2.6:
-	-rm $(COVER_DIR)/spark-grammar-26.cover
-	SPARK_PARSER_COVERAGE=$(COVER_DIR)/spark-grammar-26.cover $(PYTHON) test_pythonlib.py --bytecode-2.6
-	SPARK_PARSER_COVERAGE=$(COVER_DIR)/spark-grammar-26.cover $(PYTHON) test_pyenvlib.py --2.6.9
+	-rm $(COVER_DIR)/spark-grammar-2.6.cover || true
+	SPARK_PARSER_COVERAGE=$(COVER_DIR)/spark-grammar-2.6.cover $(PYTHON) test_pythonlib.py --bytecode-2.6
+	SPARK_PARSER_COVERAGE=$(COVER_DIR)/spark-grammar-2.6.cover $(PYTHON) test_pyenvlib.py --2.6.9 --max=800
 
 #: Get grammar coverage for Python 2.7
 grammar-coverage-2.7:
-	-rm $(COVER_DIR)/spark-grammar-27.cover
-	SPARK_PARSER_COVERAGE=$(COVER_DIR)/spark-grammar-27.cover $(PYTHON) test_pythonlib.py --bytecode-2.7
-	SPARK_PARSER_COVERAGE=$(COVER_DIR)/spark-grammar-27.cover $(PYTHON) test_pyenvlib.py --2.7.13
+	-rm $(COVER_DIR)/spark-grammar-2.7.cover || true
+	SPARK_PARSER_COVERAGE=$(COVER_DIR)/spark-grammar-2.7.cover $(PYTHON) test_pythonlib.py --bytecode-2.7
+	SPARK_PARSER_COVERAGE=$(COVER_DIR)/spark-grammar-2.7.cover $(PYTHON) test_pyenvlib.py --2.7.14 --max=600
 
 #: Get grammar coverage for Python 3.0
 grammar-coverage-3.0:
@@ -140,33 +150,39 @@ SPARK_PARSER_COVERAGE=$(COVER_DIR)/spark-grammar-30.cover $(PYTHON) test_pythonl
 
 #: Get grammar coverage for Python 3.1
 grammar-coverage-3.1:
-	-rm $(COVER_DIR)/spark-grammar-31.cover
-	SPARK_PARSER_COVERAGE=$(COVER_DIR)/spark-grammar-31.cover $(PYTHON) test_pythonlib.py --bytecode-3.1
-	SPARK_PARSER_COVERAGE=$(COVER_DIR)/spark-grammar-31.cover $(PYTHON) test_pyenvlib.py --3.1.5
+	-rm $(COVER_DIR)/spark-grammar-3.1.cover
+	SPARK_PARSER_COVERAGE=$(COVER_DIR)/spark-grammar-3.1.cover $(PYTHON) test_pythonlib.py --bytecode-3.1
+	SPARK_PARSER_COVERAGE=$(COVER_DIR)/spark-grammar-3.1.cover $(PYTHON) test_pyenvlib.py --3.1.5
 
 #: Get grammar coverage for Python 3.2
 grammar-coverage-3.2:
-	-rm $(COVER_DIR)/spark-grammar-32.cover
-	SPARK_PARSER_COVERAGE=$(COVER_DIR)/spark-grammar-32.cover $(PYTHON) test_pythonlib.py --bytecode-3.2
-	SPARK_PARSER_COVERAGE=$(COVER_DIR)/spark-grammar-32.cover $(PYTHON) test_pyenvlib.py --3.2.6
+	-rm $(COVER_DIR)/spark-grammar-3.2.cover || true
+	SPARK_PARSER_COVERAGE=$(COVER_DIR)/spark-grammar-3.2.cover $(PYTHON) test_pythonlib.py --bytecode-3.2
+	SPARK_PARSER_COVERAGE=$(COVER_DIR)/spark-grammar-3.2.cover $(PYTHON) test_pyenvlib.py --3.2.6
 
 #: Get grammar coverage for Python 3.3
 grammar-coverage-3.3:
-	-rm $(COVER_DIR)/spark-grammar-33.cover
-	SPARK_PARSER_COVERAGE=$(COVER_DIR)/spark-grammar-33.cover $(PYTHON) test_pythonlib.py --bytecode-3.3
-	SPARK_PARSER_COVERAGE=$(COVER_DIR)/spark-grammar-33.cover $(PYTHON) test_pyenvlib.py --3.3.6
+	-rm $(COVER_DIR)/spark-grammar-3.3.cover || true
+	SPARK_PARSER_COVERAGE=$(COVER_DIR)/spark-grammar-3.3.cover $(PYTHON) test_pythonlib.py --bytecode-3.3
+	SPARK_PARSER_COVERAGE=$(COVER_DIR)/spark-grammar-3.3.cover $(PYTHON) test_pyenvlib.py --3.3.7 --max=800
 
 #: Get grammar coverage for Python 3.4
 grammar-coverage-3.4:
-	-rm $(COVER_DIR)/spark-grammar-34.cover
-	SPARK_PARSER_COVERAGE=$(COVER_DIR)/spark-grammar-34.cover $(PYTHON) test_pythonlib.py --bytecode-3.4
-	SPARK_PARSER_COVERAGE=$(COVER_DIR)/spark-grammar-34.cover $(PYTHON) test_pyenvlib.py --3.4.2
+	-rm $(COVER_DIR)/spark-grammar-3.4.cover || true
+	SPARK_PARSER_COVERAGE=$(COVER_DIR)/spark-grammar-3.4.cover $(PYTHON) test_pythonlib.py --bytecode-3.4
+	SPARK_PARSER_COVERAGE=$(COVER_DIR)/spark-grammar-3.4.cover $(PYTHON) test_pyenvlib.py --3.4.8 --max=800
 
 #: Get grammar coverage for Python 3.5
 grammar-coverage-3.5:
-	rm $(COVER_DIR)/spark-grammar-35.cover || /bin/true
-	SPARK_PARSER_COVERAGE=$(COVER_DIR)/spark-grammar-35.cover $(PYTHON) test_pythonlib.py --bytecode-3.5
-	SPARK_PARSER_COVERAGE=$(COVER_DIR)/spark-grammar-35.cover $(PYTHON) test_pyenvlib.py --3.5.3
+	rm $(COVER_DIR)/spark-grammar-3.5.cover || /bin/true
+	SPARK_PARSER_COVERAGE=$(COVER_DIR)/spark-grammar-3.5.cover $(PYTHON) test_pythonlib.py --bytecode-3.5
+	SPARK_PARSER_COVERAGE=$(COVER_DIR)/spark-grammar-3.5.cover $(PYTHON) test_pyenvlib.py --3.5.5 --max=450
+
+#: Get grammar coverage for Python 3.6
+grammar-coverage-3.6:
+	rm $(COVER_DIR)/spark-grammar-3.6.cover || /bin/true
+	SPARK_PARSER_COVERAGE=$(COVER_DIR)/spark-grammar-3.6.cover $(PYTHON) test_pythonlib.py --bytecode-3.6
+	SPARK_PARSER_COVERAGE=$(COVER_DIR)/spark-grammar-3.6.cover $(PYTHON) test_pyenvlib.py --3.6.4 --max=280
 
 #: Check deparsing Python 2.6
 check-bytecode-2.6:
@@ -189,6 +205,7 @@ check-bytecode-3.1:
 #: Check deparsing Python 3.2
 check-bytecode-3.2:
 	$(PYTHON) test_pythonlib.py --bytecode-3.2 --weak-verify
+	$(PYTHON) test_pythonlib.py --bytecode-3.2-run --verify-run
 
 #: Check deparsing Python 3.3
 check-bytecode-3.3:
@@ -215,6 +232,10 @@ check-native-short:
 	$(PYTHON) test_pythonlib.py --bytecode-$(PYTHON_VERSION) --weak-verify $(COMPILE)
 	$(PYTHON) test_pythonlib.py --bytecode-$(PYTHON_VERSION)-run --verify-run $(COMPILE)
 
+#: Run longer Python 2.6's lib files known to be okay
+check-2.4-ok:
+	$(PYTHON) test_pythonlib.py --ok-2.4 --verify $(COMPILE)
+
 #: Run longer Python 2.6's lib files known to be okay
 check-2.6-ok:
 	$(PYTHON) test_pythonlib.py --ok-2.6 --weak-verify $(COMPILE)