Get ready for release 2.9.10

and some parser list nonterminal cleanup

.gitignore

@@ -1,7 +1,7 @@
+*.pyo
 *.pyc
 *_dis
 *~
-*.pyc
 /.cache
 /.eggs
 /.python-version
@@ -13,5 +13,6 @@
 /nose-*.egg
 /tmp
 /uncompyle6.egg-info
+/unpyc
 __pycache__
 build

.travis.yml

@@ -8,6 +8,7 @@ python:
   - '2.6'
   - '3.3'
   - '3.4'
+  - '3.2'
 
 install:
 - pip install -r requirements.txt

HISTORY.md

@@ -30,7 +30,7 @@ another clever idea: using table-driven semantics routines, using
 format specifiers.
 
 The last mention of a release of SPARK from John is around 2002. As
-released, although the Early Algorithm parser was in good shape, this
+released, although the Earley Algorithm parser was in good shape, this
 code was woefully lacking as serious Python deparser.
 
 In the fall of 2000, Hartmut Goebel
@@ -135,9 +135,9 @@ Hartmut a decade an a half ago:
     NB. This is not a masterpiece of software, but became more like a hack.
     Probably a complete rewrite would be sensefull. hG/2000-12-27
 
-This project deparses using an Early-algorithm parse with lots of
+This project deparses using an Earley-algorithm parse with lots of
 massaging of tokens and the grammar in the scanner
-phase. Early-algorithm parsers are context free and tend to be linear
+phase. Earley-algorithm parsers are context free and tend to be linear
 if the grammar is LR or left recursive.
 
 Another approach that doesn't use grammars is to do something like

MANIFEST.in

@@ -2,10 +2,16 @@ include README.rst
 include ChangeLog
 include HISTORY.md
 include LICENSE
+include Makefile
+include requirements.txt
+include requirements-dev.txt
 include DECOMPYLE-2.4-CHANGELOG.txt
 include __pkginfo__.py
 recursive-include uncompyle6 *.py
 include bin/uncompyle6
 include bin/pydisassemble
+include pytest/Makefile
+include test/Makefile
 recursive-include test *.py *.pyc
 recursive-include pytest *.py
+recursive-include pytest/testdata *

Makefile

@@ -33,7 +33,7 @@ check-2.7 check-3.3 check-3.4: pytest
 
 #: Tests for Python 3.2 and 3.5 - pytest doesn't work here
 # Or rather 3.5 doesn't work not on Travis
-check-3.1 check-3.2 check-3.5 check-3.6:
+check-3.0 check-3.1 check-3.2 check-3.5 check-3.6:
 	$(MAKE) -C test $@
 
 #:Tests for Python 2.6 (doesn't have pytest)

NEWS

@@ -1,3 +1,121 @@
+uncompyle6 2.9.10 2016-02-25
+
+- Python grammar rule fixes
+- Add ability to get grammar coverage on runs
+- Handle Python 3.6 opcode BUILD_CONST_KEYMAP
+
+uncompyle6 2.9.9 2016-12-16
+
+- Remaining Python 3.5 ops handled
+  (this also means more Python 3.6 ops are handled)
+- Python 3.5 and 3.6 async and await handled
+- Python 3.0 decompilation improved
+- Python 3 annotations fixed
+- Better control-flow detection
+- Code cleanups and misc bug fixes
+
+uncompyle6 2.9.8 2016-12-16
+
+- Better control-flow detection
+- pseudo instruction THEN in 2.x
+  to disambiguate if from and
+- fix bug in --verify option
+- DRY (a little) control-flow detection
+- fix syntax in tuples with one element
+- if AST rule inheritence in Python 2.5
+- NAME_MODULE removal for Python <= 2.4
+- verifycall fixes for Python <= 2.4
+- more Python lint
+
+uncompyle6 2.9.7 2016-12-16
+
+- Start to handle 3.5/3.6 build_map_unpack_with_call
+- Some Python 3.6 bytecode to wordcode conversion fixes
+- option -g: show start-end range when possible
+- track print_docstring move to help (used in python 3.1)
+- verify: allow RETURN_VALUE to match RETURN_END_IF
+- some 3.2 compatibility
+- Better Python 3 control flow detection by adding Pseudo ELSE opcodes
+
+uncompyle6 2.9.6 2016-12-04
+
+- Shorten Python3 grammars with + and *
+  this requires spark parser 1.5.1
+- Add some AST reduction checks to improve
+  decompile accuracy. This too requires
+  spark parser 1.5.1
+
+uncompyle6 2.9.6 2016-11-20
+
+- Correct MANIFEST.in
+- More AST grammar checking
+- --linemapping option or linenumbers.line_number_mapping()
+  Shows correspondence of lines between source
+  and decompiled source
+- Some control flow adjustments in code for 2.x.
+  This is probably an improvement in 2.6 and before.
+  For 2.7 things are just shuffled around a little. Sigh.
+  Overall I think we are getting more precise in
+  or analysis even if it is not always reflected
+  in the results.
+- better control flow debugging output
+- Python 2 and 3 detect structure code is more similar
+- Handle Docstrings with embedded tiple quotes (""")
+
+uncompyle6 2.9.5 2016-11-13
+
+- Fix Python 3 bugs:
+  * improprer while 1 else
+  * docstring indent
+  * 3.3 default values in lambda expressions
+  * start 3.0 decompilation (needs newer xdis)
+- Start grammar misparse checking
+
+
+uncompyle6 2.9.4 2016-11-02
+
+- Handle Python 3.x function annotations
+- track def keywoard-parameter line-splitting in source code better
+- bump min xdis version to mask previous xdis bug
+
+uncompyle6 2.9.3 2016-10-26
+
+Release forced by incompatiblity change in xdis 3.2.0.
+
+- Python 3.1 bugs:
+  * handle "with ... as"
+  * handle "with"
+  * Start handling def (...) -> yy (has bugs still)
+
+- DRY Python 3.x via inheritance
+- Python 3.6 work (from Daniel Bradburn)
+  * Handle 3.6 buildstring
+  * Handle 3.6 handle single and multiple fstring better
+
+
+uncompyle6 2.9.2 2016-10-15
+
+- use source-code line breaks to assist in where to break
+  in tuples and maps
+- Fix Python 1.5 decompyle bugs
+- Fix some Python 2.6 and below bugs
+- DRY fragments.py code a little
+
+uncompyle6 2.9.1 2016-10-09
+
+- Improved Python 1.5 decompiling
+- Handle old-style pre Python 2.2 classes
+
+uncompyle6 2.9.0 2016-10-09
+
+- Use xdis 3.0.0 protocol load_module.
+  this Forces change in requirements.txt and _pkg_info_.py
+- Start Python 1.5 decompiling; another round of work is needed to
+  remove bugs
+- Simpify python 2.1 grammar
+- Fix bug with -t ...  Wasn't showing source text when -t option was given
+- Fix 2.1-2.6 bug in list comprehension
+
 uncompyle6 2.8.4 2016-10-08
 
 - Python 3 disassembly bug fixes

README.rst

@@ -1,4 +1,4 @@
-|buildstatus|
+|buildstatus| |Supported Python Versions|
 
 uncompyle6
 ==========
@@ -20,9 +20,9 @@ Why this?
 There were a number of decompyle, uncompile, uncompyle2, uncompyle3
 forks around. All of them came basically from the same code base, and
 almost all of them no were no longer actively maintained. Only one
-handled Python 3, and even there, only 3.2. This code pulls these
-together and moves forward. It also addresses a number of open issues
-in the previous forks.
+handled Python 3, and even there, only 3.2 or 3.3 depending on which
+code is used. This code pulls these together and moves forward. It
+also addresses a number of open issues in the previous forks.
 
 What makes this different from other CPython bytecode decompilers?: its
 ability to deparse just fragments and give source-code information
@@ -44,8 +44,9 @@ Requirements
 ------------
 
 This project requires Python 2.6 or later, PyPy 3-2.4, or PyPy-5.0.1.
+Python versions 2.4-2.7 are supported in the python-2.4 branch.
 The bytecode files it can read has been tested on Python bytecodes from
-versions 2.1-2.7, and 3.2-3.6 and the above-mentioned PyPy versions.
+versions 1.5, 2.1-2.7, and 3.2-3.6 and the above-mentioned PyPy versions.
 
 Installation
 ------------
@@ -92,25 +93,71 @@ For usage help:
 
    $ uncompyle6 -h
 
+If you want strong verification of the correctness of the
+decompilation process, add the `--verify` option. But there are
+situations where this will indicate a failure, although the generated
+program is semantically equivalent. Using option `--weak-verify` will
+tell you if there is something definitely wrong. Generally, large
+swaths of code are decompiled correctly, if not the entire program.
+
+You can also cross compare the results with pycdc_ . Since they work
+differently, bugs here often aren't in that, and vice versa.
+
 
 Known Bugs/Restrictions
 -----------------------
 
-Python 2 deparsing decompiles and about 90% verifies from Python 2.3.7 to Python
-3.4.2 on the standard library packages I have on my system.
+The biggest known and possibly fixable (but hard) problem has to do
+with handling control flow. All of the Python decompilers I have looked
+at have the same problem. In some cases we can detect an erroneous
+decompilation and report that.
 
-(Verification is the process of decompiling bytecode, compiling with a
-Python for that byecode version, and then comparing the byetcode
+About 90% of the decompilation of Python standard library packages in
+Python 2.7.12 verifies correctly. Over 99% of Python 2.7 and 3.3-3.5
+"weakly" verify. Python 2.6 drops down to 96% weakly verifying.
+Other versions drop off in quality too.
+
+*Verification* is the process of decompiling bytecode, compiling with
+a Python for that bytecode version, and then comparing the bytecode
 produced by the decompiled/compiled program. Some allowance is made
-for inessential differences.)
+for inessential differences. But other semantically equivalent
+differences are not caught. For example ``1 and 0`` is decompiled to
+the equivalent ``0``; remnants of the first true evaluation (1) is
+lost when Python compiles this. When Python next compiles ``0`` the
+resulting code is simpler.
 
-Later distributions average about 200 files. At this point, 2.7
-decompilation is better than uncompyle2. A number of bugs have been
-fixed.
+*Weak Verification*
+on the other hand doesn't check bytecode for equivalence but does
+check to see if the resulting decompiled source is a valid Python
+program by running the Python interpreter. Because the Python language
+has changed so much, for best results you should use the same Python
+Version in checking as used in the bytecode.
+
+Later distributions average about 200 files. There is some work to do
+on the lower end Python versions which is more difficult for us to
+handle since we don't have a Python interpreter for versions 1.5, 1.6,
+and 2.0.
+
+In the Python 3 series, Python support is is strongest around 3.4 or
+3.3 and drops off as you move further away from those versions. Python
+3.5 largely works, but still has some bugs in it and is missing some
+opcodes.  Python 3.6 changes things drastically by using word codes
+rather than byte codes. That has been addressed, but then it also
+changes function call opcodes and its semantics and has more problems
+with control flow than 3.5 has.
+
+Currently not all Python magic numbers are supported. Specifically in
+some versions of Python, notably Python 3.6, the magic number has
+changes several times within a version. We support only the released
+magic. There are also customized Python interpreters, notably Dropbox,
+which use their own magic and encrypt bytcode. With the exception of
+the Dropbox's old Python 2.5 interpreter this kind of thing is not
+handled.
+
+We also don't handle PJOrion_ obfuscated code. For that try: PJOrion
+Deobfuscator_ to unscramble the bytecode to get valid bytecode before
+trying this tool.
 
-Python 3.5 largely works, but still has some bugs in it.
-Python 3.6 changes things drastically by using word codes rather than
-byte codes, and that needs to be addressed.
 
 There is lots to do, so please dig in and help.
 
@@ -119,6 +166,7 @@ See Also
 
 * https://github.com/zrax/pycdc : supports all versions of Python and is written in C++
 * https://code.google.com/archive/p/unpyc3/ : supports Python 3.2 only. The above projects use a different decompiling technique what is used here.
+* https://github.com/figment/unpyc3/ : fork of above, but supports Python 3.3 only. Include some fixes like supporting function annotations
 * The HISTORY_ file.
 
 .. |downloads| image:: https://img.shields.io/pypi/dd/uncompyle6.svg
@@ -130,3 +178,7 @@ See Also
 .. _this: https://github.com/rocky/python-uncompyle6/wiki/Deparsing-technology-and-its-use-in-exact-location-reporting
 .. |buildstatus| image:: https://travis-ci.org/rocky/python-uncompyle6.svg
 		 :target: https://travis-ci.org/rocky/python-uncompyle6
+.. |Supported Python Versions| image:: https://img.shields.io/pypi/pyversions/uncompyle6.svg
+   :target: https://pypi.python.org/pypi/uncompyle6/
+.. _PJOrion: http://www.koreanrandom.com/forum/topic/15280-pjorion-%D1%80%D0%B5%D0%B4%D0%B0%D0%BA%D1%82%D0%B8%D1%80%D0%BE%D0%B2%D0%B0%D0%BD%D0%B8%D0%B5-%D0%BA%D0%BE%D0%BC%D0%BF%D0%B8%D0%BB%D1%8F%D1%86%D0%B8%D1%8F-%D0%B4%D0%B5%D0%BA%D0%BE%D0%BC%D0%BF%D0%B8%D0%BB%D1%8F%D1%86%D0%B8%D1%8F-%D0%BE%D0%B1%D1%84
+.. _Deobfuscator: https://github.com/extremecoders-re/PjOrion-Deobfuscator

__pkginfo__.py

@@ -9,17 +9,19 @@
 
 # Things that change more often go here.
 copyright   = """
-Copyright (C) 2015, 2016 Rocky Bernstein <rb@dustyfeet.com>.
+Copyright (C) 2015-2017 Rocky Bernstein <rb@dustyfeet.com>.
 """
 
-classifiers =  ['Development Status :: 4 - Beta',
+classifiers =  ['Development Status :: 5 - Production/Stable',
                 'Intended Audience :: Developers',
                 'Operating System :: OS Independent',
                 'Programming Language :: Python',
-                'Programming Language :: Python :: 2',
+                'Programming Language :: Python :: 2.4',
+                'Programming Language :: Python :: 2.5',
                 'Programming Language :: Python :: 2.6',
                 'Programming Language :: Python :: 2.7',
-                'Programming Language :: Python :: 3',
+                'Programming Language :: Python :: 3.1',
+                'Programming Language :: Python :: 3.2',
                 'Programming Language :: Python :: 3.3',
                 'Programming Language :: Python :: 3.4',
                 'Programming Language :: Python :: 3.5',
@@ -37,8 +39,8 @@ entry_points={
         'pydisassemble=uncompyle6.bin.pydisassemble:main',
     ]}
 ftp_url            = None
-install_requires   = ['spark-parser >= 1.4.0',
-                      'xdis >= 2.3.1']
+install_requires   = ['spark-parser >= 1.6.0, < 1.7.0',
+                      'xdis >= 3.2.4, < 3.3.0']
 license            = 'MIT'
 mailing_list       = 'python-debugger@googlegroups.com'
 modname            = 'uncompyle6'

circle.yml

@@ -1,6 +1,6 @@
 machine:
     python:
-        version: 2.7.8
+        version: 2.7.10
     environment:
         COMPILE: --compile
 

pytest/.gitignore

@@ -1 +1,2 @@
+/.hypothesis
 /__pycache__

pytest/test_build_const_key_map.py

@@ -0,0 +1,21 @@
+import pytest
+# uncompyle6
+from uncompyle6 import PYTHON_VERSION
+from validate import validate_uncompyle
+
+
+@pytest.mark.skipif(PYTHON_VERSION < 3.6, reason='need at least python 3.6')
+@pytest.mark.parametrize('text', (
+    "{0.: 'a', -1: 'b'}",   # BUILD_MAP
+    "{'a':'b'}",            # BUILD_MAP
+    "{0: 1}",               # BUILD_MAP
+    "{b'0':1, b'2':3}",     # BUILD_CONST_KEY_MAP
+    "{0: 1, 2: 3}",         # BUILD_CONST_KEY_MAP
+    "{'a':'b','c':'d'}",    # BUILD_CONST_KEY_MAP
+    "{0: 1, 2: 3}",         # BUILD_CONST_KEY_MAP
+    "{'a': 1, 'b': 2}",     # BUILD_CONST_KEY_MAP
+    "{'a':'b','c':'d'}",    # BUILD_CONST_KEY_MAP
+    "{0.0:'b',0.1:'d'}",    # BUILD_CONST_KEY_MAP
+))
+def test_build_const_key_map(text):
+    validate_uncompyle(text)

pytest/test_docstring.py

@@ -0,0 +1,78 @@
+import sys
+from uncompyle6 import PYTHON3
+if PYTHON3:
+    from io import StringIO
+    minint = -sys.maxsize-1
+    maxint = sys.maxsize
+else:
+    from StringIO import StringIO
+    minint = -sys.maxint-1
+    maxint = sys.maxint
+from uncompyle6.semantics.helper import print_docstring
+
+class PrintFake():
+    def __init__(self):
+        self.pending_newlines = 0
+        self.f = StringIO()
+
+    def write(self, *data):
+        if (len(data) == 0) or (len(data) == 1 and data[0] == ''):
+            return
+        out = ''.join((str(j) for j in data))
+        n = 0
+        for i in out:
+            if i == '\n':
+                n += 1
+                if n == len(out):
+                    self.pending_newlines = max(self.pending_newlines, n)
+                    return
+            elif n:
+                self.pending_newlines = max(self.pending_newlines, n)
+                out = out[n:]
+                break
+            else:
+                break
+
+        if self.pending_newlines > 0:
+            self.f.write('\n'*self.pending_newlines)
+            self.pending_newlines = 0
+
+        for i in out[::-1]:
+            if i == '\n':
+                self.pending_newlines += 1
+            else:
+                break
+
+        if self.pending_newlines:
+            out = out[:-self.pending_newlines]
+        self.f.write(out)
+    def println(self, *data):
+        if data and not(len(data) == 1 and data[0] ==''):
+            self.write(*data)
+        self.pending_newlines = max(self.pending_newlines, 1)
+        return
+    pass
+
+def test_docstring():
+
+    for doc, expect in (
+            ("Now is the time",
+             '  """Now is the time"""'),
+           ("""
+Now is the time
+""",
+            '''  """
+  Now is the time
+  """''')
+
+            # (r'''func placeholder - ' and with ("""\nstring\n  """)''',
+            #  """  r'''func placeholder - ' and with (\"\"\"\nstring\n\"\"\")'''"""),
+            # (r"""func placeholder - ' and with ('''\nstring\n''') and \"\"\"\nstring\n\"\"\" """,
+            # """  r\"\"\"func placeholder - ' and with ('''\nstring\n''') and \"\"\"\nstring\n\"\"\" \"\"\"""")
+            ):
+
+        o = PrintFake()
+        # print(doc)
+        # print(expect)
+        print_docstring(o, '  ', doc)
+        assert expect == o.f.getvalue()

pytest/test_fjt.py

@@ -8,6 +8,18 @@ def bug(state, slotstate):
             for key, value in slotstate.items():
                 setattr(state, key, 2)
 
+# From 2.7 disassemble
+# Problem is not getting while, because
+# COME_FROM not added
+def bug_loop(disassemble, tb=None):
+    if tb:
+        try:
+            tb = 5
+        except AttributeError:
+            raise RuntimeError
+        while tb: tb = tb.tb_next
+    disassemble(tb)
+
 def test_if_in_for():
     code = bug.__code__
     scan = get_scanner(PYTHON_VERSION)
@@ -16,18 +28,35 @@ def test_if_in_for():
         n = scan.setup_code(code)
         scan.build_lines_data(code, n)
         scan.build_prev_op(n)
-        fjt = scan.find_jump_targets()
+        fjt = scan.find_jump_targets(False)
         assert {15: [3], 69: [66], 63: [18]} == fjt
         assert scan.structs == \
           [{'start': 0, 'end': 72, 'type': 'root'},
-           {'start': 18, 'end': 66, 'type': 'if-then'},
+           {'start': 15, 'end': 66, 'type': 'if-then'},
            {'start': 31, 'end': 59, 'type': 'for-loop'},
            {'start': 62, 'end': 63, 'type': 'for-else'}]
+
+        code = bug_loop.__code__
+        n = scan.setup_code(code)
+        scan.build_lines_data(code, n)
+        scan.build_prev_op(n)
+        fjt = scan.find_jump_targets(False)
+        assert{64: [42], 67: [42, 42], 42: [16, 41], 19: [6]} == fjt
+        assert scan.structs == [
+            {'start': 0, 'end': 80, 'type': 'root'},
+            {'start': 3, 'end': 64, 'type': 'if-then'},
+            {'start': 6, 'end': 15, 'type': 'try'},
+            {'start': 19, 'end': 38, 'type': 'except'},
+            {'start': 45, 'end': 67, 'type': 'while-loop'},
+            {'start': 70, 'end': 64, 'type': 'while-else'},
+            # previous bug was not mistaking while-loop for if-then
+            {'start': 48, 'end': 67, 'type': 'while-loop'}]
+
     elif 3.2 < PYTHON_VERSION <= 3.4:
         scan.code = array('B', code.co_code)
         scan.build_lines_data(code)
         scan.build_prev_op()
-        fjt  = scan.find_jump_targets()
+        fjt  = scan.find_jump_targets(False)
         assert {69: [66], 63: [18]} == fjt
         assert scan.structs == \
           [{'end': 72, 'type': 'root', 'start': 0},

pytest/test_fstring.py

@@ -21,9 +21,13 @@ def expressions(draw):
         'container',
         'self.attribute',
         'self.method()',
-        'sorted(items, key=lambda x: x.name)',
-        'func(*args, **kwargs)',
-        'text or default',
+        # These expressions are failing, I think these are control
+        # flow problems rather than problems with FORMAT_VALUE,
+        # however I need to confirm this...
+        #'sorted(items, key=lambda x: x.name)',
+        #'func(*args, **kwargs)',
+        #'text or default',
+        #'43 if life_the_universe and everything else None'
     )))
 
 
@@ -119,6 +123,8 @@ def test_format_specifiers(format_specifier):
 
 
 def run_test(text):
+    hypothesis.assume(len(text))
+    hypothesis.assume("f'{" in text)
     expr = text + '\n'
     code = compile(expr, '<string>', 'single')
     deparsed = deparse_code(PYTHON_VERSION, code, compile_mode='single')
@@ -136,9 +142,9 @@ def test_uncompyle_fstring(fstring):
 
 @pytest.mark.skipif(PYTHON_VERSION < 3.6, reason='need at least python 3.6')
 @pytest.mark.parametrize('fstring', [
-    #"f'{abc}{abc!s}'",
-    "f'{abc!s}'",
+    "f'{abc}{abc!s}'",
+    "f'{abc}0'",
 ])
 def test_uncompyle_direct(fstring):
     """useful for debugging"""
-    run_test(fstring)
+    run_test(fstring)

pytest/test_grammar.py

@@ -1,6 +1,6 @@
-import pytest, re
+import re
 from uncompyle6 import PYTHON_VERSION, PYTHON3, IS_PYPY # , PYTHON_VERSION
-from uncompyle6.parser import get_python_parser
+from uncompyle6.parser import get_python_parser, python_parser
 from uncompyle6.scanner import get_scanner
 
 def test_grammar():
@@ -16,14 +16,21 @@ def test_grammar():
     p = get_python_parser(PYTHON_VERSION, is_pypy=IS_PYPY)
     lhs, rhs, tokens, right_recursive = p.checkSets()
     expect_lhs = set(['expr1024', 'pos_arg'])
-    unused_rhs = set(['build_list', 'call_function', 'mkfunc', 'mklambda',
+    unused_rhs = set(['build_list', 'call_function', 'mkfunc',
+                      'mklambda',
                       'unpack', 'unpack_list'])
     expect_right_recursive = [['designList', ('designator', 'DUP_TOP', 'designList')]]
     if PYTHON3:
         expect_lhs.add('load_genexpr')
+
         unused_rhs = unused_rhs.union(set("""
         except_pop_except genexpr classdefdeco2 listcomp
         """.split()))
+        if 3.0 <= PYTHON_VERSION:
+            expect_lhs.add("annotate_arg")
+            expect_lhs.add("annotate_tuple")
+            unused_rhs.add("mkfunc_annotate")
+            pass
     else:
         expect_lhs.add('kwarg')
     assert expect_lhs == set(lhs)
@@ -34,7 +41,7 @@ def test_grammar():
             """
             JUMP_BACK CONTINUE RETURN_END_IF
             COME_FROM COME_FROM_EXCEPT COME_FROM_LOOP COME_FROM_WITH
-            COME_FROM_FINALLY
+            COME_FROM_FINALLY ELSE
             LOAD_GENEXPR LOAD_ASSERT LOAD_SETCOMP LOAD_DICTCOMP
             LAMBDA_MARKER RETURN_LAST
             """.split())
@@ -43,5 +50,14 @@ def test_grammar():
         check_tokens(tokens, opcode_set)
     elif PYTHON_VERSION == 3.4:
         ignore_set.add('LOAD_CLASSNAME')
+        ignore_set.add('STORE_LOCALS')
         opcode_set = set(s.opc.opname).union(ignore_set)
         check_tokens(tokens, opcode_set)
+
+def test_dup_rule():
+    import inspect
+    python_parser(PYTHON_VERSION, inspect.currentframe().f_code,
+                  is_pypy=IS_PYPY,
+                  parser_debug={
+                      'dups': True, 'transition': False, 'reduce': False,
+                      'rules': False, 'errorstack': None, 'context': True})

pytest/validate.py

@@ -0,0 +1,143 @@
+# future
+from __future__ import print_function
+# std
+import os
+import dis
+import difflib
+import subprocess
+import tempfile
+# compatability
+import six
+# uncompyle6 / xdis
+from uncompyle6 import PYTHON_VERSION, deparse_code
+
+
+def _dis_to_text(co):
+    return dis.Bytecode(co).dis()
+
+
+def print_diff(original, uncompyled):
+    """
+    Try and display a pretty html line difference between the original and
+    uncompyled code and bytecode if elinks and BeautifulSoup are installed
+    otherwise just show the diff.
+
+    :param original: Text describing the original code object.
+    :param uncompyled: Text describing the uncompyled code object.
+    """
+    original_lines = original.split('\n')
+    uncompyled_lines = uncompyled.split('\n')
+    args = original_lines, uncompyled_lines, 'original', 'uncompyled'
+    try:
+        from bs4 import BeautifulSoup
+        diff = difflib.HtmlDiff().make_file(*args)
+        diff = BeautifulSoup(diff, "html.parser")
+        diff.select_one('table[summary="Legends"]').extract()
+    except ImportError:
+        print('\nTo display diff highlighting run:\n    pip install BeautifulSoup4')
+        diff = difflib.HtmlDiff().make_table(*args)
+
+    with tempfile.NamedTemporaryFile(delete=False) as f:
+        f.write(str(diff).encode('utf-8'))
+
+    try:
+        print()
+        html = subprocess.check_output([
+            'elinks',
+            '-dump',
+            '-no-references',
+            '-dump-color-mode',
+            '1',
+            f.name,
+        ]).decode('utf-8')
+        print(html)
+    except:
+        print('\nFor side by side diff install elinks')
+        diff = difflib.Differ().compare(original_lines, uncompyled_lines)
+        print('\n'.join(diff))
+    finally:
+        os.unlink(f.name)
+
+
+def are_instructions_equal(i1, i2):
+    """
+    Determine if two instructions are approximately equal,
+    ignoring certain fields which we allow to differ, namely:
+
+    * code objects are ignore (should probaby be checked) due to address
+    * line numbers
+
+    :param i1: left instruction to compare
+    :param i2: right instruction to compare
+
+    :return: True if the two instructions are approximately equal, otherwise False.
+    """
+    result = (1==1
+        and i1.opname == i2.opname
+        and i1.opcode == i2.opcode
+        and i1.arg == i2.arg
+        # ignore differences due to code objects
+        # TODO : Better way of ignoring address
+        and (i1.argval == i2.argval or '<code object' in str(i1.argval))
+        # TODO : Should probably recurse to check code objects
+        and (i1.argrepr == i2.argrepr or '<code object' in i1.argrepr)
+        and i1.offset == i2.offset
+        # ignore differences in line numbers
+        #and i1.starts_line
+        and i1.is_jump_target == i2.is_jump_target
+    )
+    return result
+
+
+def are_code_objects_equal(co1, co2):
+    """
+    Determine if two code objects are approximately equal,
+    see are_instructions_equal for more information.
+
+    :param i1: left code object to compare
+    :param i2: right code object to compare
+
+    :return: True if the two code objects are approximately equal, otherwise False.
+    """
+    # TODO : Use xdis for python2 compatability
+    instructions1 = dis.Bytecode(co1)
+    instructions2 = dis.Bytecode(co2)
+    for opcode1, opcode2 in zip(instructions1, instructions2):
+        if not are_instructions_equal(opcode1, opcode2):
+            return False
+    return True
+
+
+def validate_uncompyle(text, mode='exec'):
+    """
+    Validate decompilation of the given source code.
+
+    :param text: Source to validate decompilation of.
+    """
+    original_code = compile(text, '<string>', mode)
+    original_dis = _dis_to_text(original_code)
+    original_text = text
+
+    deparsed = deparse_code(PYTHON_VERSION, original_code,
+                            compile_mode=mode, out=six.StringIO())
+    uncompyled_text = deparsed.text
+    uncompyled_code = compile(uncompyled_text, '<string>', 'exec')
+
+    if not are_code_objects_equal(uncompyled_code, original_code):
+
+        uncompyled_dis = _dis_to_text(uncompyled_text)
+
+        def output(text, dis):
+            width = 60
+            return '\n\n'.join([
+                ' SOURCE CODE '.center(width, '#'),
+                text.strip(),
+                ' BYTECODE '.center(width, '#'),
+                dis
+            ])
+
+        original = output(original_text, original_dis)
+        uncompyled = output(uncompyled_text, uncompyled_dis)
+        print_diff(original, uncompyled)
+
+        assert 'original' == 'uncompyled'

requirements-dev.txt

@@ -1,3 +1,4 @@
 pytest
 flake8
-hypothesis
+hypothesis
+six

requirements.txt

@@ -1,2 +1,2 @@
-spark-parser >= 1.4.0
-xdis >= 2.2.2
+# Pick up stuff from setup.py
+-e .

setup.py

@@ -24,6 +24,6 @@ setup(
        py_modules         = py_modules,
        test_suite         = 'nose.collector',
        url                = web,
-       setup_requires     = ['nose>=1.0'],
+       tests_require     = ['nose>=1.0'],
        version            = VERSION,
        zip_safe           = zip_safe)

test/Makefile

@@ -20,7 +20,11 @@ check:
 	$(MAKE) check-$$PYTHON_VERSION
 
 #: Run working tests from Python 2.6 or 2.7
-check-2.6 check-2.7: check-bytecode-2 check-bytecode-3  check-2.7-ok
+check-2.6 check-2.7: check-bytecode-2 check-bytecode-3 check-bytecode-1  check-2.7-ok
+
+#: Run working tests from Python 3.0
+check-3.0: check-bytecode
+	$(PYTHON) test_pythonlib.py --bytecode-3.0 --weak-verify $(COMPILE)
 
 #: Run working tests from Python 3.1
 check-3.1: check-bytecode
@@ -32,11 +36,11 @@ check-3.2: check-bytecode
 
 #: Run working tests from Python 3.3
 check-3.3: check-bytecode
-	$(PYTHON) test_pythonlib.py --bytecode-3.3 --weak-verify $(COMPILE)
+	$(PYTHON) test_pythonlib.py --bytecode-3.3 --verify $(COMPILE)
 
 #: Run working tests from Python 3.4
 check-3.4: check-bytecode check-3.4-ok check-2.7-ok
-	$(PYTHON) test_pythonlib.py --bytecode-3.4 --weak-verify $(COMPILE)
+	$(PYTHON) test_pythonlib.py --bytecode-3.4 --verify $(COMPILE)
 
 #: Run working tests from Python 3.5
 check-3.5: check-bytecode
@@ -50,6 +54,10 @@ check-3.6: check-bytecode
 check-disasm:
 	$(PYTHON) dis-compare.py
 
+#: Check deparsing bytecode 1.x only
+check-bytecode-1:
+	$(PYTHON) test_pythonlib.py --bytecode-1.5
+
 #: Check deparsing bytecode 2.x only
 check-bytecode-2:
 	$(PYTHON) test_pythonlib.py \
@@ -58,14 +66,17 @@ check-bytecode-2:
 
 #: Check deparsing bytecode 3.x only
 check-bytecode-3:
-	$(PYTHON) test_pythonlib.py --bytecode-3.2 --bytecode-3.3 \
+	$(PYTHON) test_pythonlib.py --bytecode-3.0 \
+       --bytecode-3.1 --bytecode-3.2 --bytecode-3.3 \
        --bytecode-3.4 --bytecode-3.5 --bytecode-pypy3.2
 
 #: Check deparsing bytecode that works running Python 2 and Python 3
 check-bytecode: check-bytecode-3
 	$(PYTHON) test_pythonlib.py \
 	--bytecode-2.1 --bytecode-2.2 --bytecode-2.3 --bytecode-2.4 \
-        --bytecode-2.5 --bytecode-2.6 --bytecode-2.7 --bytecode-pypy2.7
+        --bytecode-2.5 --bytecode-2.6 --bytecode-2.7 \
+        --bytecode-pypy2.7 --bytecode-1
+
 
 #: Check deparsing Python 2.1
 check-bytecode-2.1:
@@ -87,13 +98,32 @@ check-bytecode-2.4:
 check-bytecode-2.5:
 	$(PYTHON) test_pythonlib.py --bytecode-2.5
 
+#: Get grammar coverage for Python 2.5
+grammar-coverage-2.5:
+	SPARK_PARSER_COVERAGE=/tmp/spark-grammar-25.cover $(PYTHON) test_pythonlib.py --bytecode-2.5
+	SPARK_PARSER_COVERAGE=/tmp/spark-grammar-25.cover $(PYTHON) test_pyenvlib.py --2.5.6
+
+#: Get grammar coverage for Python 2.6
+grammar-coverage-2.6:
+	SPARK_PARSER_COVERAGE=/tmp/spark-grammar-26.cover $(PYTHON) test_pythonlib.py --bytecode-2.6
+	SPARK_PARSER_COVERAGE=/tmp/spark-grammar-26.cover $(PYTHON) test_pyenvlib.py --2.6.9
+
+#: Get grammar coverage for Python 2.7
+grammar-coverage-2.7:
+	SPARK_PARSER_COVERAGE=/tmp/spark-grammar-27.cover $(PYTHON) test_pythonlib.py --bytecode-2.7
+	SPARK_PARSER_COVERAGE=/tmp/spark-grammar-27.cover $(PYTHON) test_pyenvlib.py --2.7.13
+
 #: Check deparsing Python 2.6
 check-bytecode-2.6:
-	$(PYTHON) test_pythonlib.py --bytecode-2.6
+	$(PYTHON) test_pythonlib.py --bytecode-2.6 --weak-verify
 
 #: Check deparsing Python 2.7
 check-bytecode-2.7:
-	$(PYTHON) test_pythonlib.py --bytecode-2.7
+	$(PYTHON) test_pythonlib.py --bytecode-2.7 --verify
+
+#: Check deparsing Python 3.0
+check-bytecode-3.0:
+	$(PYTHON) test_pythonlib.py --bytecode-3.0
 
 #: Check deparsing Python 3.1
 check-bytecode-3.1: