Merge branch 'master' into python-2.4

.gitignore

@@ -1,11 +1,15 @@
-*.pyo
 *.pyc
+*.pyo
 *_dis
 *~
 /.cache
 /.eggs
+/.hypothesis
+/.idea
+/.pytest_cache
 /.python-version
 /.tox
+/.venv*
 /README
 /__pkginfo__.pyc
 /dist
@@ -14,9 +18,6 @@
 /tmp
 /uncompyle6.egg-info
 /unpyc
+ChangeLog
 __pycache__
 build
-/.venv*
-/.idea
-/.hypothesis
-ChangeLog

NEWS

@@ -1,3 +1,13 @@
+uncompyle6 3.0.1 2018-02-17
+
+- All Python 2.6.9 standard library files weakly verify
+- Many 3.6 fixes. 84% of the first 200 standard library files weakly compile.
+  One more big push is needed to get the remaining to compile
+- Many decompilation fixes for other Python versions
+- Add more to the test framework
+- And more add tests target previous existing bugs more completely
+- sync recent license changes in metadata
+
 uncompyle6 3.0.0 2018-02-17
 
 - deparse_code() and lookalikes from the various semantic actions are
@@ -10,7 +20,7 @@ uncompyle6 3.0.0 2018-02-17
   of jumps, and messes up the peephole-like analysis that is
   done for control flow since we don't have something better in place.
 - Code has been reorganized to be more instruction nametuple based where it
-  has been more byecode array based. There was and still is code that had
+  has been more bytecode array based. There was and still is code that had
   had magic numbers to advance instructions or to pick out operands.
 - Bug fixes in numerous other Python versions
 - Instruction display improved
@@ -28,7 +38,7 @@ uncompyle6 2.16.0 2018-02-17
 - Better 2.7 end_if and COME_FROM determination
 - Fix up 3.6+ CALL_FUNCTION_EX
 - Misc pydisasm fixes
-- Wierd comprehension bug seen via new loctraceback
+- Weird comprehension bug seen via new loctraceback
 - Fix Python 3.5+ CALL_FUNCTION_VAR and BUILD_LIST_UNPACK in call; with this
   we can can handle 3.5+ f(a, b, *c, *d, *e) now
 
@@ -81,7 +91,7 @@ Decompilation bug fixes, mostly 3.6 and pre 2.7
 - limit pypy customization to pypy
 - Add addr fields in COME_FROMS
 - Allow use of full instructions in parser reduction routines
-- Reduce grammar in Pythion 3 by specialization more to specific
+- Reduce grammar in Python 3 by specialization more to specific
   Python versions
 - Match Python AST names more closely when possible
 
@@ -99,7 +109,7 @@ uncompyle6 2.14.0 2017-11-26 johnnybamazing
   and remove used grammar rules
 - Fix a number of bytecode decompile problems
   (many more remain)
-- Add stdlib/runtests.sh for even more rigourous testing
+- Add stdlib/runtests.sh for even more rigorous testing
 
 uncompyle6 2.13.3 2017-11-13
 
@@ -123,7 +133,7 @@ Overall: better 3.6 decompiling and some much needed code refactoring and cleanu
   added to assist here. Ignoring errors may be okay because the fragment parser often just needs,
   well, *fragments*.
 - Distinguish RETURN_VALUE from RETURN_END_IF in exception bodies better in 3.6
-- bug in 3.x language changes: import queue va import Queue
+- bug in 3.x language changes: import queue via import Queue
 - reinstate some bytecode tests since decompiling has gotten better
 - Revise how to report a bug
 
@@ -158,8 +168,8 @@ uncompyle6 2.11.4 2017-08-15
 
 * scanner and parser now allow 3-part version string lookups,
   e.g. 2.7.1 We allow a float here, but if passed a string like '2.7'. or
-* unpin 3.5.1. xdis 3.5.4 has been releasd and fixes the problems we had. Use that.
-* some routnes here moved to xdis. Use the xdis version
+* unpin 3.5.1. xdis 3.5.4 has been release and fixes the problems we had. Use that.
+* some routines here moved to xdis. Use the xdis version
 * README.rst: Link typo Name is trepan2 now not trepan
 * xdis-forced change adjust for COMPARE_OP "is-not" in
   semanatic routines. We need "is not".
@@ -255,9 +265,9 @@ uncompyle6 2.9.8 2016-12-16
 - fix bug in --verify option
 - DRY (a little) control-flow detection
 - fix syntax in tuples with one element
-- if AST rule inheritence in Python 2.5
+- if AST rule inheritance in Python 2.5
 - NAME_MODULE removal for Python <= 2.4
-- verifycall fixes for Python <= 2.4
+- verify call fixes for Python <= 2.4
 - more Python lint
 
 uncompyle6 2.9.7 2016-12-16
@@ -293,7 +303,7 @@ uncompyle6 2.9.6 2016-11-20
   in the results.
 - better control flow debugging output
 - Python 2 and 3 detect structure code is more similar
-- Handle Docstrings with embedded tiple quotes (""")
+- Handle Docstrings with embedded triple quotes (""")
 
 uncompyle6 2.9.5 2016-11-13
 

__pkginfo__.py

@@ -56,7 +56,7 @@ entry_points       = {
     ]}
 ftp_url            = None
 install_requires   = ['spark-parser >= 1.8.5, < 1.9.0',
-                      'xdis >= 3.6.9, < 3.7.0']
+                      'xdis >= 3.7.0, < 3.8.0']
 license            = 'GPL3'
 mailing_list       = 'python-debugger@googlegroups.com'
 modname            = 'uncompyle6'

test/run-and-email.sh

@@ -0,0 +1,13 @@
+#!/usr/bin/bash
+EMAIL=${EMAIL:-rb@dustyfeet.com}
+for VERSION in 2.7.14 2.6.9 ; do
+    LOGFILE=/tmp/pyenlib-$VERSION-$$.log
+    python ./test_pyenvlib.py --max 800 --weak-verify --$VERSION  >$LOGFILE 2>&1
+    rc=$?
+    if ((rc == 0)); then
+	tail -v $LOGFILE | mail -s \""$VERSION ok"\" rocky@localhost
+    else
+	tail -v $LOGFILE | mail -s \""$VERSION not ok"\" rocky@localhost
+	tail -v $LOGFILE | mail -s \""$VERSION not ok"\" rb@dustyfeet.com
+    fi
+done

test/simple_source/bug26/02_while1_if_and.py

@@ -6,3 +6,14 @@ def getmultiline(line):
             if line[2] and line[5]:
                 break
     return
+
+# From 2.6.9 refactor.py
+def _detect_future_features(tp):
+    while True:
+        if tp == 6:
+            while tp == 7:
+                if tp != 11:
+                    break
+        else:
+            break
+    return

test/simple_source/bug26/03_tryelse_continue.py

@@ -21,5 +21,33 @@ def call(*args):
     except KeyError:
         return 2
     except TypeError:
-        # Unhashable argument
         return 3
+
+
+# From 2.6.9 pdb.py
+# Here we have a "try/except" inside a "try/except/else and we can't
+# distinguish which COME_FROM comes from which "try".
+
+def do_jump(self, arg):
+    try:
+        arg(1)
+    except ValueError:
+        arg(2)
+    else:
+        try:
+            arg(3)
+        except ValueError:
+            arg(4)
+
+# From 2.6.9 smtpd.py
+# Bug was that the for can cause multiple COME_FROMs at the
+# of the try block
+def _deliver(self, s, mailfrom, rcpttos):
+    try:
+        mailfrom(1)
+    except RuntimeError:
+        mailfrom(2)
+    except IndexError:
+        for r in s:
+            mailfrom()
+    return

test/simple_source/stmts/02_while1else.py

@@ -6,3 +6,16 @@ def PipeClient(address):
     else:
         raise
     return
+
+# From 2.6.9 sre.py
+# Bug was parsing inner while1. Our massaging adds a COME_FROM
+# possibly in the wrong place. When control flow is
+# redone possibly all of this mess will disappear.
+def _parse(source, state, this, group, char):
+    while 1:
+        if this:
+            while 1:
+                raise RuntimeError
+        else:
+            raise IndexError
+    return

test/stdlib/run-and-email.sh

@@ -0,0 +1,13 @@
+#!/usr/bin/bash
+EMAIL=${EMAIL:-rb@dustyfeet.com}
+for VERSION in 2.7.14 2.6.9 ; do
+    LOGFILE=/tmp/runtests-$VERSION-$$.log
+    ./runtests.sh  >$LOGFILE 2>&1
+    rc=$?
+    if ((rc == 0)); then
+	tail -v $LOGFILE | mail -s \""$VERSION ok"\" rocky@localhost
+    else
+	tail -v $LOGFILE | mail -s \""$VERSION not ok"\" rocky@localhost
+	tail -v $LOGFILE | mail -s \""$VERSION not ok"\" rb@dustyfeet.com
+    fi
+done

test/stdlib/runtests.sh

@@ -52,7 +52,11 @@ case $PYVERSION in
 	    [test_opcodes.py]=1
 	    [test_pwd.py]=1 # Long test - might work? Control flow?
 	    [test_re.py]=1 # Probably Control flow?
+	    [test_queue.py]=1 # Control flow?
+	    [test_strftime.py]=1
 	    [test_trace.py]=1  # Line numbers are expected to be different
+	    [test_zipfile64.py]=1  # Skip Long test
+	    [test_zlib.py]=1  # Look at
 	    # .pyenv/versions/2.6.9/lib/python2.6/lib2to3/refactor.pyc
 	    # .pyenv/versions/2.6.9/lib/python2.6/pyclbr.pyc
 	    # .pyenv/versions/2.6.9/lib/python2.6/quopri.pyc -- look at ishex, is short
@@ -65,6 +69,11 @@ case $PYVERSION in
 	;;
     2.7)
 	SKIP_TESTS=(
+	    # These are ok, but our test machine POWER has problems
+	    # so we skip..
+	    [test_httplib.py]=1  # Ok, but POWER has problems with this
+	    [test_pdb.py]=1 # Ok, but POWER has problems with this
+
 	    [test_curses.py]=1  # Possibly fails on its own but not detected
 	    [test_dis.py]=1   # We change line numbers - duh!
 	    [test_doctest.py]=1 # Fails on its own

test/test_pyenvlib.py

@@ -116,8 +116,10 @@ def do_tests(src_dir, patterns, target_dir, start_with=None,
             files = files[:max_files]
 
     print(time.ctime())
-    main.main(src_dir, target_dir, files, [], do_verify=do_verify)
+    (tot_files, okay_files, failed_files,
+     verify_failed_files) = main.main(src_dir, target_dir, files, [], do_verify=do_verify)
     print(time.ctime())
+    return verify_failed_files + failed_files
 
 if __name__ == '__main__':
     import getopt, sys
@@ -161,15 +163,19 @@ if __name__ == '__main__':
             '/tmp/spark-grammar-%s.cover' % vers
             )
 
+    failed = 0
     for src_dir, pattern, target_dir in test_dirs:
         if os.path.exists(src_dir):
             target_dir = os.path.join(target_base, target_dir)
             if os.path.exists(target_dir):
                 shutil.rmtree(target_dir, ignore_errors=1)
-            do_tests(src_dir, pattern, target_dir, start_with,
-                     do_verify, test_options['max='])
+            failed += do_tests(src_dir, pattern, target_dir, start_with,
+                               do_verify, test_options['max='])
         else:
             print("### Path %s doesn't exist; skipping" % src_dir)
+            pass
+        pass
+    sys.exit(failed)
 
 # python 1.5:
 

uncompyle6/parsers/parse26.py

@@ -22,7 +22,7 @@ class Python26Parser(Python2Parser):
                            JUMP_IF_FALSE POP_TOP POP_TOP store POP_TOP
 
         except_handler ::= JUMP_FORWARD COME_FROM except_stmts
-                           come_from_pop END_FINALLY come_froms
+                           come_froms_pop END_FINALLY come_froms
 
         except_handler ::= JUMP_FORWARD COME_FROM except_stmts END_FINALLY
                            come_froms
@@ -77,8 +77,7 @@ class Python26Parser(Python2Parser):
         jb_cont      ::= JUMP_BACK
         jb_cont      ::= CONTINUE
 
-        jb_cf_pop ::= JUMP_BACK come_froms POP_TOP
-        jb_cf_pop ::= JUMP_BACK POP_TOP
+        jb_cf_pop ::= come_from_opt JUMP_BACK _come_froms POP_TOP
         ja_cf_pop ::= JUMP_ABSOLUTE come_froms POP_TOP
         jf_cf_pop ::= JUMP_FORWARD come_froms POP_TOP
 
@@ -134,7 +133,7 @@ class Python26Parser(Python2Parser):
         setup_finally ::= STORE_FAST SETUP_FINALLY LOAD_FAST DELETE_FAST
         setup_finally ::= STORE_NAME SETUP_FINALLY LOAD_NAME DELETE_NAME
 
-        while1stmt     ::= SETUP_LOOP l_stmts_opt JUMP_BACK _come_froms
+        while1stmt     ::= SETUP_LOOP l_stmts_opt come_from_opt JUMP_BACK _come_froms
 
         # Sometimes JUMP_BACK is misclassified as CONTINUE.
         # workaround until we have better control flow in place
@@ -316,7 +315,7 @@ class Python26Parser(Python2Parser):
         self.check_reduce['and'] = 'AST'
         self.check_reduce['list_for'] = 'AST'
         self.check_reduce['try_except'] = 'tokens'
-        self.check_reduce['tryelsestmt'] = 'tokens'
+        self.check_reduce['tryelsestmt'] = 'AST'
 
     def reduce_is_invalid(self, rule, ast, tokens, first, last):
         invalid = super(Python26Parser,
@@ -366,29 +365,49 @@ class Python26Parser(Python2Parser):
                 return (tokens[last-3].kind not in frozenset(('JUMP_FORWARD', 'RETURN_VALUE'))
                         or (tokens[last-3] == 'JUMP_FORWARD' and tokens[last-3].attr != 2))
         elif rule[0] == 'tryelsestmt':
+
             # We need to distingush try_except from tryelsestmt and we do that
-            # by checking the jump before the END_FINALLY
-            # If we have:
-            #    insn
-            #    POP_TOP
-            #    END_FINALLY
-            #    COME_FROM
-            # then insn is neither a JUMP_FORWARD nor RETURN_VALUE,
-            # or if it is JUMP_FORWARD, then it can't be a JUMP_FORWARD to right after
-            # COME_FROM
-            if last == len(tokens):
-                last -= 1
-            while tokens[last-1] == 'COME_FROM' and tokens[last-2] == 'COME_FROM':
-                last -= 1
-            if tokens[last] == 'COME_FROM' and tokens[last-1] == 'COME_FROM':
-                last -= 1
-            if (tokens[last] == 'COME_FROM'
-                and tokens[last-1] == 'END_FINALLY'
-                    and tokens[last-2] == 'POP_TOP'):
-                # A jump of 2 is a jump around POP_TOP, END_FINALLY which
-                # would indicate try/else rather than try
-                return (tokens[last-3].kind in frozenset(('JUMP_FORWARD', 'RETURN_VALUE'))
-                        and (tokens[last-3] != 'JUMP_FORWARD' or tokens[last-3].attr == 2))
+            # by making sure that the jump before the except handler jumps to
+            # code somewhere before the end of the construct.
+            # This AST method is slower, but more correct than what we had
+            # which is given after this.
+
+            if ast[3] == 'except_handler':
+                except_handler = ast[3]
+                if except_handler[0] == 'JUMP_FORWARD':
+                    else_start = int(except_handler[0].pattr)
+                    if last == len(tokens):
+                        last -= 1
+                    if tokens[last] == 'COME_FROM' and isinstance:
+                        last_offset = int(tokens[last].offset.split('_')[0])
+                        return else_start >= last_offset
+
+            # Before we resorted to using the AST here is the more hacky way we
+            # did things. This fails with a "try" embedded inside a "try/else"
+            # since we can't detect COME_FROM boundaries.
+            # # We need to distinguish try_except from tryelsestmt and we do that
+            # # by checking the jump before the END_FINALLY
+            # # If we have:
+            # #    insn
+            # #    POP_TOP
+            # #    END_FINALLY
+            # #    COME_FROM
+            # # then insn is neither a JUMP_FORWARD nor RETURN_VALUE,
+            # # or if it is JUMP_FORWARD, then it can't be a JUMP_FORWARD to right after
+            # # COME_FROM
+            # if last == len(tokens):
+            #     last -= 1
+            # while tokens[last-1] == 'COME_FROM' and tokens[last-2] == 'COME_FROM':
+            #     last -= 1
+            # if tokens[last] == 'COME_FROM' and tokens[last-1] == 'COME_FROM':
+            #     last -= 1
+            # if (tokens[last] == 'COME_FROM'
+            #     and tokens[last-1] == 'END_FINALLY'
+            #         and tokens[last-2] == 'POP_TOP'):
+            #     # A jump of 2 is a jump around POP_TOP, END_FINALLY which
+            #     # would indicate try/else rather than try
+            #     return (tokens[last-3].kind in frozenset(('JUMP_FORWARD', 'RETURN_VALUE'))
+            #             and (tokens[last-3] != 'JUMP_FORWARD' or tokens[last-3].attr == 2))
 
 
         return False

uncompyle6/version.py

@@ -12,4 +12,4 @@
 #  along with this program.  If not, see <http://www.gnu.org/licenses/>.
 # This file is suitable for sourcing inside bash as
 # well as importing into Python
-VERSION='3.0.0'
+VERSION='3.0.1'