Merge branch 'master' into python-3.3-to-3.5

test/simple_source/looping/08_while1_if_continue.py

@@ -19,7 +19,7 @@ def _parse(a, b, source, state):
         else:
             raise
 
-def _parse2(source, state):
+def _parse2(source, state, a, b, this):
     while 1:
         if a:
             if b:
@@ -32,10 +32,10 @@ def _parse2(source, state):
             if b:
                 break
 
-        x = 3
+        x = this
 
 # Bug was in 2.3 decompilation
-def _parse3(source, state):
+def _parse3(source, state, a, b):
     while 1:
         if a:
             if b:

test/stdlib/2.6-exclude.sh

@@ -7,9 +7,7 @@ SKIP_TESTS=(
     #   assert 0  # shouldn't reach here.
     [test_shutil.py]=1
 
-    [test___all__.py]=1  # it fails on its own
     [test___all__.py]=1 # it fails on its own
-    [test_aepack.py]=1 # Fails on its own
     [test_aepack.py]=1 # it fails on its own
     [test_al.py]=1 # it fails on its own
     [test_anydbm.py]=1 # it fails on its own
@@ -59,7 +57,7 @@ SKIP_TESTS=(
     [test_ossaudiodev.py]=1 # it fails on its own
 
     [test_pep277.py]=1 # it fails on its own
-    [test_pyclbr.py]=1 # Investigate
+    [test_pyclbr.py]=1 # it fails on its own
     [test_py3kwarn.py]=1 # it fails on its own
 
     [test_scriptpackages.py]=1 # it fails on its own
@@ -85,7 +83,6 @@ SKIP_TESTS=(
 
     [test_zipimport_support.py]=1
     [test_zipfile64.py]=1  # Skip Long test
-    [test_zlib.py]=1  #
     # .pyenv/versions/2.6.9/lib/python2.6/lib2to3/refactor.pyc
     # .pyenv/versions/2.6.9/lib/python2.6/pyclbr.pyc
 )

test/stdlib/runtests.sh

@@ -1,6 +1,9 @@
 #!/bin/bash
 me=${BASH_SOURCE[0]}
 
+# Note: for 2.6 sometimes we need to set PYTHON=pytest
+PYTHON=${PYTHON:-python}
+
 typeset -i BATCH=${BATCH:-0}
 if (( ! BATCH )) ; then
     isatty=$(/usr/bin/tty 2>/dev/null)
@@ -191,7 +194,7 @@ for file in $files; do
     if [ ! -r $file ]; then
 	echo "Skipping test $file -- not readable. Does it exist?"
 	continue
-    elif ! python $file >/dev/null 2>&1 ; then
+    elif ! $PYTHON $file >/dev/null 2>&1 ; then
 	echo "Skipping test $file -- it fails on its own"
 	continue
     fi
@@ -214,7 +217,7 @@ for file in $files; do
     rc=$?
     if (( rc == 0 )) ; then
 	echo ========== $(date +%X) Running $file ===========
-	timeout_cmd python $file
+	timeout_cmd $PYTHON $file
 	rc=$?
     else
 	echo ======= Skipping $file due to compile/decompile errors ========

uncompyle6/parsers/parse2.py

@@ -27,7 +27,7 @@ that a later phase can turn into a sequence of ASCII text.
 
 from __future__ import print_function
 
-from uncompyle6.parsers.reducecheck import (except_handler_else, ifelsestmt, tryelsestmt)
+from uncompyle6.parsers.reducecheck import except_handler_else, ifelsestmt, tryelsestmt
 from uncompyle6.parser import PythonParser, PythonParserSingle, nop_func
 from uncompyle6.parsers.treenode import SyntaxTree
 from spark_parser import DEFAULT_DEBUG as PARSER_DEFAULT_DEBUG
@@ -313,11 +313,14 @@ class Python2Parser(PythonParser):
             opname_base = opname[: opname.rfind("_")]
 
             if opname in ("BUILD_CONST_LIST", "BUILD_CONST_SET"):
-                rule = """
+                rule = (
+                    """
                        add_consts          ::= ADD_VALUE*
                        const_list          ::= COLLECTION_START add_consts %s
                        expr                ::= const_list
-                       """ % opname
+                       """
+                    % opname
+                )
                 self.addRule(rule, nop_func)
 
             # The order of opname listed is roughly sorted below
@@ -696,7 +699,7 @@ class Python2Parser(PythonParser):
             # an optimization where the "and" jump_false is back to a loop.
             jmp_false = ast[1]
             if jmp_false[0] == "POP_JUMP_IF_FALSE":
-                while (first < last and isinstance(tokens[last].offset, str)):
+                while first < last and isinstance(tokens[last].offset, str):
                     last -= 1
                 if jmp_false[0].attr < tokens[last].offset:
                     return True
@@ -705,8 +708,10 @@ class Python2Parser(PythonParser):
             # or that it jumps to the same place as the end of "and"
             jmp_false = ast[1][0]
             jmp_target = jmp_false.offset + jmp_false.attr + 3
-            return not (jmp_target == tokens[last].offset or
-                        tokens[last].pattr == jmp_false.pattr)
+            return not (
+                jmp_target == tokens[last].offset
+                or tokens[last].pattr == jmp_false.pattr
+            )
         # Dead code testing...
         # if lhs == 'while1elsestmt':
         #     from trepan.api import debug; debug()

uncompyle6/parsers/parse24.py

@@ -97,6 +97,10 @@ class Python24Parser(Python25Parser):
         if self.version[:2] == (2, 4):
             self.check_reduce['nop_stmt'] = 'tokens'
 
+        if self.version[:2] <= (2, 4):
+            # TODO: We may add something different or customize something
+            del self.reduce_check_table["ifelsestmt"]
+
     def reduce_is_invalid(self, rule, ast, tokens, first, last):
         invalid = super(Python24Parser,
                         self).reduce_is_invalid(rule, ast,

uncompyle6/parsers/parse26.py

@@ -6,7 +6,7 @@ spark grammar differences over Python2 for Python 2.6.
 from uncompyle6.parser import PythonParserSingle
 from spark_parser import DEFAULT_DEBUG as PARSER_DEFAULT_DEBUG
 from uncompyle6.parsers.parse2 import Python2Parser
-from uncompyle6.parsers.reducecheck import (except_handler, tryexcept, tryelsestmt)
+from uncompyle6.parsers.reducecheck import (except_handler, ifelsestmt2, tryexcept, tryelsestmt)
 
 class Python26Parser(Python2Parser):
 
@@ -350,6 +350,7 @@ class Python26Parser(Python2Parser):
         super(Python26Parser, self).customize_grammar_rules(tokens, customize)
         self.reduce_check_table = {
             "except_handler": except_handler,
+            "ifelsestmt": ifelsestmt2,
             "tryelsestmt": tryelsestmt,
             "try_except": tryexcept,
             "tryelsestmtl": tryelsestmt,

uncompyle6/parsers/reducecheck/__init__.py

@@ -3,6 +3,7 @@ from uncompyle6.parsers.reducecheck.aug_assign import *
 from uncompyle6.parsers.reducecheck.except_handler import *
 from uncompyle6.parsers.reducecheck.except_handler_else import *
 from uncompyle6.parsers.reducecheck.ifelsestmt import *
+from uncompyle6.parsers.reducecheck.ifelsestmt2 import *
 from uncompyle6.parsers.reducecheck.iflaststmt import *
 from uncompyle6.parsers.reducecheck.ifstmt import *
 from uncompyle6.parsers.reducecheck.ifstmts_jump import *

uncompyle6/parsers/reducecheck/ifelsestmt2.py

@@ -0,0 +1,147 @@
+#  Copyright (c) 2020-2022 Rocky Bernstein
+
+IFELSE_STMT_RULES = frozenset(
+    [
+        (
+            "ifelsestmt",
+            (
+                "testexpr_then",
+                "pass",
+                "filler",
+                "else_suitel",
+                "COME_FROM",
+                "POP_TOP",
+            ),
+        ),
+        (
+            "ifelsestmt",
+            (
+                "testexpr_then",
+                "c_stmts_opt",
+                "\\e_filler",
+                "else_suitel",
+                "come_froms",
+                "POP_TOP",
+            ),
+        ),
+        (
+            "ifelsestmt",
+            (
+                "testexpr_then",
+                "\\e_c_stmts_opt",
+                "\\e_filler",
+                "else_suitel",
+                "come_froms",
+                "POP_TOP",
+            ),
+        ),
+        # We may do something like add these in the future:
+    ]
+)
+
+
+def ifelsestmt2(self, lhs, n, rule, tree, tokens, first, last):
+
+    if (last + 1) < n and tokens[last + 1] == "COME_FROM_LOOP" and lhs != "ifelsestmtc":
+        # ifelsestmt jumped outside of loop. No good.
+        return True
+
+    # print("XXX", first, last)
+    # for t in range(first, last):
+    #     print(tokens[t])
+    # print("=" * 30)
+
+    if rule not in IFELSE_STMT_RULES:
+        # print("XXX", rule)
+        return False
+
+    # Avoid if/else where the "then" is a "raise_stmt1" for an
+    # assert statement. Parse this as an "assert" instead.
+    stmts = tree[1]
+    if stmts in ("c_stmts",) and len(stmts) == 1:
+        raise_stmt1 = stmts[0]
+        if raise_stmt1 == "raise_stmt1" and raise_stmt1[0] in ("LOAD_ASSERT",):
+            return True
+
+    # Make sure all of the "come froms" offset at the
+    # end of the "if" come from somewhere inside the "if".
+    # Since the come_froms are ordered so that lowest
+    # offset COME_FROM is last, it is sufficient to test
+    # just the last one.
+    if len(tree) == 6 and tree[-1] == "POP_TOP":
+        # FIXME: There is weirdness in the grammar we need to work around.
+        # we need to clean up the grammar.
+        last_token = tree[-2]
+        if last_token == "COME_FROM" and tokens[first].offset > last_token.attr:
+            if (
+                self.insts[self.offset2inst_index[last_token.attr]].opname
+                != "SETUP_LOOP"
+            ):
+                return True
+
+    testexpr = tree[0]
+
+    # Check that the condition portion of the "if"
+    # jumps to the "else" part.
+    if testexpr[0] in ("testtrue", "testfalse", "testfalse_then"):
+        if_condition = testexpr[0]
+
+        else_suite = tree[3]
+        assert else_suite.kind.startswith("else_suite")
+
+        if len(if_condition) > 1 and if_condition[1].kind.startswith("jmp_"):
+            if last == n:
+                last -= 1
+            jmp = if_condition[1]
+            jmp_target = int(jmp[0].pattr)
+
+            # Below we check that jmp_target is jumping to a feasible
+            # location. It should be to the transition after the "then"
+            # block and to the beginning of the "else" block.
+            # However the "if/else" is inside a loop the false test can be
+            # back to the loop.
+
+            # FIXME: the below logic for jf_cfs could probably be
+            # simplified.
+            if tree[2] == "filler":
+                jump_else_end = tree[3]
+            else:
+                jump_else_end = tree[2]
+
+            if jump_else_end == "jf_cfs":
+                jump_else_end = jump_else_end[0]
+
+            if jump_else_end == "JUMP_FORWARD":
+                endif_target = int(jump_else_end.pattr)
+                last_offset = tokens[last].off2int()
+                if endif_target != last_offset:
+                    return True
+            last_offset = tokens[last].off2int(prefer_last=False)
+            if jmp_target <= last_offset:
+                # jmp_target should be jumping to the end of the if/then/else
+                # but is it jumping to the beginning of the "else" or before
+                return True
+            if (
+                jump_else_end in ("jf_cfs", "jump_forward_else")
+                and jump_else_end[0] == "JUMP_FORWARD"
+            ):
+                # If the "else" jump jumps before the end of the the "if .. else end", then this
+                # is not this kind of "ifelsestmt".
+                jump_else_forward = jump_else_end[0]
+                jump_else_forward_target = jump_else_forward.attr
+                if jump_else_forward_target < last_offset:
+                    return True
+                pass
+            if (
+                jump_else_end in ("jb_elsec", "jb_elsel", "jf_cfs", "jb_cfs")
+                and jump_else_end[-1] == "COME_FROM"
+            ):
+                if jump_else_end[-1].off2int() != jmp_target:
+                    return True
+
+            if tokens[first].off2int() > jmp_target:
+                return True
+
+            return (jmp_target > last_offset) and tokens[last] != "JUMP_FORWARD"
+
+    return False