Merge branch 'master' into python-2.4

test/simple_source/branching/02_ifelse_lambda.py

@@ -3,8 +3,6 @@
 
 f = lambda x: 1 if x<2 else 3
 f(5)
-<<<<<<< HEAD
-=======
 
 # If that wasn't enough ...
 # Python will create dead code
@@ -16,4 +14,6 @@ g()
 
 h = lambda: 1 if False else 3
 h()
->>>>>>> master
+
+# From 2.7 test_builtin
+lambda c: 'a' <= c <= 'z', 'Hello World'

test/simple_source/bug33/01_triple_compare.py

@@ -1,5 +1,13 @@
 # In Python 3.3+ this uses grammar rule
 # compare_chained2 ::= expr COMPARE_OP RETURN_VALUE
 
-def _is_valid_netmask(self, netmask):
+# Seen in Python 3.3 ipaddress.py
-    return 0 <= netmask <= self._max_prefixlen
+
+def _is_valid_netmask(netmask):
+    return 0 <= netmask <= 10
+
+# There were also bugs in 2.6- involving the use of "or" twice in its "or"
+# detections
+
+# See in 2.6.9 quopri.py ishex():
+'0' <= __file__ <= '9' or 'a' <= __file__ <= 'f' or 'A' <= __file__ <= 'F'

test/stdlib/runtests.sh

@@ -61,17 +61,48 @@ case $PYVERSION in
 	    [test_ftplib.py]=1  # Control flow?
 	    [test_funcattrs.py]=1  # Control flow?
 	    [test_grp.py]=1      # Long test - might work Control flow?
+	    [test_imp.py]=1
+	    [test_int.py]=1
+	    [test_long.py]=1
+	    [test_pty.py]=1
 	    [test_pwd.py]=1 # Long test - might work? Control flow?
 	    [test_queue.py]=1 # Control flow?
+	    [test_re.py]=1 # Probably Control flow?
+	    [test_strftime.py]=1
+	    [test_trace.py]=1  # Line numbers are expected to be different
+	    # .pyenv/versions/2.6.9/lib/python2.6/lib2to3/refactor.pyc
+	    # .pyenv/versions/2.6.9/lib/python2.6/mailbox.pyc
+	    # .pyenv/versions/2.6.9/lib/python2.6/markupbase.pyc
+	    # .pyenv/versions/2.6.9/lib/python2.6/pstats.pyc
+	    # .pyenv/versions/2.6.9/lib/python2.6/pyclbr.pyc
+	    # .pyenv/versions/2.6.9/lib/python2.6/quopri.pyc -- look at ishex, is short
+	    # .pyenv/versions/2.6.9/lib/python2.6/random.pyc
+	    # .pyenv/versions/2.6.9/lib/python2.6/smtpd.pyc
+	    # .pyenv/versions/2.6.9/lib/python2.6/sre_parse.pyc
+	    # .pyenv/versions/2.6.9/lib/python2.6/tabnanny.pyc
+	    # .pyenv/versions/2.6.9/lib/python2.6/tarfile.pyc
 	    )
 	;;
     2.7)
 	SKIP_TESTS=(
-	    [test_builtin.py]=1  # Syntax error, look at
 	    [test_dis.py]=1   # We change line numbers - duh!
 	    [test_grammar.py]=1  # Too many stmts. Handle large stmts
+	    [test_io.py]=1 # Test takes too long to run
 	    [test_ioctl.py]=1 # Test takes too long to run
 	    [test_itertools.py]=1 # Syntax error - look at!
+	    [test_memoryio.py]=1 # ?
+	    [test_multiprocessing.py]=1 # ?
+	    [test_pep352.py]=1 # ?
+	    [test_re.py]=1 # ?
+	    [test_sys_settrace.py]=1 # Line numbers are expected to be different
+	    [test_strtod.py]=1
+	    [test_traceback.py]=1
+	    [test_unicode.py]=1
+	    # Syntax errors:
+	    # .pyenv/versions/2.7.14/lib/python2.7/mimify.pyc
+	    # .pyenv/versions/2.7.14/lib/python2.7/netrc.pyc
+	    # .pyenv/versions/2.7.14/lib/python2.7/pyclbr.pyc
+	    # .pyenv/versions/2.7.14/lib/python2.7/sre_compile.pyc
         )
 	;;
     *)
@@ -101,13 +132,22 @@ if [[ -n $1 ]] ; then
     files=$1
     SKIP_TESTS=()
 else
-    files=test_[m]*.py
+    files=test_*.py
 fi
 for file in $files; do
     [[ -v SKIP_TESTS[$file] ]] && continue
 
     # If the fails *before* decompiling, skip it!
+    typeset -i STARTTIME=$(date +%s)
     if ! python $file >/dev/null 2>&1 ; then
+	echo "Skipping test $file -- it fails on its own"
+	continue
+    fi
+    typeset -i ENDTIME=$(date +%s)
+    typeset -i time_diff
+    (( time_diff =  ENDTIME - STARTTIME))
+    if (( time_diff > 10 )) ; then
+	echo "Skipping test $file -- test takes too long to run: $time_diff seconds"
 	continue
     fi
 
@@ -117,10 +157,11 @@ for file in $files; do
     decompiled_file=$short_name-${PYVERSION}.pyc
     $fulldir/compile-file.py $file && \
     mv $file{,.orig} && \
+    echo ==========  $(date +%X) Decompiling $file ===========
     $fulldir/../../bin/uncompyle6 $decompiled_file > $file
     rc=$?
     if (( rc == 0 )) ; then
-	echo ========== Running $file ===========
+	echo ========== $(date +%X) Running $file ===========
 	python $file
 	rc=$?
     else

test/test_pyenvlib.py

@@ -101,6 +101,9 @@ def do_tests(src_dir, patterns, target_dir, start_with=None, do_verify=False):
         files = [file for file in files if not 'site-packages' in file]
         files = [file for file in files if not 'test' in file]
         if len(files) > 200:
+            # print("Numer of files %d - truncating to last 200" % len(files))
+            # files = files[-200:]
+            print("Numer of files %d - truncating to first 200" % len(files))
             files = files[:200]
 
     print(time.ctime())

uncompyle6/parser.py

@@ -266,7 +266,7 @@ class PythonParser(GenericASTBuilder):
         stmt ::= while1stmt
         stmt ::= whileelsestmt
         stmt ::= while1elsestmt
-        stmt ::= forstmt
+        stmt ::= for
         stmt ::= forelsestmt
         stmt ::= try_except
         stmt ::= tryelsestmt
@@ -374,20 +374,20 @@ class PythonParser(GenericASTBuilder):
 
     def p_forstmt(self, args):
         """
-        _for ::= GET_ITER FOR_ITER
+        for_iter ::= GET_ITER FOR_ITER
 
         for_block ::= l_stmts_opt _come_froms JUMP_BACK
 
-        forstmt ::= SETUP_LOOP expr _for store
+        for       ::= SETUP_LOOP expr for_iter store
-                for_block POP_BLOCK _come_froms
+                      for_block POP_BLOCK _come_froms
 
-        forelsestmt ::= SETUP_LOOP expr _for store
+        forelsestmt ::= SETUP_LOOP expr for_iter store
                 for_block POP_BLOCK else_suite _come_froms
 
-        forelselaststmt ::= SETUP_LOOP expr _for store
+        forelselaststmt ::= SETUP_LOOP expr for_iter store
                 for_block POP_BLOCK else_suitec _come_froms
 
-        forelselaststmtl ::= SETUP_LOOP expr _for store
+        forelselaststmtl ::= SETUP_LOOP expr for_iter store
                 for_block POP_BLOCK else_suitel _come_froms
         """
 

uncompyle6/parsers/parse2.py

@@ -244,7 +244,7 @@ class Python2Parser(PythonParser):
                         stmt ::= assign2_pypy
                         assign3_pypy ::= expr expr expr store store store
                         assign2_pypy ::= expr expr store store
-                        list_comp    ::= expr  BUILD_LIST_FROM_ARG _for store list_iter
+                        list_comp    ::= expr  BUILD_LIST_FROM_ARG for_iter store list_iter
                                          JUMP_BACK
                         """, nop_func)
 

uncompyle6/parsers/parse21.py

@@ -13,11 +13,11 @@ class Python21Parser(Python22Parser):
 
     def p_forstmt21(self, args):
         """
-        _for        ::= LOAD_CONST FOR_LOOP
+        for_iter    ::= LOAD_CONST FOR_LOOP
-        forstmt     ::= SETUP_LOOP expr _for store
+        for         ::= SETUP_LOOP expr for_iter store
                         return_stmts
                         POP_BLOCK COME_FROM
-        forstmt     ::= SETUP_LOOP expr _for store
+        for         ::= SETUP_LOOP expr for_iter store
                         l_stmts_opt _jump_back
                         POP_BLOCK COME_FROM
 

uncompyle6/parsers/parse22.py

@@ -13,10 +13,10 @@ class Python22Parser(Python23Parser):
 
     def p_misc22(self, args):
         '''
-        _for ::= LOAD_CONST FOR_LOOP
+        for_iter  ::= LOAD_CONST FOR_LOOP
         list_iter ::= list_if JUMP_FORWARD
                       COME_FROM POP_TOP COME_FROM
-        list_for  ::= expr _for store list_iter CONTINUE JUMP_FORWARD
+        list_for  ::= expr for_iter store list_iter CONTINUE JUMP_FORWARD
                       COME_FROM POP_TOP COME_FROM
         '''
 

uncompyle6/parsers/parse23.py

@@ -36,7 +36,7 @@ class Python23Parser(Python24Parser):
                        COME_FROM POP_TOP POP_BLOCK COME_FROM
 
         list_comp  ::= BUILD_LIST_0 DUP_TOP LOAD_ATTR store list_iter del_stmt
-        list_for   ::= expr _for store list_iter JUMP_BACK come_froms POP_TOP JUMP_BACK
+        list_for   ::= expr for_iter store list_iter JUMP_BACK come_froms POP_TOP JUMP_BACK
 
         lc_body ::= LOAD_NAME expr CALL_FUNCTION_1 POP_TOP
         lc_body ::= LOAD_FAST expr CALL_FUNCTION_1 POP_TOP

uncompyle6/parsers/parse26.py

@@ -180,23 +180,23 @@ class Python26Parser(Python2Parser):
 
     def p_comp26(self, args):
         '''
-        list_for ::= expr _for store list_iter JUMP_BACK come_froms POP_TOP
+        list_for ::= expr for_iter store list_iter JUMP_BACK come_froms POP_TOP
 
         # The JUMP FORWARD below jumps to the JUMP BACK. It seems to happen
         # in rare cases that may have to with length of code
         # FIXME: we can add a reduction check for this
 
-        list_for ::= expr _for store list_iter JUMP_FORWARD come_froms POP_TOP
+        list_for ::= expr for_iter store list_iter JUMP_FORWARD come_froms POP_TOP
                      COME_FROM JUMP_BACK
 
-        list_for ::= expr _for store list_iter jb_cont
+        list_for ::= expr for_iter store list_iter jb_cont
 
         # This is for a really funky:
         #   [  x for x in range(10) if x % 2 if x % 3 ]
         # the JUMP_ABSOLUTE is to the instruction after the last POP_TOP
         #  we have a reduction check for this
 
-        list_for ::= expr _for store list_iter JUMP_ABSOLUTE come_froms
+        list_for ::= expr for_iter store list_iter JUMP_ABSOLUTE come_froms
                      POP_TOP jb_pop
 
         list_iter  ::= list_if JUMP_BACK
@@ -208,7 +208,7 @@ class Python26Parser(Python2Parser):
         lc_body    ::= LOAD_NAME expr LIST_APPEND
 	lc_body    ::= LOAD_FAST expr LIST_APPEND
 
-        comp_for ::= SETUP_LOOP expr _for store comp_iter jb_pb_come_from
+        comp_for ::= SETUP_LOOP expr for_iter store comp_iter jb_pb_come_from
 
         comp_body ::= gen_comp_body
 
@@ -259,10 +259,24 @@ class Python26Parser(Python2Parser):
 
         # compare_chained is like x <= y <= z
         compare_chained    ::= expr compare_chained1 ROT_TWO COME_FROM POP_TOP _come_froms
+        compare_chained1   ::= expr DUP_TOP ROT_THREE COMPARE_OP
+                               jmp_false_then compare_chained1 _come_froms
+        compare_chained1   ::= expr DUP_TOP ROT_THREE COMPARE_OP
+                               jmp_false_then compare_chained2 _come_froms
         compare_chained1   ::= expr DUP_TOP ROT_THREE COMPARE_OP
                                jmp_false compare_chained1 _come_froms
         compare_chained1   ::= expr DUP_TOP ROT_THREE COMPARE_OP
                                jmp_false compare_chained2 _come_froms
+        compare_chained1   ::= expr DUP_TOP ROT_THREE COMPARE_OP
+                               jmp_false_then compare_chained2 _come_froms
+
+        return_lambda      ::= RETURN_VALUE
+        return_lambda      ::= RETURN_END_IF
+        return_lambda      ::= RETURN_END_IF_LAMBDA
+        return_lambda      ::= RETURN_VALUE_LAMBDA
+
+        compare_chained2   ::= expr COMPARE_OP return_lambda
+
         return_if_lambda   ::= RETURN_END_IF_LAMBDA POP_TOP
         stmt               ::= conditional_lambda
         conditional_lambda ::= expr jmp_false_then expr return_if_lambda
@@ -314,7 +328,7 @@ class Python26Parser(Python2Parser):
                         tokens[last].pattr == jmp_false.pattr)
         elif rule == (
                 'list_for',
-                ('expr', '_for', 'store', 'list_iter',
+                ('expr', 'for_iter', 'store', 'list_iter',
                  'JUMP_ABSOLUTE', 'come_froms', 'POP_TOP', 'jb_pop')):
             # The JUMP_ABSOLUTE has to be to the last POP_TOP or this is invalid
             ja_attr = ast[4].attr

uncompyle6/parsers/parse27.py

@@ -14,7 +14,7 @@ class Python27Parser(Python2Parser):
 
     def p_comprehension27(self, args):
         """
-        list_for  ::= expr _for store list_iter JUMP_BACK
+        list_for  ::= expr for_iter store list_iter JUMP_BACK
         list_comp ::= BUILD_LIST_0 list_iter
         lc_body   ::= expr LIST_APPEND
 
@@ -33,7 +33,7 @@ class Python27Parser(Python2Parser):
 
         comp_body ::= dict_comp_body
         comp_body ::= set_comp_body
-        comp_for ::= expr _for store comp_iter JUMP_BACK
+        comp_for ::= expr for_iter store comp_iter JUMP_BACK
 
         dict_comp_body ::= expr expr MAP_ADD
         set_comp_body ::= expr SET_ADD
@@ -82,7 +82,12 @@ class Python27Parser(Python2Parser):
                              compare_chained1 COME_FROM
         compare_chained1 ::= expr DUP_TOP ROT_THREE COMPARE_OP JUMP_IF_FALSE_OR_POP
                              compare_chained2 COME_FROM
-        compare_chained2 ::= expr COMPARE_OP RETURN_VALUE
+
+        return_lambda      ::= RETURN_VALUE
+        return_lambda      ::= RETURN_VALUE_LAMBDA
+
+        compare_chained2 ::= expr COMPARE_OP return_lambda
+        compare_chained2 ::= expr COMPARE_OP return_lambda
 
         # conditional_true are for conditions which always evaluate true
         # There is dead or non-optional remnants of the condition code though,

uncompyle6/parsers/parse3.py

@@ -36,8 +36,8 @@ class Python3Parser(PythonParser):
         # one may be a continue - sometimes classifies a JUMP_BACK
         # as a CONTINUE. The two are kind of the same in a comprehension.
 
-        comp_for ::= expr _for store comp_iter CONTINUE
+        comp_for ::= expr for_iter store comp_iter CONTINUE
-        comp_for ::= expr _for store comp_iter JUMP_BACK
+        comp_for ::= expr for_iter store comp_iter JUMP_BACK
 
         list_comp ::= BUILD_LIST_0 list_iter
         lc_body   ::= expr LIST_APPEND
@@ -326,16 +326,16 @@ class Python3Parser(PythonParser):
 
     def p_loop_stmt3(self, args):
         """
-        forstmt           ::= SETUP_LOOP expr _for store for_block POP_BLOCK
+        for               ::= SETUP_LOOP expr for_iter store for_block POP_BLOCK
                               COME_FROM_LOOP
 
-        forelsestmt       ::= SETUP_LOOP expr _for store for_block POP_BLOCK else_suite
+        forelsestmt       ::= SETUP_LOOP expr for_iter store for_block POP_BLOCK else_suite
                               COME_FROM_LOOP
 
-        forelselaststmt   ::= SETUP_LOOP expr _for store for_block POP_BLOCK else_suitec
+        forelselaststmt   ::= SETUP_LOOP expr for_iter store for_block POP_BLOCK else_suitec
                               COME_FROM_LOOP
 
-        forelselaststmtl  ::= SETUP_LOOP expr _for store for_block POP_BLOCK else_suitel
+        forelselaststmtl  ::= SETUP_LOOP expr for_iter store for_block POP_BLOCK else_suitel
                               COME_FROM_LOOP
 
         whilestmt         ::= SETUP_LOOP testexpr l_stmts_opt COME_FROM JUMP_BACK POP_BLOCK
@@ -370,7 +370,7 @@ class Python3Parser(PythonParser):
                               COME_FROM_LOOP
         whileTruestmt     ::= SETUP_LOOP l_stmts_opt JUMP_BACK POP_BLOCK NOP
                               COME_FROM_LOOP
-        forstmt           ::= SETUP_LOOP expr _for store for_block POP_BLOCK NOP
+        for               ::= SETUP_LOOP expr for_iter store for_block POP_BLOCK NOP
                               COME_FROM_LOOP
         """
 

uncompyle6/scanners/scanner2.py

@@ -715,18 +715,22 @@ class Scanner2(Scanner):
             # rocky: if we have a conditional jump to the next instruction, then
             # possibly I am "skipping over" a "pass" or null statement.
 
+            test_target = target
             if self.version < 2.7:
-                op_testset = set([self.opc.POP_TOP,
+                # Before 2.6 we have to deal with the fact that there is an extra
-                                   self.opc.JUMP_IF_TRUE, self.opc.JUMP_IF_FALSE])
+                # POP_TOP that is logically associated with the JUMP_IF's (even though
+                # the instance set is called "self.pop_jump_if")
+                if code[pre[test_target]] == self.opc.POP_TOP:
+                    test_target = pre[test_target]
+                test_set = self.pop_jump_if
             else:
-                op_testset = self.pop_jump_if_or_pop | self.pop_jump_if
+                test_set = self.pop_jump_if_or_pop | self.pop_jump_if
 
-            if ( code[pre[target]] in op_testset
+            if ( code[pre[test_target]] in test_set and target > offset ):
-                 and (target > offset) ):
                 self.fixed_jumps[offset] = pre[target]
                 self.structs.append({'type':  'and/or',
-                                       'start': start,
+                                     'start': start,
-                                       'end':   pre[target]})
+                                     'end':   pre[target]})
                 return
 
             # The op offset just before the target jump offset is important

uncompyle6/scanners/scanner3.py

@@ -196,8 +196,12 @@ class Scanner3(Scanner):
         # 'LOAD_ASSERT' is used in assert statements.
         self.load_asserts = set()
         self.insts = list(bytecode)
+        self.offset2inst_index = {}
         n = len(self.insts)
         for i, inst in enumerate(self.insts):
+
+            self.offset2inst_index[inst.offset] = i
+
             # We need to detect the difference between:
             #   raise AssertionError
             #  and
@@ -953,8 +957,18 @@ class Scanner3(Scanner):
                     self.fixed_jumps[offset] = rtarget
                     self.not_continue.add(pre_rtarget)
             else:
-                # For now, we'll only tag forward jump.
+
-                if self.version >= 3.6:
+                # FIXME: this is very convoluted and based on rather hacky
+                # empirical evidence. It should go a way when
+                # we have better control-flow analysis
+                normal_jump = self.version >= 3.6
+                if self.version == 3.5:
+                    j = self.offset2inst_index[target]
+                    if j+2 < len(self.insts) and self.insts[j+2].is_jump_target:
+                        normal_jump = self.insts[j+1].opname == 'POP_BLOCK'
+
+                if normal_jump:
+                    # For now, we'll only tag forward jump.
                     if target > offset:
                         self.fixed_jumps[offset] = target
                         pass

uncompyle6/semantics/consts.py

@@ -258,7 +258,7 @@ TABLE_DIRECT = {
     'while1elsestmt':   ( '%|while 1:\n%+%c%-%|else:\n%+%c%-\n\n', 1, -2 ),
     'whileelsestmt':	( '%|while %c:\n%+%c%-%|else:\n%+%c%-\n\n', 1, 2, -2 ),
     'whileelselaststmt':	( '%|while %c:\n%+%c%-%|else:\n%+%c%-', 1, 2, -2 ),
-    'forstmt':		    ( '%|for %c in %c:\n%+%c%-\n\n', 3, 1, 4 ),
+    'for':		        ( '%|for %c in %c:\n%+%c%-\n\n', 3, 1, 4 ),
     'forelsestmt':	    (
         '%|for %c in %c:\n%+%c%-%|else:\n%+%c%-\n\n', 3, 1, 4, -2 ),
     'forelselaststmt':	(

uncompyle6/semantics/fragments.py

@@ -105,7 +105,7 @@ TABLE_DIRECT_FRAGMENT = {
     'whilestmt':	( '%|while%b %c:\n%+%c%-\n\n', 0, 1, 2 ),
     'whileelsestmt':	( '%|while%b %c:\n%+%c%-%|else:\n%+%c%-\n\n', 0, 1, 2, -2 ),
     'whileelselaststmt':	( '%|while%b %c:\n%+%c%-%|else:\n%+%c%-', 0, 1, 2, -2 ),
-    'forstmt':		( '%|for%b %c in %c:\n%+%c%-\n\n', 0, 3, 1, 4 ),
+    'for':	        ( '%|for%b %c in %c:\n%+%c%-\n\n', 0, 3, 1, 4 ),
     }