Merge branch 'master' into python-2.4

test/simple_source/bug33/01_triple_compare.py

@@ -1,4 +1,5 @@
 # In Python 3.3+ this uses grammar rule
-# cmp_list2 ::= expr COMPARE_OP RETURN_VALUE
+# compare_chained2 ::= expr COMPARE_OP RETURN_VALUE
+
 def _is_valid_netmask(self, netmask):
     return 0 <= netmask <= self._max_prefixlen

test/simple_source/expression/01_float.py

@@ -0,0 +1,4 @@
+a = 1e300 * 1e300 * 0
+b = -1e300 * 1e300 * 0
+c = 1e300 * 1e300
+d = -1e300 * 1e300

test/simple_source/expression/03_tuple_assign.py

@@ -6,7 +6,12 @@ def some_other_function():
     some_variable, = some_function()
     print(some_variable)
 
+# From 2.7 test_compile.py
+# Bug is adding erroneous parens in d[(1:2, 1:2)] += 1
+def bug(d):
+    d[1:2, 1:2] += 1
+
 empty_tup = ()
 one_item_tup = ("item1", )
-one_item_tup_without_parentheses = "item", 
+one_item_tup_without_parentheses = "item",
 many_items_tup = ("item1", "item2", "item3")

test/simple_source/expression/10_lambda.py

@@ -26,3 +26,11 @@ class ExtendedInterpolation():
         value_getter = lambda option: self._interpolation.before_get(self,
             section, option, d[option], d)
         return value_getter
+
+# Bug from Python 2.7's test_collections.py
+# is that the lambda function has two
+# statements in it, one for returning *after* the yield
+# The return None statement should be removed and the
+# yield should be turned into a statement
+def test_Iterable(self):
+    return (lambda: (yield))()

test/simple_source/stmts/04_withas.py

@@ -10,3 +10,9 @@
 def formatweekday(self):
     with self as encoding:
         return encoding
+
+# Bug in 2.7.14 test_contextlib.py. Bug was not enclosing (x,y) in parenthesis
+def withas_bug(self, nested, a, b):
+    with self.assertRaises(ZeroDivisionError):
+        with nested(a(), b()) as (x, y):
+                1 // 0

test/stdlib/README.md

@@ -0,0 +1,3 @@
+The programs in here are to test Python stdlib tests  in lib/pythonX.Y/test
+
+We'll compile a test, then decompile it and then run the test

test/stdlib/compile-file.py

@@ -0,0 +1,23 @@
+#!/usr/bin/env python
+import sys
+if len(sys.argv) != 2:
+    print("Usage: compile-file.py *python-file*")
+    sys.exit(1)
+source = sys.argv[1]
+
+assert source.endswith('.py')
+basename = source[:-3]
+
+# We do this crazy way to support Python 2.6 which
+# doesn't support version_major, and has a bug in
+# floating point so we can't divide 26 by 10 and get
+# 2.6
+PY_VERSION = sys.version_info[0] + (sys.version_info[1] / 10.0)
+
+bytecode = "%s-%s.pyc" % (basename, PY_VERSION)
+
+import py_compile
+print("compiling %s to %s" % (source, bytecode))
+py_compile.compile(source, bytecode, 'exec')
+# import os
+# os.system("../bin/uncompyle6 %s" % bytecode)

test/stdlib/runtests.sh

@@ -0,0 +1,58 @@
+#!/bin/bash
+me=${BASH_SOURCE[0]}
+
+# Python version setup
+FULLVERSION=${1:-2.7.14}
+PYVERSION=${FULLVERSION%.*}
+MINOR=${FULLVERSION##?.?.}
+
+typeset -i STOP_ONERROR=1
+typeset -A SKIP_TESTS=( [test_aepack.py]=1 [audiotests.py]=1)
+
+# Test directory setup
+srcdir=$(dirname $me)
+cd $srcdir
+fulldir=$(pwd)
+TESTDIR=/tmp/test${PYVERSION}
+if [[ -e $TESTDIR ]] ; then
+    rm -fr $TESTDIR
+fi
+mkdir $TESTDIR || exit $?
+cp -r ~/.pyenv/versions/${PYVERSION}.${MINOR}/lib/python${PYVERSION}/test $TESTDIR
+cd $TESTDIR/test
+export PYTHONPATH=$TESTDIR
+
+# Run tests
+typeset -i i=0
+typeset -i allerrs=0
+for file in test_*.py; do
+    [[ -v SKIP_TESTS[$file] ]] && continue
+
+    # If the fails *before* decompiling, skip it!
+    if ! python $file >/dev/null 2>&1 ; then
+	continue
+    fi
+
+    ((i++))
+    # (( i > 40 )) && break
+    short_name=$(basename $file .py)
+    decompiled_file=$short_name-${PYVERSION}.pyc
+    $fulldir/compile-file.py $file && \
+    mv $file{,.orig} && \
+    $fulldir/../../bin/uncompyle6 $decompiled_file > $file
+    rc=$?
+    if (( rc == 0 )) ; then
+	echo ========== Running $file ===========
+	python $file
+	rc=$?
+    else
+	echo ======= Skipping $file due to compile/decompile errors ========
+    fi
+    (( rc != 0 && allerrs++ ))
+    if (( STOP_ONERROR && rc )) ; then
+	echo "** Ran $i tests before failure **"
+	exit $allerrs
+    fi
+done
+echo "Ran $i tests"
+exit $allerrs

uncompyle6/parser.py

@@ -442,7 +442,7 @@ class PythonParser(GenericASTBuilder):
         expr ::= load_attr
         expr ::= binary_expr
         expr ::= build_list
-        expr ::= cmp
+        expr ::= compare
         expr ::= mapexpr
         expr ::= and
         expr ::= or
@@ -508,14 +508,17 @@ class PythonParser(GenericASTBuilder):
         return_lambda ::= ret_expr RETURN_VALUE_LAMBDA LAMBDA_MARKER
         return_lambda ::= ret_expr RETURN_VALUE_LAMBDA
 
-        # Doesn't seemt to be used anymore, but other conditional_lambda's are
+        # Doesn't seem to be used anymore, but other conditional_lambda's are
         # conditional_lambda ::= expr jmp_false return_if_stmt return_stmt LAMBDA_MARKER
 
-        cmp ::= cmp_list
+        compare        ::= compare_chained
-        cmp ::= compare
+        compare        ::= compare_single
-        compare ::= expr expr COMPARE_OP
+        compare_single ::= expr expr COMPARE_OP
-        cmp_list ::= expr cmp_list1 ROT_TWO POP_TOP _come_from
+
-        cmp_list2 ::= expr COMPARE_OP JUMP_FORWARD
+        # A compare_chained is two comparisions like x <= y <= z
+        compare_chained  ::= expr compare_chained1 ROT_TWO POP_TOP _come_from
+        compare_chained2 ::= expr COMPARE_OP JUMP_FORWARD
+
         mapexpr ::= BUILD_MAP kvlist
 
         kvlist ::= kvlist kv

uncompyle6/parsers/parse2.py

@@ -201,6 +201,10 @@ class Python2Parser(PythonParser):
         binary_subscr2 ::= expr expr DUP_TOPX_2 BINARY_SUBSCR
 
         conditional ::= expr jmp_false expr JUMP_ABSOLUTE expr
+
+        # compare_chained2 is used in a "chained_compare": x <= y <= z
+        compare_chained2   ::= expr COMPARE_OP RETURN_VALUE
+        compare_chained2   ::= expr COMPARE_OP RETURN_VALUE_LAMBDA
         """
 
     def p_slice2(self, args):

uncompyle6/parsers/parse26.py

@@ -243,10 +243,12 @@ class Python26Parser(Python2Parser):
     def p_misc26(self, args):
         """
         conditional  ::= expr jmp_false expr jf_cf_pop expr come_from_opt
-        and  ::= expr JUMP_IF_FALSE POP_TOP expr JUMP_IF_FALSE POP_TOP
+        and          ::= expr JUMP_IF_FALSE POP_TOP expr JUMP_IF_FALSE POP_TOP
-        cmp_list ::=  expr cmp_list1 ROT_TWO COME_FROM POP_TOP _come_from
+
-        cmp_list1 ::= expr DUP_TOP ROT_THREE COMPARE_OP jmp_false cmp_list1 _come_from
+        # compare_chained is like x <= y <= z
-        cmp_list1 ::= expr DUP_TOP ROT_THREE COMPARE_OP jmp_false cmp_list2 _come_from
+        compare_chained  ::= expr compare_chained1 ROT_TWO COME_FROM POP_TOP _come_from
+        compare_chained1 ::= expr DUP_TOP ROT_THREE COMPARE_OP jmp_false compare_chained1 _come_from
+        compare_chained1 ::= expr DUP_TOP ROT_THREE COMPARE_OP jmp_false compare_chained2 _come_from
 
         return_if_lambda   ::= RETURN_END_IF_LAMBDA POP_TOP
         conditional_lambda ::= expr jmp_false_then expr return_if_lambda

uncompyle6/parsers/parse27.py

@@ -77,12 +77,11 @@ class Python27Parser(Python2Parser):
         or   ::= expr JUMP_IF_TRUE_OR_POP expr COME_FROM
         and  ::= expr JUMP_IF_FALSE_OR_POP expr COME_FROM
 
-        cmp_list1 ::= expr DUP_TOP ROT_THREE
+        # compare_chained1 is used exclusively in chained_compare
-                COMPARE_OP JUMP_IF_FALSE_OR_POP
+        compare_chained1 ::= expr DUP_TOP ROT_THREE COMPARE_OP JUMP_IF_FALSE_OR_POP
-                cmp_list1 COME_FROM
+                             compare_chained1 COME_FROM
-        cmp_list1 ::= expr DUP_TOP ROT_THREE
+        compare_chained1 ::= expr DUP_TOP ROT_THREE COMPARE_OP JUMP_IF_FALSE_OR_POP
-                COMPARE_OP JUMP_IF_FALSE_OR_POP
+                             compare_chained2 COME_FROM
-                cmp_list2 COME_FROM
         """
 
     def p_stmt27(self, args):

uncompyle6/parsers/parse3.py

@@ -332,10 +332,11 @@ class Python3Parser(PythonParser):
         or   ::= expr JUMP_IF_TRUE_OR_POP expr COME_FROM
         and  ::= expr JUMP_IF_FALSE_OR_POP expr COME_FROM
 
-        cmp_list1 ::= expr DUP_TOP ROT_THREE COMPARE_OP JUMP_IF_FALSE_OR_POP
+        # compare_chained1 is used exclusively in chained_compare
-                      cmp_list1 COME_FROM
+        compare_chained1 ::= expr DUP_TOP ROT_THREE COMPARE_OP JUMP_IF_FALSE_OR_POP
-        cmp_list1 ::= expr DUP_TOP ROT_THREE COMPARE_OP JUMP_IF_FALSE_OR_POP
+                             compare_chained1 COME_FROM
-                      cmp_list2 COME_FROM
+        compare_chained1 ::= expr DUP_TOP ROT_THREE COMPARE_OP JUMP_IF_FALSE_OR_POP
+                             compare_chained2 COME_FROM
         """
 
     def p_stmt3(self, args):

uncompyle6/parsers/parse32.py

@@ -8,8 +8,9 @@ from uncompyle6.parsers.parse3 import Python3Parser
 class Python32Parser(Python3Parser):
     def p_32to35(self, args):
         """
-        conditional    ::= expr jmp_false expr jump_forward_else expr COME_FROM
+        conditional      ::= expr jmp_false expr jump_forward_else expr COME_FROM
-        cmp_list2      ::= expr COMPARE_OP RETURN_VALUE
+        # used exclusively in compare_chained
+        compare_chained2 ::= expr COMPARE_OP RETURN_VALUE
 
         # Store locals is only in Python 3.0 to 3.3
         stmt           ::= store_locals
@@ -55,7 +56,7 @@ class Python32Parser(Python3Parser):
 
     def add_custom_rules(self, tokens, customize):
         # self.remove_rules("""
-        #     cmp_list2 ::= expr COMPARE_OP RETURN_VALUE
+        #     compare_chained2 ::= expr COMPARE_OP RETURN_VALUE
         # """)
         super(Python32Parser, self).add_custom_rules(tokens, customize)
         for i, token in enumerate(tokens):

uncompyle6/semantics/consts.py

@@ -140,9 +140,9 @@ TABLE_DIRECT = {
     'binary_subscr':            ( '%c[%p]',
                                       (0, 'expr'),
                                       (1, 100) ),
-    'binary_subscr2':           ( '%c[%p]',
+    'binary_subscr2':           ( '%c[%c]',
                                       (0, 'expr'),
-                                      (1, 100) ),
+                                      (1, 'expr') ),
     'store_subscr':	        ( '%c[%c]', 0, 1),
     'STORE_FAST':	        ( '%{pattr}', ),
     'STORE_NAME':	        ( '%{pattr}', ),
@@ -192,10 +192,10 @@ TABLE_DIRECT = {
     'ret_cond_not':     ( '%p if not %p else %p', (2, 27), (0, 22), (-1, 27) ),
     'conditional_lambda':  ( '%c if %c else %c', 2, 0, 4),
 
-    'compare':		    ( '%p %[-1]{pattr.replace("-", " ")} %p', (0, 19), (1, 19) ),
+    'compare_single':	    ( '%p %[-1]{pattr.replace("-", " ")} %p', (0, 19), (1, 19) ),
-    'cmp_list':		    ( '%p %p', (0, 29), (1, 30)),
+    'compare_chained':	    ( '%p %p', (0, 29), (1, 30)),
-    'cmp_list1':	    ( '%[3]{pattr} %p %p', (0, 19), (-2, 19)),
+    'compare_chained1':	    ( '%[3]{pattr} %p %p', (0, 19), (-2, 19)),
-    'cmp_list2':	    ( '%[1]{pattr} %p', (0, 19)),
+    'compare_chained2':	    ( '%[1]{pattr} %p', (0, 19)),
 #   'classdef': 	(), # handled by n_classdef()
     'funcdef':  	    ( '\n\n%|def %c\n', -2), # -2 to handle closures
     'funcdefdeco':  	( '\n\n%c', 0),
@@ -340,7 +340,7 @@ PRECEDENCE = {
 
     'BINARY_OR':            18,
 
-    'cmp':                  20,
+    'compare':              20,
 
     'unary_not':            22,
 

uncompyle6/semantics/make_function.py

@@ -329,7 +329,6 @@ def make_function2(self, node, isLambda, nested=1, codeNode=None):
     else:
         defparams = node[:args_node.attr]
         kw_args  = 0
-        annotate_argc  = 0
         pass
 
     lambda_index = None
@@ -361,10 +360,14 @@ def make_function2(self, node, isLambda, nested=1, codeNode=None):
             self.ERROR = p
         return
 
+<<<<<<< HEAD
     if self.version >= 3.0:
         kw_pairs = args_node.attr[1]
     else:
         kw_pairs = 0
+=======
+    kw_pairs = 0
+>>>>>>> master
     indent = self.indent
 
     # build parameters
@@ -380,6 +383,22 @@ def make_function2(self, node, isLambda, nested=1, codeNode=None):
     # dump parameter list (with default values)
     if isLambda:
         self.write("lambda ", ", ".join(params))
+        # If the last statement is None (which is the
+        # same thing as "return None" in a lambda) and the
+        # next to last statement is a "yield". Then we want to
+        # drop the (return) None since that was just put there
+        # to have something to after the yield finishes.
+        # FIXME: this is a bit hoaky and not general
+        if (len(ast) > 1 and
+            self.traverse(ast[-1]) == 'None' and
+            self.traverse(ast[-2]).strip().startswith('yield')):
+            del ast[-1]
+            # Now pick out the expr part of the last statement
+            ast_expr = ast[-1]
+            while ast_expr.kind != 'expr':
+                ast_expr = ast_expr[0]
+            ast[-1] = ast_expr
+            pass
     else:
         self.write("(", ", ".join(params))
 
@@ -417,7 +436,8 @@ def make_function2(self, node, isLambda, nested=1, codeNode=None):
         print_docstring(self, indent, code.co_consts[0])
 
     code._tokens = None # save memory
-    assert ast == 'stmts'
+    if not isLambda:
+        assert ast == 'stmts'
 
     all_globals = find_all_globals(ast, set())
     for g in ((all_globals & self.mod_globs) | find_globals(ast, set())):
@@ -540,6 +560,25 @@ def make_function3(self, node, isLambda, nested=1, codeNode=None):
             self.write("(", ", ".join(params))
         # self.println(indent, '#flags:\t', int(code.co_flags))
 
+    # dump parameter list (with default values)
+    if isLambda:
+        self.write("lambda ", ", ".join(params))
+        # If the last statement is None (which is the
+        # same thing as "return None" in a lambda) and the
+        # next to last statement is a "yield". Then we want to
+        # drop the (return) None since that was just put there
+        # to have something to after the yield finishes.
+        # FIXME: this is a bit hoaky and not general
+        if (len(ast) > 1 and
+            self.traverse(ast[-1]) == 'None' and
+            self.traverse(ast[-2]).strip().startswith('yield')):
+            del ast[-1]
+            # Now pick out the expr part of the last statement
+            ast_expr = ast[-1]
+            while ast_expr.kind != 'expr':
+                ast_expr = ast_expr[0]
+            ast[-1] = ast_expr
+            pass
     else:
         if isLambda:
             self.write("lambda ")

uncompyle6/semantics/pysource.py

@@ -313,8 +313,10 @@ class SourceWalker(GenericASTTraversal, object):
                 'importmultiple': ( '%|import %c%c\n', 2, 3 ),
                 'import_cont'   : ( ', %c', 2 ),
                 # With/as is allowed as "from future" thing in 2.5
+                # Note: It is safe to put the variables after "as" in parenthesis,
+                # and sometimes it is needed.
                 'withstmt':     ( '%|with %c:\n%+%c%-', 0, 3),
-                'withasstmt':   ( '%|with %c as %c:\n%+%c%-', 0, 2, 3),
+                'withasstmt':   ( '%|with %c as (%c):\n%+%c%-', 0, 2, 3),
             })
 
         ########################################
@@ -804,7 +806,12 @@ class SourceWalker(GenericASTTraversal, object):
 
     def n_LOAD_CONST(self, node):
         data = node.pattr; datatype = type(data)
-        if isinstance(datatype, int) and data == minint:
+        if isinstance(data, float) and str(data) in frozenset(['nan', '-nan', 'inf', '-inf']):
+            # float values 'nan' and 'inf' are not directly representable in Python at least
+            # before 3.5 and even there it is via a library constant.
+            # So we will canonicalize their representation as float('nan') and float('inf')
+            self.write("float('%s')" % data)
+        elif isinstance(datatype, int) and data == minint:
             # convert to hex, since decimal representation
             # would result in 'LOAD_CONST; UNARY_NEGATIVE'
             # change:hG/2002-02-07: this was done for all negative integers
@@ -1484,6 +1491,11 @@ class SourceWalker(GenericASTTraversal, object):
             if hasattr(buildclass[-3][0], 'attr'):
                 subclass_code = buildclass[-3][0].attr
                 class_name = buildclass[0].pattr
+            elif (buildclass[-3] == 'mkfunc' and
+                  node == 'classdefdeco2' and
+                  buildclass[-3][0] == 'load_closure'):
+                subclass_code = buildclass[-3][1].attr
+                class_name = buildclass[-3][0][0].pattr
             elif hasattr(node[0][0], 'pattr'):
                 subclass_code = buildclass[-3][1].attr
                 class_name = node[0][0].pattr
@@ -1745,7 +1757,21 @@ class SourceWalker(GenericASTTraversal, object):
         if lastnodetype.startswith('BUILD_LIST'):
             self.write('['); endchar = ']'
         elif lastnodetype.startswith('BUILD_TUPLE'):
-            self.write('('); endchar = ')'
+            # Tuples can appear places that can NOT
+            # have parenthesis around them, like array
+            # subscripts. We check for that by seeing
+            # if a tuple item is some sort of slice.
+            no_parens = False
+            for n in node:
+                if n == 'expr' and n[0].kind.startswith('buildslice'):
+                    no_parens = True
+                    break
+                pass
+            if no_parens:
+                endchar = ''
+            else:
+                self.write('('); endchar = ')'
+                pass
         elif lastnodetype.startswith('BUILD_SET'):
             self.write('{'); endchar = '}'
         elif lastnodetype.startswith('BUILD_MAP_UNPACK'):