Merge branch 'master' into python-2.4

This commit is contained in:
rocky
2017-11-24 21:48:14 -05:00
28 changed files with 221 additions and 38 deletions

Binary file not shown.

Binary file not shown.

Binary file not shown.

Binary file not shown.

Binary file not shown.

Binary file not shown.

Binary file not shown.

Binary file not shown.

Binary file not shown.

Binary file not shown.

View File

@@ -1,4 +1,5 @@
# In Python 3.3+ this uses grammar rule
# cmp_list2 ::= expr COMPARE_OP RETURN_VALUE
# compare_chained2 ::= expr COMPARE_OP RETURN_VALUE
def _is_valid_netmask(self, netmask):
return 0 <= netmask <= self._max_prefixlen

View File

@@ -0,0 +1,4 @@
a = 1e300 * 1e300 * 0
b = -1e300 * 1e300 * 0
c = 1e300 * 1e300
d = -1e300 * 1e300

View File

@@ -6,7 +6,12 @@ def some_other_function():
some_variable, = some_function()
print(some_variable)
# From 2.7 test_compile.py
# Bug is adding erroneous parens in d[(1:2, 1:2)] += 1
def bug(d):
d[1:2, 1:2] += 1
empty_tup = ()
one_item_tup = ("item1", )
one_item_tup_without_parentheses = "item",
one_item_tup_without_parentheses = "item",
many_items_tup = ("item1", "item2", "item3")

View File

@@ -26,3 +26,11 @@ class ExtendedInterpolation():
value_getter = lambda option: self._interpolation.before_get(self,
section, option, d[option], d)
return value_getter
# Bug from Python 2.7's test_collections.py
# is that the lambda function has two
# statements in it, one for returning *after* the yield
# The return None statement should be removed and the
# yield should be turned into a statement
def test_Iterable(self):
return (lambda: (yield))()

View File

@@ -10,3 +10,9 @@
def formatweekday(self):
with self as encoding:
return encoding
# Bug in 2.7.14 test_contextlib.py. Bug was not enclosing (x,y) in parenthesis
def withas_bug(self, nested, a, b):
with self.assertRaises(ZeroDivisionError):
with nested(a(), b()) as (x, y):
1 // 0

3
test/stdlib/README.md Normal file
View File

@@ -0,0 +1,3 @@
The programs in here are to test Python stdlib tests in lib/pythonX.Y/test
We'll compile a test, then decompile it and then run the test

23
test/stdlib/compile-file.py Executable file
View File

@@ -0,0 +1,23 @@
#!/usr/bin/env python
import sys
if len(sys.argv) != 2:
print("Usage: compile-file.py *python-file*")
sys.exit(1)
source = sys.argv[1]
assert source.endswith('.py')
basename = source[:-3]
# We do this crazy way to support Python 2.6 which
# doesn't support version_major, and has a bug in
# floating point so we can't divide 26 by 10 and get
# 2.6
PY_VERSION = sys.version_info[0] + (sys.version_info[1] / 10.0)
bytecode = "%s-%s.pyc" % (basename, PY_VERSION)
import py_compile
print("compiling %s to %s" % (source, bytecode))
py_compile.compile(source, bytecode, 'exec')
# import os
# os.system("../bin/uncompyle6 %s" % bytecode)

58
test/stdlib/runtests.sh Executable file
View File

@@ -0,0 +1,58 @@
#!/bin/bash
me=${BASH_SOURCE[0]}
# Python version setup
FULLVERSION=${1:-2.7.14}
PYVERSION=${FULLVERSION%.*}
MINOR=${FULLVERSION##?.?.}
typeset -i STOP_ONERROR=1
typeset -A SKIP_TESTS=( [test_aepack.py]=1 [audiotests.py]=1)
# Test directory setup
srcdir=$(dirname $me)
cd $srcdir
fulldir=$(pwd)
TESTDIR=/tmp/test${PYVERSION}
if [[ -e $TESTDIR ]] ; then
rm -fr $TESTDIR
fi
mkdir $TESTDIR || exit $?
cp -r ~/.pyenv/versions/${PYVERSION}.${MINOR}/lib/python${PYVERSION}/test $TESTDIR
cd $TESTDIR/test
export PYTHONPATH=$TESTDIR
# Run tests
typeset -i i=0
typeset -i allerrs=0
for file in test_*.py; do
[[ -v SKIP_TESTS[$file] ]] && continue
# If the fails *before* decompiling, skip it!
if ! python $file >/dev/null 2>&1 ; then
continue
fi
((i++))
# (( i > 40 )) && break
short_name=$(basename $file .py)
decompiled_file=$short_name-${PYVERSION}.pyc
$fulldir/compile-file.py $file && \
mv $file{,.orig} && \
$fulldir/../../bin/uncompyle6 $decompiled_file > $file
rc=$?
if (( rc == 0 )) ; then
echo ========== Running $file ===========
python $file
rc=$?
else
echo ======= Skipping $file due to compile/decompile errors ========
fi
(( rc != 0 && allerrs++ ))
if (( STOP_ONERROR && rc )) ; then
echo "** Ran $i tests before failure **"
exit $allerrs
fi
done
echo "Ran $i tests"
exit $allerrs

View File

@@ -442,7 +442,7 @@ class PythonParser(GenericASTBuilder):
expr ::= load_attr
expr ::= binary_expr
expr ::= build_list
expr ::= cmp
expr ::= compare
expr ::= mapexpr
expr ::= and
expr ::= or
@@ -508,14 +508,17 @@ class PythonParser(GenericASTBuilder):
return_lambda ::= ret_expr RETURN_VALUE_LAMBDA LAMBDA_MARKER
return_lambda ::= ret_expr RETURN_VALUE_LAMBDA
# Doesn't seemt to be used anymore, but other conditional_lambda's are
# Doesn't seem to be used anymore, but other conditional_lambda's are
# conditional_lambda ::= expr jmp_false return_if_stmt return_stmt LAMBDA_MARKER
cmp ::= cmp_list
cmp ::= compare
compare ::= expr expr COMPARE_OP
cmp_list ::= expr cmp_list1 ROT_TWO POP_TOP _come_from
cmp_list2 ::= expr COMPARE_OP JUMP_FORWARD
compare ::= compare_chained
compare ::= compare_single
compare_single ::= expr expr COMPARE_OP
# A compare_chained is two comparisions like x <= y <= z
compare_chained ::= expr compare_chained1 ROT_TWO POP_TOP _come_from
compare_chained2 ::= expr COMPARE_OP JUMP_FORWARD
mapexpr ::= BUILD_MAP kvlist
kvlist ::= kvlist kv

View File

@@ -201,6 +201,10 @@ class Python2Parser(PythonParser):
binary_subscr2 ::= expr expr DUP_TOPX_2 BINARY_SUBSCR
conditional ::= expr jmp_false expr JUMP_ABSOLUTE expr
# compare_chained2 is used in a "chained_compare": x <= y <= z
compare_chained2 ::= expr COMPARE_OP RETURN_VALUE
compare_chained2 ::= expr COMPARE_OP RETURN_VALUE_LAMBDA
"""
def p_slice2(self, args):

View File

@@ -243,10 +243,12 @@ class Python26Parser(Python2Parser):
def p_misc26(self, args):
"""
conditional ::= expr jmp_false expr jf_cf_pop expr come_from_opt
and ::= expr JUMP_IF_FALSE POP_TOP expr JUMP_IF_FALSE POP_TOP
cmp_list ::= expr cmp_list1 ROT_TWO COME_FROM POP_TOP _come_from
cmp_list1 ::= expr DUP_TOP ROT_THREE COMPARE_OP jmp_false cmp_list1 _come_from
cmp_list1 ::= expr DUP_TOP ROT_THREE COMPARE_OP jmp_false cmp_list2 _come_from
and ::= expr JUMP_IF_FALSE POP_TOP expr JUMP_IF_FALSE POP_TOP
# compare_chained is like x <= y <= z
compare_chained ::= expr compare_chained1 ROT_TWO COME_FROM POP_TOP _come_from
compare_chained1 ::= expr DUP_TOP ROT_THREE COMPARE_OP jmp_false compare_chained1 _come_from
compare_chained1 ::= expr DUP_TOP ROT_THREE COMPARE_OP jmp_false compare_chained2 _come_from
return_if_lambda ::= RETURN_END_IF_LAMBDA POP_TOP
conditional_lambda ::= expr jmp_false_then expr return_if_lambda

View File

@@ -77,12 +77,11 @@ class Python27Parser(Python2Parser):
or ::= expr JUMP_IF_TRUE_OR_POP expr COME_FROM
and ::= expr JUMP_IF_FALSE_OR_POP expr COME_FROM
cmp_list1 ::= expr DUP_TOP ROT_THREE
COMPARE_OP JUMP_IF_FALSE_OR_POP
cmp_list1 COME_FROM
cmp_list1 ::= expr DUP_TOP ROT_THREE
COMPARE_OP JUMP_IF_FALSE_OR_POP
cmp_list2 COME_FROM
# compare_chained1 is used exclusively in chained_compare
compare_chained1 ::= expr DUP_TOP ROT_THREE COMPARE_OP JUMP_IF_FALSE_OR_POP
compare_chained1 COME_FROM
compare_chained1 ::= expr DUP_TOP ROT_THREE COMPARE_OP JUMP_IF_FALSE_OR_POP
compare_chained2 COME_FROM
"""
def p_stmt27(self, args):

View File

@@ -332,10 +332,11 @@ class Python3Parser(PythonParser):
or ::= expr JUMP_IF_TRUE_OR_POP expr COME_FROM
and ::= expr JUMP_IF_FALSE_OR_POP expr COME_FROM
cmp_list1 ::= expr DUP_TOP ROT_THREE COMPARE_OP JUMP_IF_FALSE_OR_POP
cmp_list1 COME_FROM
cmp_list1 ::= expr DUP_TOP ROT_THREE COMPARE_OP JUMP_IF_FALSE_OR_POP
cmp_list2 COME_FROM
# compare_chained1 is used exclusively in chained_compare
compare_chained1 ::= expr DUP_TOP ROT_THREE COMPARE_OP JUMP_IF_FALSE_OR_POP
compare_chained1 COME_FROM
compare_chained1 ::= expr DUP_TOP ROT_THREE COMPARE_OP JUMP_IF_FALSE_OR_POP
compare_chained2 COME_FROM
"""
def p_stmt3(self, args):

View File

@@ -8,8 +8,9 @@ from uncompyle6.parsers.parse3 import Python3Parser
class Python32Parser(Python3Parser):
def p_32to35(self, args):
"""
conditional ::= expr jmp_false expr jump_forward_else expr COME_FROM
cmp_list2 ::= expr COMPARE_OP RETURN_VALUE
conditional ::= expr jmp_false expr jump_forward_else expr COME_FROM
# used exclusively in compare_chained
compare_chained2 ::= expr COMPARE_OP RETURN_VALUE
# Store locals is only in Python 3.0 to 3.3
stmt ::= store_locals
@@ -55,7 +56,7 @@ class Python32Parser(Python3Parser):
def add_custom_rules(self, tokens, customize):
# self.remove_rules("""
# cmp_list2 ::= expr COMPARE_OP RETURN_VALUE
# compare_chained2 ::= expr COMPARE_OP RETURN_VALUE
# """)
super(Python32Parser, self).add_custom_rules(tokens, customize)
for i, token in enumerate(tokens):

View File

@@ -140,9 +140,9 @@ TABLE_DIRECT = {
'binary_subscr': ( '%c[%p]',
(0, 'expr'),
(1, 100) ),
'binary_subscr2': ( '%c[%p]',
'binary_subscr2': ( '%c[%c]',
(0, 'expr'),
(1, 100) ),
(1, 'expr') ),
'store_subscr': ( '%c[%c]', 0, 1),
'STORE_FAST': ( '%{pattr}', ),
'STORE_NAME': ( '%{pattr}', ),
@@ -192,10 +192,10 @@ TABLE_DIRECT = {
'ret_cond_not': ( '%p if not %p else %p', (2, 27), (0, 22), (-1, 27) ),
'conditional_lambda': ( '%c if %c else %c', 2, 0, 4),
'compare': ( '%p %[-1]{pattr.replace("-", " ")} %p', (0, 19), (1, 19) ),
'cmp_list': ( '%p %p', (0, 29), (1, 30)),
'cmp_list1': ( '%[3]{pattr} %p %p', (0, 19), (-2, 19)),
'cmp_list2': ( '%[1]{pattr} %p', (0, 19)),
'compare_single': ( '%p %[-1]{pattr.replace("-", " ")} %p', (0, 19), (1, 19) ),
'compare_chained': ( '%p %p', (0, 29), (1, 30)),
'compare_chained1': ( '%[3]{pattr} %p %p', (0, 19), (-2, 19)),
'compare_chained2': ( '%[1]{pattr} %p', (0, 19)),
# 'classdef': (), # handled by n_classdef()
'funcdef': ( '\n\n%|def %c\n', -2), # -2 to handle closures
'funcdefdeco': ( '\n\n%c', 0),
@@ -340,7 +340,7 @@ PRECEDENCE = {
'BINARY_OR': 18,
'cmp': 20,
'compare': 20,
'unary_not': 22,

View File

@@ -329,7 +329,6 @@ def make_function2(self, node, isLambda, nested=1, codeNode=None):
else:
defparams = node[:args_node.attr]
kw_args = 0
annotate_argc = 0
pass
lambda_index = None
@@ -361,10 +360,14 @@ def make_function2(self, node, isLambda, nested=1, codeNode=None):
self.ERROR = p
return
<<<<<<< HEAD
if self.version >= 3.0:
kw_pairs = args_node.attr[1]
else:
kw_pairs = 0
=======
kw_pairs = 0
>>>>>>> master
indent = self.indent
# build parameters
@@ -380,6 +383,22 @@ def make_function2(self, node, isLambda, nested=1, codeNode=None):
# dump parameter list (with default values)
if isLambda:
self.write("lambda ", ", ".join(params))
# If the last statement is None (which is the
# same thing as "return None" in a lambda) and the
# next to last statement is a "yield". Then we want to
# drop the (return) None since that was just put there
# to have something to after the yield finishes.
# FIXME: this is a bit hoaky and not general
if (len(ast) > 1 and
self.traverse(ast[-1]) == 'None' and
self.traverse(ast[-2]).strip().startswith('yield')):
del ast[-1]
# Now pick out the expr part of the last statement
ast_expr = ast[-1]
while ast_expr.kind != 'expr':
ast_expr = ast_expr[0]
ast[-1] = ast_expr
pass
else:
self.write("(", ", ".join(params))
@@ -417,7 +436,8 @@ def make_function2(self, node, isLambda, nested=1, codeNode=None):
print_docstring(self, indent, code.co_consts[0])
code._tokens = None # save memory
assert ast == 'stmts'
if not isLambda:
assert ast == 'stmts'
all_globals = find_all_globals(ast, set())
for g in ((all_globals & self.mod_globs) | find_globals(ast, set())):
@@ -540,6 +560,25 @@ def make_function3(self, node, isLambda, nested=1, codeNode=None):
self.write("(", ", ".join(params))
# self.println(indent, '#flags:\t', int(code.co_flags))
# dump parameter list (with default values)
if isLambda:
self.write("lambda ", ", ".join(params))
# If the last statement is None (which is the
# same thing as "return None" in a lambda) and the
# next to last statement is a "yield". Then we want to
# drop the (return) None since that was just put there
# to have something to after the yield finishes.
# FIXME: this is a bit hoaky and not general
if (len(ast) > 1 and
self.traverse(ast[-1]) == 'None' and
self.traverse(ast[-2]).strip().startswith('yield')):
del ast[-1]
# Now pick out the expr part of the last statement
ast_expr = ast[-1]
while ast_expr.kind != 'expr':
ast_expr = ast_expr[0]
ast[-1] = ast_expr
pass
else:
if isLambda:
self.write("lambda ")

View File

@@ -313,8 +313,10 @@ class SourceWalker(GenericASTTraversal, object):
'importmultiple': ( '%|import %c%c\n', 2, 3 ),
'import_cont' : ( ', %c', 2 ),
# With/as is allowed as "from future" thing in 2.5
# Note: It is safe to put the variables after "as" in parenthesis,
# and sometimes it is needed.
'withstmt': ( '%|with %c:\n%+%c%-', 0, 3),
'withasstmt': ( '%|with %c as %c:\n%+%c%-', 0, 2, 3),
'withasstmt': ( '%|with %c as (%c):\n%+%c%-', 0, 2, 3),
})
########################################
@@ -804,7 +806,12 @@ class SourceWalker(GenericASTTraversal, object):
def n_LOAD_CONST(self, node):
data = node.pattr; datatype = type(data)
if isinstance(datatype, int) and data == minint:
if isinstance(data, float) and str(data) in frozenset(['nan', '-nan', 'inf', '-inf']):
# float values 'nan' and 'inf' are not directly representable in Python at least
# before 3.5 and even there it is via a library constant.
# So we will canonicalize their representation as float('nan') and float('inf')
self.write("float('%s')" % data)
elif isinstance(datatype, int) and data == minint:
# convert to hex, since decimal representation
# would result in 'LOAD_CONST; UNARY_NEGATIVE'
# change:hG/2002-02-07: this was done for all negative integers
@@ -1484,6 +1491,11 @@ class SourceWalker(GenericASTTraversal, object):
if hasattr(buildclass[-3][0], 'attr'):
subclass_code = buildclass[-3][0].attr
class_name = buildclass[0].pattr
elif (buildclass[-3] == 'mkfunc' and
node == 'classdefdeco2' and
buildclass[-3][0] == 'load_closure'):
subclass_code = buildclass[-3][1].attr
class_name = buildclass[-3][0][0].pattr
elif hasattr(node[0][0], 'pattr'):
subclass_code = buildclass[-3][1].attr
class_name = node[0][0].pattr
@@ -1745,7 +1757,21 @@ class SourceWalker(GenericASTTraversal, object):
if lastnodetype.startswith('BUILD_LIST'):
self.write('['); endchar = ']'
elif lastnodetype.startswith('BUILD_TUPLE'):
self.write('('); endchar = ')'
# Tuples can appear places that can NOT
# have parenthesis around them, like array
# subscripts. We check for that by seeing
# if a tuple item is some sort of slice.
no_parens = False
for n in node:
if n == 'expr' and n[0].kind.startswith('buildslice'):
no_parens = True
break
pass
if no_parens:
endchar = ''
else:
self.write('('); endchar = ')'
pass
elif lastnodetype.startswith('BUILD_SET'):
self.write('{'); endchar = '}'
elif lastnodetype.startswith('BUILD_MAP_UNPACK'):