Merge branch 'master' into python-2.4

This commit is contained in:
rocky
2019-05-05 16:10:44 -04:00
19 changed files with 109 additions and 58 deletions

View File

@@ -269,6 +269,7 @@ check-bytecode-3.6:
#: Check deparsing Python 3.7
check-bytecode-3.7:
$(PYTHON) test_pythonlib.py --bytecode-3.7-run --verify-run
$(PYTHON) test_pythonlib.py --bytecode-3.7 --weak-verify
#: Check deparsing Python 3.8

Binary file not shown.

Binary file not shown.

Binary file not shown.

View File

@@ -22,7 +22,7 @@ assert i[0]('a') == True
assert i[0]('A') == False
# Issue #170. Bug is needing an "conditional_not_lambda" grammar rule
# in addition the the "conditional_lambda" rule
# in addition the the "if_expr_lambda" rule
j = lambda a: False if not a else True
assert j(True) == True
assert j(False) == False

View File

@@ -8,7 +8,7 @@ list(x for x in range(10) if x % 2 if x % 3)
# expresion which evaluates True unconditionally,
# but leave dead code or junk around that we have to match on.
# Tests "conditional_true" rule
# Tests "if_expr_true" rule
5 if 1 else 2
0 or max(5, 3) if 0 else 3

View File

@@ -0,0 +1,9 @@
# From 3.7.3 dataclasses.py
# Bug was handling precedence. Need parenthesis before IfExp.
#
# RUNNABLE!
def _hash_add(fields):
flds = [f for f in fields if (4 if f is None else f)]
return flds
assert _hash_add([None, True, False, 3]) == [None, True, 3]

View File

@@ -1,6 +1,6 @@
# Bug found in 2.7 test_itertools.py
# Bug was erroneously using reduction to unconditional_true
# A proper fix would be to use unconditional_true only when we
# Bug was erroneously using reduction to if_expr_true
# A proper fix would be to use if_expr_true only when we
# can determine there is or was dead code.
from itertools import izip_longest
for args in [['abc', range(6)]]:

View File

@@ -0,0 +1,16 @@
# From 3.7.3 base64.py
# Bug was handling "and not" in an
# if/else in the presence of better Python bytecode generatation
# RUNNABLE!
def foo(foldnuls, word):
x = 5 if foldnuls and not word else 6
return x
for expect, foldnuls, word in (
(6, True, True),
(5, True, False),
(6, False, True),
(6, False, False)
):
assert foo(foldnuls, word) == expect

View File

@@ -96,8 +96,8 @@ class Python2Parser(PythonParser):
for ::= SETUP_LOOP expr for_iter store
for_block POP_BLOCK _come_froms
del_stmt ::= delete_subscr
delete_subscr ::= expr expr DELETE_SUBSCR
del_stmt ::= delete_subscript
delete_subscript ::= expr expr DELETE_SUBSCR
del_stmt ::= expr DELETE_ATTR
_mklambda ::= load_closure mklambda
@@ -388,10 +388,10 @@ class Python2Parser(PythonParser):
continue
elif opname == 'DELETE_SUBSCR':
self.addRule("""
del_stmt ::= delete_subscr
delete_subscr ::= expr expr DELETE_SUBSCR
del_stmt ::= delete_subscript
delete_subscript ::= expr expr DELETE_SUBSCR
""", nop_func)
self.check_reduce['delete_subscr'] = 'AST'
self.check_reduce['delete_subscript'] = 'AST'
custom_seen_ops.add(opname)
continue
elif opname == 'GET_ITER':
@@ -547,7 +547,7 @@ class Python2Parser(PythonParser):
elif rule == ('or', ('expr', 'jmp_true', 'expr', '\\e_come_from_opt')):
expr2 = ast[2]
return expr2 == 'expr' and expr2[0] == 'LOAD_ASSERT'
elif lhs in ('delete_subscr', 'del_expr'):
elif lhs in ('delete_subscript', 'del_expr'):
op = ast[0][0]
return op.kind in ('and', 'or')

View File

@@ -82,9 +82,9 @@ class Python25Parser(Python26Parser):
return_stmt_lambda ::= ret_expr RETURN_VALUE_LAMBDA
setupwithas ::= DUP_TOP LOAD_ATTR ROT_TWO LOAD_ATTR CALL_FUNCTION_0 setup_finally
stmt ::= classdefdeco
stmt ::= conditional_lambda
stmt ::= if_expr_lambda
stmt ::= conditional_not_lambda
conditional_lambda ::= expr jmp_false_then expr return_if_lambda
if_expr_lambda ::= expr jmp_false_then expr return_if_lambda
return_stmt_lambda LAMBDA_MARKER
conditional_not_lambda
::= expr jmp_true_then expr return_if_lambda

View File

@@ -1,4 +1,4 @@
# Copyright (c) 2017-2018 Rocky Bernstein
# Copyright (c) 2017-2019 Rocky Bernstein
"""
spark grammar differences over Python2 for Python 2.6.
"""
@@ -293,19 +293,19 @@ class Python26Parser(Python2Parser):
compare_chained2 ::= expr COMPARE_OP return_lambda
return_if_lambda ::= RETURN_END_IF_LAMBDA POP_TOP
stmt ::= conditional_lambda
stmt ::= if_expr_lambda
stmt ::= conditional_not_lambda
conditional_lambda ::= expr jmp_false_then expr return_if_lambda
if_expr_lambda ::= expr jmp_false_then expr return_if_lambda
return_stmt_lambda LAMBDA_MARKER
conditional_not_lambda ::=
expr jmp_true_then expr return_if_lambda
return_stmt_lambda LAMBDA_MARKER
# conditional_true are for conditions which always evaluate true
# if_expr_true are for conditions which always evaluate true
# There is dead or non-optional remnants of the condition code though,
# and we use that to match on to reconstruct the source more accurately
expr ::= conditional_true
conditional_true ::= expr jf_pop expr COME_FROM
expr ::= if_expr_true
if_expr_true ::= expr jf_pop expr COME_FROM
# This comes from
# 0 or max(5, 3) if 0 else 3

View File

@@ -112,14 +112,14 @@ class Python27Parser(Python2Parser):
compare_chained2 ::= expr COMPARE_OP return_lambda
compare_chained2 ::= expr COMPARE_OP return_lambda
# conditional_true are for conditions which always evaluate true
# if_expr_true are for conditions which always evaluate true
# There is dead or non-optional remnants of the condition code though,
# and we use that to match on to reconstruct the source more accurately.
# FIXME: we should do analysis and reduce *only* if there is dead code?
# right now we check that expr is "or". Any other nodes types?
expr ::= conditional_true
conditional_true ::= expr JUMP_FORWARD expr COME_FROM
expr ::= if_expr_true
if_expr_true ::= expr JUMP_FORWARD expr COME_FROM
conditional ::= expr jmp_false expr JUMP_FORWARD expr COME_FROM
conditional ::= expr jmp_false expr JUMP_ABSOLUTE expr
@@ -181,9 +181,9 @@ class Python27Parser(Python2Parser):
# Common with 2.6
return_if_lambda ::= RETURN_END_IF_LAMBDA COME_FROM
stmt ::= conditional_lambda
stmt ::= if_expr_lambda
stmt ::= conditional_not_lambda
conditional_lambda ::= expr jmp_false expr return_if_lambda
if_expr_lambda ::= expr jmp_false expr return_if_lambda
return_stmt_lambda LAMBDA_MARKER
conditional_not_lambda
::= expr jmp_true expr return_if_lambda
@@ -216,7 +216,7 @@ class Python27Parser(Python2Parser):
self.check_reduce['raise_stmt1'] = 'AST'
self.check_reduce['list_if_not'] = 'AST'
self.check_reduce['list_if'] = 'AST'
self.check_reduce['conditional_true'] = 'AST'
self.check_reduce['if_expr_true'] = 'tokens'
self.check_reduce['whilestmt'] = 'tokens'
return
@@ -268,11 +268,8 @@ class Python27Parser(Python2Parser):
while (tokens[i] != 'JUMP_BACK'):
i -= 1
return tokens[i].attr != tokens[i-1].attr
# elif rule[0] == ('conditional_true'):
# # FIXME: the below is a hack: we check expr for
# # nodes that could have possibly been a been a Boolean.
# # We should also look for the presence of dead code.
# return ast[0] == 'expr' and ast[0] == 'or'
elif rule[0] == 'if_expr_true':
return (first) > 0 and tokens[first-1] == 'POP_JUMP_IF_FALSE'
return False

View File

@@ -325,9 +325,9 @@ class Python3Parser(PythonParser):
def p_stmt3(self, args):
"""
stmt ::= conditional_lambda
stmt ::= if_expr_lambda
stmt ::= conditional_not_lambda
conditional_lambda ::= expr jmp_false expr return_if_lambda
if_expr_lambda ::= expr jmp_false expr return_if_lambda
return_stmt_lambda LAMBDA_MARKER
conditional_not_lambda
::= expr jmp_true expr return_if_lambda
@@ -407,11 +407,11 @@ class Python3Parser(PythonParser):
# a JUMP_ABSOLUTE with no COME_FROM
conditional ::= expr jmp_false expr jump_absolute_else expr
# conditional_true are for conditions which always evaluate true
# if_expr_true are for conditions which always evaluate true
# There is dead or non-optional remnants of the condition code though,
# and we use that to match on to reconstruct the source more accurately
expr ::= conditional_true
conditional_true ::= expr JUMP_FORWARD expr COME_FROM
expr ::= if_expr_true
if_expr_true ::= expr JUMP_FORWARD expr COME_FROM
"""
@staticmethod
@@ -585,9 +585,9 @@ class Python3Parser(PythonParser):
stmt ::= assign2_pypy
assign3_pypy ::= expr expr expr store store store
assign2_pypy ::= expr expr store store
stmt ::= conditional_lambda
stmt ::= if_expr_lambda
stmt ::= conditional_not_lambda
conditional_lambda ::= expr jmp_false expr return_if_lambda
if_expr_lambda ::= expr jmp_false expr return_if_lambda
return_lambda LAMBDA_MARKER
conditional_not_lambda
::= expr jmp_true expr return_if_lambda
@@ -784,8 +784,8 @@ class Python3Parser(PythonParser):
custom_ops_processed.add(opname)
elif opname == 'DELETE_SUBSCR':
self.addRule("""
del_stmt ::= delete_subscr
delete_subscr ::= expr expr DELETE_SUBSCR
del_stmt ::= delete_subscript
delete_subscript ::= expr expr DELETE_SUBSCR
""", nop_func)
custom_ops_processed.add(opname)
elif opname == 'GET_ITER':

View File

@@ -106,7 +106,20 @@ class Python37Parser(Python36Parser):
compare_chained2c_37 ::= expr DUP_TOP ROT_THREE COMPARE_OP come_from_opt POP_JUMP_IF_FALSE
compare_chained2a_false_37 ELSE
jf_cfs ::= JUMP_FORWARD come_froms
ifelsestmt ::= testexpr c_stmts_opt jf_cfs else_suite opt_come_from_except
jmp_false37 ::= POP_JUMP_IF_FALSE COME_FROM
list_if ::= expr jmp_false37 list_iter
_ifstmts_jump ::= c_stmts_opt come_froms
and_not ::= expr jmp_false expr POP_JUMP_IF_TRUE
expr ::= if_exp_37a
expr ::= if_exp_37b
if_exp_37a ::= and_not expr JUMP_FORWARD COME_FROM expr COME_FROM
if_exp_37b ::= expr jmp_false expr POP_JUMP_IF_FALSE jump_forward_else expr
"""
def customize_grammar_rules(self, tokens, customize):

View File

@@ -35,6 +35,11 @@ else:
# Things at the top of this list below with low-value precidence will
# tend to have parenthesis around them. Things at the bottom
# of the list will tend not to have parenthesis around them.
# Note: The values in this table tend to be even value. Inside
# various templates we use odd values. Avoiding equal-precident comparisons
# avoids ambiguity what to do when the precedence is equal.
PRECEDENCE = {
'list': 0,
'dict': 0,
@@ -49,7 +54,7 @@ PRECEDENCE = {
'subscript': 2,
'subscript2': 2,
'store_subscript': 2,
'delete_subscr': 2,
'delete_subscript': 2,
'slice0': 2,
'slice1': 2,
'slice2': 2,
@@ -90,6 +95,7 @@ PRECEDENCE = {
'conditional_lamdba': 28,
'conditional_not_lamdba': 28,
'conditionalnot': 28,
'if_expr_true': 28,
'ret_cond': 28,
'_mklambda': 30,
@@ -221,7 +227,7 @@ TABLE_DIRECT = {
'DELETE_FAST': ( '%|del %{pattr}\n', ),
'DELETE_NAME': ( '%|del %{pattr}\n', ),
'DELETE_GLOBAL': ( '%|del %{pattr}\n', ),
'delete_subscr': ( '%|del %p[%c]\n',
'delete_subscript': ( '%|del %p[%c]\n',
(0, 'expr', PRECEDENCE['subscript']), (1, 'expr') ),
'subscript': ( '%p[%c]',
(0, 'expr', PRECEDENCE['subscript']),
@@ -252,7 +258,8 @@ TABLE_DIRECT = {
'list_iter': ( '%c', 0 ),
'list_for': ( ' for %c in %c%c', 2, 0, 3 ),
'list_if': ( ' if %c%c', 0, 2 ),
'list_if': ( ' if %p%c',
(0, 'expr', 27), 2 ),
'list_if_not': ( ' if not %p%c',
(0, 'expr', PRECEDENCE['unary_not']),
2 ),
@@ -281,19 +288,19 @@ TABLE_DIRECT = {
'and2': ( '%c', 3 ),
'or': ( '%c or %c', 0, 2 ),
'ret_or': ( '%c or %c', 0, 2 ),
'conditional': ( '%p if %p else %p', (2, 27), (0, 27), (4, 27) ),
'conditional_true': ( '%p if 1 else %p', (0, 27), (2, 27) ),
'conditional': ( '%p if %c else %c',
(2, 'expr', 27), 0, 4 ),
'if_expr_lambda': ( '%p if %c else %c',
(2, 'expr', 27), (0, 'expr'), 4 ),
'if_expr_true': ( '%p if 1 else %c', (0, 'expr', 27), 2 ),
'ret_cond': ( '%p if %p else %p', (2, 27), (0, 27), (-1, 27) ),
'conditional_not': ( '%p if not %p else %p',
(2, 27),
(0, "expr", PRECEDENCE['unary_not']),
(4, 27) ),
'conditional_lambda':
( '%c if %c else %c',
(2, 'expr'), 0, 4 ),
'conditional_not_lambda':
( '%c if not %c else %c',
(2, 'expr'), 0, 4 ),
( '%p if not %c else %c',
(2, 'expr', 27), 0, 4 ),
'compare_single': ( '%p %[-1]{pattr.replace("-", " ")} %p', (0, 19), (1, 19) ),
'compare_chained': ( '%p %p', (0, 29), (1, 30)),

View File

@@ -23,7 +23,12 @@ def customize_for_version37(self, version):
#######################
PRECEDENCE['attribute37'] = 2
PRECEDENCE['if_exp_37a'] = 28
PRECEDENCE['if_exp_37b'] = 28
TABLE_DIRECT.update({
'and_not': ( '%c and not %c',
(0, 'expr'), (2, 'expr') ),
'async_forelse_stmt': (
'%|async for %c in %c:\n%+%c%-%|else:\n%+%c%-\n\n',
(7, 'store'), (1, 'expr'), (17, 'for_block'), (25, 'else_suite') ),
@@ -54,5 +59,7 @@ def customize_for_version37(self, version):
(0, 19 ) ),
'compare_chained2c_37': (
'%[3]{pattr.replace("-", " ")} %p %p', (0, 19), (6, 19) ),
'if_exp_37a': ( '%p if %p else %p', (1, 'expr', 27), (0, 27), (4, 'expr', 27) ),
'if_exp_37b': ( '%p if %p else %p', (2, 'expr', 27), (0, 'expr', 27), (5, 'expr', 27) ),
})

View File

@@ -50,7 +50,7 @@ def find_globals_and_nonlocals(node, globs, nonlocals, code, version):
# # print("XXX", n.kind, global_ops)
# if isinstance(n, SyntaxTree):
# # FIXME: do I need a caser for n.kind="mkfunc"?
# if n.kind in ("conditional_lambda", "return_lambda"):
# if n.kind in ("if_expr_lambda", "return_lambda"):
# globs = find_globals(n, globs, mklambda_globals)
# else:
# globs = find_globals(n, globs, global_ops)

View File

@@ -611,13 +611,13 @@ class SourceWalker(GenericASTTraversal, object):
# LOAD_CONST is a terminal, so stop processing/recursing early
self.prune()
def n_delete_subscr(self, node):
def n_delete_subscript(self, node):
if node[-2][0] == 'build_list' and node[-2][0][-1].kind.startswith('BUILD_TUPLE'):
if node[-2][0][-1] != 'BUILD_TUPLE_0':
node[-2][0].kind = 'build_tuple2'
self.default(node)
n_store_subscript = n_subscript = n_delete_subscr
n_store_subscript = n_subscript = n_delete_subscript
# Note: this node is only in Python 2.x
# FIXME: figure out how to get this into customization
@@ -1168,6 +1168,7 @@ class SourceWalker(GenericASTTraversal, object):
self.write(' if ')
if have_not:
self.write('not ')
self.prec = 27
self.preorder(if_node)
pass
self.prec = p