Merge branch 'master' into python-2.4

This commit is contained in:
rocky
2017-11-23 12:37:00 -05:00
19 changed files with 128 additions and 104 deletions

View File

@@ -119,8 +119,14 @@ grammar-coverage-2.6:
#: Get grammar coverage for Python 2.7
grammar-coverage-2.7:
SPARK_PARSER_COVERAGE=/tmp/spark-grammar-27.cover $(PYTHON) test_pythonlib.py --bytecode-2.7
SPARK_PARSER_COVERAGE=/tmp/spark-grammar-27.cover $(PYTHON) test_pyenvlib.py --2.7.13
SPARK_PARSER_COVERAGE=$(COVER_DIR)/spark-grammar-27.cover $(PYTHON) test_pythonlib.py --bytecode-2.7
SPARK_PARSER_COVERAGE=$(COVER_DIR)/spark-grammar-27.cover $(PYTHON) test_pyenvlib.py --2.7.13
#: Get grammar coverage for Python 3.2
grammar-coverage-3.2:
rm $(COVER_DIR)/spark-grammar-32.cover || /bin/true
SPARK_PARSER_COVERAGE=$(COVER_DIR)/spark-grammar-32.cover $(PYTHON) test_pythonlib.py --bytecode-3.2
SPARK_PARSER_COVERAGE=$(COVER_DIR)/spark-grammar-32.cover $(PYTHON) test_pyenvlib.py --3.2.6
#: Check deparsing Python 2.6
pcheck-bytecode-2.6:

Binary file not shown.

Binary file not shown.

Binary file not shown.

Binary file not shown.

Binary file not shown.

Binary file not shown.

View File

@@ -0,0 +1,16 @@
# Statements to beef up grammar coverage rules
# Force "inplace" ops
y = +10 # UNARY_POSITIVE
y /= 1 # INPLACE_DIVIDE
y %= 4 # INPLACE_MODULO
y **= 1 # INPLACE POWER
y >>= 2 # INPLACE_RSHIFT
y <<= 2 # INPLACE_LSHIFT
y //= 1 # INPLACE_TRUE_DIVIDE
y &= 1 # INPLACE_AND
y ^= 1 # INPLACE_XOR
# Beef up augassign and STORE_SLICE+3
x = [1,2,3,4,5]
x[0:1] = 1
x[0:3] += 1, 2, 3

View File

@@ -0,0 +1,4 @@
# In Python 3.3+ this uses grammar rule
# cmp_list2 ::= expr COMPARE_OP RETURN_VALUE
def _is_valid_netmask(self, netmask):
return 0 <= netmask <= self._max_prefixlen

View File

@@ -1,3 +1,4 @@
# Tests BINARY_TRUE_DIVIDE
# Tests BINARY_TRUE_DIVIDE and INPLACE_TRUE_DIVIDE
from __future__ import division # needed on 2.6 and 2.7
x = len(__file__) / 1
x /= 1

View File

@@ -203,7 +203,6 @@ class PythonParser(GenericASTBuilder):
c_stmts ::= continue_stmts
lastc_stmt ::= iflaststmt
lastc_stmt ::= whileelselaststmt
lastc_stmt ::= forelselaststmt
lastc_stmt ::= ifelsestmtr
lastc_stmt ::= ifelsestmtc
@@ -376,25 +375,6 @@ class PythonParser(GenericASTBuilder):
for_block POP_BLOCK else_suitel _come_from
"""
def p_whilestmt(self, args):
"""
whilestmt ::= SETUP_LOOP testexpr l_stmts_opt JUMP_BACK
POP_BLOCK _come_from
whilestmt ::= SETUP_LOOP testexpr return_stmts
POP_BLOCK COME_FROM
whileelsestmt ::= SETUP_LOOP testexpr
l_stmts_opt JUMP_BACK
POP_BLOCK
else_suite COME_FROM
whileelselaststmt ::= SETUP_LOOP testexpr
l_stmts_opt JUMP_BACK
POP_BLOCK
else_suitec COME_FROM
"""
def p_import20(self, args):
"""
stmt ::= importstmt
@@ -510,10 +490,6 @@ class PythonParser(GenericASTBuilder):
expr ::= conditional
conditional ::= expr jmp_false expr JUMP_FORWARD expr COME_FROM
conditional ::= expr jmp_false expr JUMP_ABSOLUTE expr
expr ::= conditionalnot
conditionalnot ::= expr jmp_true expr _jump expr COME_FROM
expr ::= conditionalTrue
conditionalTrue ::= expr JUMP_FORWARD expr COME_FROM
@@ -532,21 +508,14 @@ class PythonParser(GenericASTBuilder):
return_lambda ::= ret_expr RETURN_VALUE_LAMBDA LAMBDA_MARKER
return_lambda ::= ret_expr RETURN_VALUE_LAMBDA
conditional_lambda ::= expr jmp_false return_if_stmt return_stmt LAMBDA_MARKER
# Doesn't seemt to be used anymore, but other conditional_lambda's are
# conditional_lambda ::= expr jmp_false return_if_stmt return_stmt LAMBDA_MARKER
cmp ::= cmp_list
cmp ::= compare
compare ::= expr expr COMPARE_OP
cmp_list ::= expr cmp_list1 ROT_TWO POP_TOP
_come_from
cmp_list1 ::= expr DUP_TOP ROT_THREE
COMPARE_OP jmp_false
cmp_list1 _come_from
cmp_list1 ::= expr DUP_TOP ROT_THREE
COMPARE_OP jmp_false
cmp_list2 _come_from
cmp_list ::= expr cmp_list1 ROT_TWO POP_TOP _come_from
cmp_list2 ::= expr COMPARE_OP JUMP_FORWARD
cmp_list2 ::= expr COMPARE_OP RETURN_VALUE
mapexpr ::= BUILD_MAP kvlist
kvlist ::= kvlist kv

View File

@@ -38,10 +38,6 @@ class Python2Parser(PythonParser):
def p_stmt2(self, args):
"""
# Note: these are removed in 2.7
while1stmt ::= SETUP_LOOP l_stmts JUMP_BACK COME_FROM
while1elsestmt ::= SETUP_LOOP l_stmts JUMP_BACK else_suite COME_FROM
exec_stmt ::= expr exprlist DUP_TOP EXEC_STMT
exec_stmt ::= expr exprlist EXEC_STMT
@@ -203,6 +199,8 @@ class Python2Parser(PythonParser):
# In Python 3, DUP_TOPX_2 is DUP_TOP_TWO
binary_subscr2 ::= expr expr DUP_TOPX_2 BINARY_SUBSCR
conditional ::= expr jmp_false expr JUMP_ABSOLUTE expr
"""
def p_slice2(self, args):

View File

@@ -53,7 +53,6 @@ class Python25Parser(Python26Parser):
POP_BLOCK LOAD_CONST COME_FROM WITH_CLEANUP END_FINALLY
withasstmt ::= expr setupwithas designator suite_stmts_opt
POP_BLOCK LOAD_CONST COME_FROM WITH_CLEANUP END_FINALLY
whilestmt ::= SETUP_LOOP testexpr return_stmts come_froms POP_TOP bp_come_from
assert2 ::= assert_expr jmp_true LOAD_ASSERT expr CALL_FUNCTION_1 RAISE_VARARGS_1
""")
super(Python25Parser, self).add_custom_rules(tokens, customize)

View File

@@ -126,27 +126,28 @@ class Python26Parser(Python2Parser):
POP_BLOCK LOAD_CONST COME_FROM WITH_CLEANUP END_FINALLY
# Semantic actions want designator to be at index 2
# Rule is possibly 2.6 only
withasstmt ::= expr setupwithas designator suite_stmts_opt
POP_BLOCK LOAD_CONST COME_FROM WITH_CLEANUP END_FINALLY
# This is truly weird. 2.7 does this (not including POP_TOP) with
# opcode SETUP_WITH
setupwith ::= DUP_TOP LOAD_ATTR ROT_TWO LOAD_ATTR CALL_FUNCTION_0 POP_TOP
# Possibly 2.6 only
setupwithas ::= DUP_TOP LOAD_ATTR ROT_TWO LOAD_ATTR CALL_FUNCTION_0 setup_finally
setupwith ::= DUP_TOP LOAD_ATTR ROT_TWO LOAD_ATTR CALL_FUNCTION_0 POP_TOP
setupwithas ::= DUP_TOP LOAD_ATTR ROT_TWO LOAD_ATTR CALL_FUNCTION_0 setup_finally
setup_finally ::= STORE_FAST SETUP_FINALLY LOAD_FAST DELETE_FAST
setup_finally ::= STORE_NAME SETUP_FINALLY LOAD_NAME DELETE_NAME
while1stmt ::= SETUP_LOOP l_stmts JUMP_BACK COME_FROM
whilestmt ::= SETUP_LOOP testexpr l_stmts_opt jb_pop POP_BLOCK _come_from
whilestmt ::= SETUP_LOOP testexpr l_stmts_opt jb_cf_pop bp_come_from
whilestmt ::= SETUP_LOOP testexpr return_stmts come_froms POP_TOP bp_come_from
whilestmt ::= SETUP_LOOP testexpr l_stmts_opt jb_pop POP_BLOCK _come_from
whilestmt ::= SETUP_LOOP testexpr l_stmts_opt jb_cf_pop bp_come_from
whilestmt ::= SETUP_LOOP testexpr return_stmts come_froms POP_TOP bp_come_from
whilestmt ::= SETUP_LOOP testexpr return_stmts POP_BLOCK COME_FROM
whileelsestmt ::= SETUP_LOOP testexpr l_stmts_opt jb_pop POP_BLOCK
else_suite COME_FROM
whileelsestmt ::= SETUP_LOOP testexpr l_stmts_opt jb_pop POP_BLOCK
else_suite COME_FROM
while1elsestmt ::= SETUP_LOOP l_stmts JUMP_BACK else_suite COME_FROM
return_stmt ::= ret_expr RETURN_END_IF POP_TOP
return_stmt ::= ret_expr RETURN_VALUE POP_TOP
@@ -243,7 +244,9 @@ class Python26Parser(Python2Parser):
"""
conditional ::= expr jmp_false expr jf_cf_pop expr come_from_opt
and ::= expr JUMP_IF_FALSE POP_TOP expr JUMP_IF_FALSE POP_TOP
cmp_list ::= expr cmp_list1 ROT_TWO COME_FROM POP_TOP _come_from
cmp_list ::= expr cmp_list1 ROT_TWO COME_FROM POP_TOP _come_from
cmp_list1 ::= expr DUP_TOP ROT_THREE COMPARE_OP jmp_false cmp_list1 _come_from
cmp_list1 ::= expr DUP_TOP ROT_THREE COMPARE_OP jmp_false cmp_list2 _come_from
return_if_lambda ::= RETURN_END_IF_LAMBDA POP_TOP
conditional_lambda ::= expr jmp_false_then expr return_if_lambda

View File

@@ -102,9 +102,13 @@ class Python27Parser(Python2Parser):
POP_BLOCK LOAD_CONST COME_FROM_WITH
WITH_CLEANUP END_FINALLY
while1stmt ::= SETUP_LOOP return_stmts bp_come_from
while1elsestmt ::= SETUP_LOOP l_stmts JUMP_BACK POP_BLOCK else_suite COME_FROM
while1stmt ::= SETUP_LOOP l_stmts_opt JUMP_BACK POP_BLOCK COME_FROM
while1stmt ::= SETUP_LOOP return_stmts bp_come_from
while1stmt ::= SETUP_LOOP l_stmts_opt JUMP_BACK POP_BLOCK COME_FROM
whilestmt ::= SETUP_LOOP testexpr l_stmts_opt JUMP_BACK POP_BLOCK _come_from
while1elsestmt ::= SETUP_LOOP l_stmts JUMP_BACK POP_BLOCK
else_suite COME_FROM
whileelsestmt ::= SETUP_LOOP testexpr l_stmts_opt JUMP_BACK POP_BLOCK
else_suite COME_FROM
# Common with 2.6
return_if_lambda ::= RETURN_END_IF_LAMBDA COME_FROM

View File

@@ -277,8 +277,6 @@ class Python3Parser(PythonParser):
for_block ::= l_stmts_opt opt_come_from_loop JUMP_BACK
for_block ::= l_stmts
iflaststmtl ::= testexpr c_stmts_opt
iflaststmt ::= testexpr c_stmts_opt33
c_stmts_opt33 ::= JUMP_BACK JUMP_ABSOLUTE c_stmts_opt
"""
def p_def_annotations3(self, args):
@@ -334,12 +332,10 @@ class Python3Parser(PythonParser):
or ::= expr JUMP_IF_TRUE_OR_POP expr COME_FROM
and ::= expr JUMP_IF_FALSE_OR_POP expr COME_FROM
cmp_list1 ::= expr DUP_TOP ROT_THREE
COMPARE_OP JUMP_IF_FALSE_OR_POP
cmp_list1 COME_FROM
cmp_list1 ::= expr DUP_TOP ROT_THREE
COMPARE_OP JUMP_IF_FALSE_OR_POP
cmp_list2 COME_FROM
cmp_list1 ::= expr DUP_TOP ROT_THREE COMPARE_OP JUMP_IF_FALSE_OR_POP
cmp_list1 COME_FROM
cmp_list1 ::= expr DUP_TOP ROT_THREE COMPARE_OP JUMP_IF_FALSE_OR_POP
cmp_list2 COME_FROM
"""
def p_stmt3(self, args):
@@ -385,9 +381,6 @@ class Python3Parser(PythonParser):
whileelsestmt ::= SETUP_LOOP testexpr l_stmts_opt JUMP_BACK POP_BLOCK
else_suite COME_FROM_LOOP
whileelselaststmt ::= SETUP_LOOP testexpr l_stmts_opt JUMP_BACK POP_BLOCK
else_suitec COME_FROM_LOOP
whileTruestmt ::= SETUP_LOOP l_stmts_opt JUMP_BACK POP_BLOCK
COME_FROM_LOOP
@@ -420,7 +413,7 @@ class Python3Parser(PythonParser):
def p_expr3(self, args):
"""
conditional ::= expr jmp_false expr jump_forward_else expr COME_FROM
expr ::= conditionalnot
conditionalnot ::= expr jmp_true expr jump_forward_else expr COME_FROM
# a JUMP_FORWARD to another JUMP_FORWARD can get turned into
@@ -437,7 +430,10 @@ class Python3Parser(PythonParser):
@staticmethod
def call_fn_name(token):
"""Customize CALL_FUNCTION to add the number of positional arguments"""
return '%s_%i' % (token.kind, token.attr)
if token.attr is not None:
return '%s_%i' % (token.kind, token.attr)
else:
return '%s_0' % (token.kind)
def custom_build_class_rule(self, opname, i, token, tokens, customize):
'''
@@ -620,8 +616,9 @@ class Python3Parser(PythonParser):
call_function ::= expr CALL_METHOD
"""
seen_LOAD_BUILD_CLASS = False
seen_LOAD_DICTCOMP = False
seen_LOAD_LISTCOMP = False
seen_LOAD_DICTCOMP = False
seen_LOAD_LISTCOMP = False
seen_LOAD_SETCOMP = False
# Loop over instructions adding custom grammar rules based on
# a specific instruction seen.
@@ -634,6 +631,13 @@ class Python3Parser(PythonParser):
assign2_pypy ::= expr expr designator designator
""", nop_func)
has_get_iter_call_function1 = False
n = len(tokens)
for i, token in enumerate(tokens):
if token == 'GET_ITER' and i < n-2 and self.call_fn_name(tokens[i+1]) == 'CALL_FUNCTION_1':
has_get_iter_call_function1 = True
break
for i, token in enumerate(tokens):
opname = token.kind
opname_base = opname[:opname.rfind('_')]
@@ -766,17 +770,20 @@ class Python3Parser(PythonParser):
continue
elif opname == 'LOAD_DICTCOMP':
seen_LOAD_DICTCOMP = True
rule_pat = ("dictcomp ::= LOAD_DICTCOMP %sMAKE_FUNCTION_0 expr "
"GET_ITER CALL_FUNCTION_1")
self.add_make_function_rule(rule_pat, opname, token.attr, customize)
if has_get_iter_call_function1:
rule_pat = ("dictcomp ::= LOAD_DICTCOMP %sMAKE_FUNCTION_0 expr "
"GET_ITER CALL_FUNCTION_1")
self.add_make_function_rule(rule_pat, opname, token.attr, customize)
elif opname == 'LOAD_LISTCOMP':
seen_LOAD_LISTCOMP = True
continue
elif opname == 'LOAD_SETCOMP':
seen_LOAD_SETCOMP = True
# Should this be generalized and put under MAKE_FUNCTION?
rule_pat = ("setcomp ::= LOAD_SETCOMP %sMAKE_FUNCTION_0 expr "
"GET_ITER CALL_FUNCTION_1")
self.add_make_function_rule(rule_pat, opname, token.attr, customize)
if has_get_iter_call_function1:
rule_pat = ("setcomp ::= LOAD_SETCOMP %sMAKE_FUNCTION_0 expr "
"GET_ITER CALL_FUNCTION_1")
self.add_make_function_rule(rule_pat, opname, token.attr, customize)
elif opname == 'LOOKUP_METHOD':
# A PyPy speciality - DRY with parse2
self.add_unique_rule("load_attr ::= expr LOOKUP_METHOD",
@@ -787,24 +794,27 @@ class Python3Parser(PythonParser):
# Note: this probably doesn't handle kwargs proprerly
args_pos, args_kw, annotate_args = token.attr
rule_pat = ("genexpr ::= %sload_closure load_genexpr %%s%s expr "
"GET_ITER CALL_FUNCTION_1" % ('pos_arg '* args_pos, opname))
self.add_make_function_rule(rule_pat, opname, token.attr, customize)
rule_pat = ('mklambda ::= %sload_closure LOAD_LAMBDA %%s%s' %
('pos_arg '* args_pos, opname))
self.add_make_function_rule(rule_pat, opname, token.attr, customize)
rule_pat = ('listcomp ::= %sload_closure LOAD_LISTCOMP %%s%s expr '
'GET_ITER CALL_FUNCTION_1' % ('pos_arg ' * args_pos, opname))
self.add_make_function_rule(rule_pat, opname, token.attr, customize)
rule_pat = ('setcomp ::= %sload_closure LOAD_SETCOMP %%s%s expr '
'GET_ITER CALL_FUNCTION_1' % ('pos_arg ' * args_pos, opname))
self.add_make_function_rule(rule_pat, opname, token.attr, customize)
if seen_LOAD_DICTCOMP:
self.add_unique_rule('dictcomp ::= %sload_closure LOAD_DICTCOMP %s '
'expr GET_ITER CALL_FUNCTION_1' %
('pos_arg '* args_pos, opname),
opname, token.attr, customize)
if has_get_iter_call_function1:
rule_pat = ("genexpr ::= %sload_closure load_genexpr %%s%s expr "
"GET_ITER CALL_FUNCTION_1" % ('pos_arg '* args_pos, opname))
self.add_make_function_rule(rule_pat, opname, token.attr, customize)
if seen_LOAD_LISTCOMP:
rule_pat = ('listcomp ::= %sload_closure LOAD_LISTCOMP %%s%s expr '
'GET_ITER CALL_FUNCTION_1' % ('pos_arg ' * args_pos, opname))
self.add_make_function_rule(rule_pat, opname, token.attr, customize)
if seen_LOAD_SETCOMP:
rule_pat = ('setcomp ::= %sload_closure LOAD_SETCOMP %%s%s expr '
'GET_ITER CALL_FUNCTION_1' % ('pos_arg ' * args_pos, opname))
self.add_make_function_rule(rule_pat, opname, token.attr, customize)
if seen_LOAD_DICTCOMP:
self.add_unique_rule('dictcomp ::= %sload_closure LOAD_DICTCOMP %s '
'expr GET_ITER CALL_FUNCTION_1' %
('pos_arg '* args_pos, opname),
opname, token.attr, customize)
# FIXME: kwarg processing is missing here.
# Note order of kwargs and pos args changed between 3.3-3.4
@@ -847,7 +857,7 @@ class Python3Parser(PythonParser):
('kwarg '* args_kw),
opname))
self.add_make_function_rule(rule_pat, opname, token.attr, customize)
if seen_LOAD_LISTCOMP:
if seen_LOAD_LISTCOMP and has_get_iter_call_function1:
rule_pat = ("listcomp ::= %sLOAD_LISTCOMP %%s%s expr "
"GET_ITER CALL_FUNCTION_1" % ('expr ' * args_pos, opname))
self.add_make_function_rule(rule_pat, opname, token.attr, customize)
@@ -857,17 +867,20 @@ class Python3Parser(PythonParser):
else:
args_pos, args_kw, annotate_args, closure = token.attr
rule_pat = ("genexpr ::= %sload_genexpr %%s%s expr "
"GET_ITER CALL_FUNCTION_1" % ('pos_arg '* args_pos, opname))
self.add_make_function_rule(rule_pat, opname, token.attr, customize)
if has_get_iter_call_function1:
rule_pat = ("genexpr ::= %sload_genexpr %%s%s expr "
"GET_ITER CALL_FUNCTION_1" % ('pos_arg '* args_pos, opname))
self.add_make_function_rule(rule_pat, opname, token.attr, customize)
rule_pat = ('mklambda ::= %s%sLOAD_LAMBDA %%s%s' %
(('pos_arg '* args_pos),
('kwarg '* args_kw),
opname))
self.add_make_function_rule(rule_pat, opname, token.attr, customize)
rule_pat = ("listcomp ::= %sLOAD_LISTCOMP %%s%s expr "
"GET_ITER CALL_FUNCTION_1" % ('expr ' * args_pos, opname))
self.add_make_function_rule(rule_pat, opname, token.attr, customize)
if seen_LOAD_LISTCOMP and has_get_iter_call_function1:
rule_pat = ("listcomp ::= %sLOAD_LISTCOMP %%s%s expr "
"GET_ITER CALL_FUNCTION_1" % ('expr ' * args_pos, opname))
self.add_make_function_rule(rule_pat, opname, token.attr, customize)
if self.version == 3.3:
# positional args after keyword args

View File

@@ -1,4 +1,4 @@
# Copyright (c) 2016 Rocky Bernstein
# Copyright (c) 2016-2017 Rocky Bernstein
"""
spark grammar differences over Python 3 for Python 3.2.
"""
@@ -8,15 +8,15 @@ from uncompyle6.parsers.parse3 import Python3Parser
class Python32Parser(Python3Parser):
def p_32to35(self, args):
"""
conditional ::= expr jmp_false expr jump_forward_else expr COME_FROM
# Store locals is only in Python 3.0 to 3.3
stmt ::= store_locals
store_locals ::= LOAD_FAST STORE_LOCALS
stmt ::= store_locals
store_locals ::= LOAD_FAST STORE_LOCALS
# Python < 3.5 no POP BLOCK
whileTruestmt ::= SETUP_LOOP l_stmts_opt JUMP_BACK
COME_FROM_LOOP
whileTruestmt ::= SETUP_LOOP return_stmts
COME_FROM_LOOP
whileTruestmt ::= SETUP_LOOP l_stmts_opt JUMP_BACK COME_FROM_LOOP
whileTruestmt ::= SETUP_LOOP return_stmts COME_FROM_LOOP
try_middle ::= JUMP_FORWARD COME_FROM_EXCEPT except_stmts
END_FINALLY
@@ -40,6 +40,9 @@ class Python32Parser(Python3Parser):
pass
def add_custom_rules(self, tokens, customize):
# self.remove_rules("""
# cmp_list2 ::= expr COMPARE_OP RETURN_VALUE
# """)
super(Python32Parser, self).add_custom_rules(tokens, customize)
for i, token in enumerate(tokens):
opname = token.kind

View File

@@ -14,6 +14,8 @@ class Python33Parser(Python32Parser):
expr ::= yield_from
yield_from ::= expr expr YIELD_FROM
cmp_list2 ::= expr COMPARE_OP RETURN_VALUE
# We do the grammar hackery below for semantics
# actions that want c_stmts_opt at index 1

View File

@@ -17,6 +17,12 @@ class Python34Parser(Python33Parser):
"""
expr ::= LOAD_ASSERT
# FIXME the below masks a bug in not detecting COME_FROM_LOOP
# grammar rules with COME_FROM -> COME_FROM_LOOP alreadly exist
whileelsestmt ::= SETUP_LOOP testexpr l_stmts_opt JUMP_BACK POP_BLOCK
else_suite COME_FROM
# Python 3.4+ optimizes the trailing two JUMPS away
# Is this 3.4 only?